diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 00000000..e62d5cd2 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,63 @@ +name: Docs Build + +on: + push: + branches: [ main ] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Install uv + run: pip install uv + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/uv + key: uv-${{ hashFiles('pyproject.toml') }} + + - name: Install dependencies + run: | + uv venv + uv pip install -e ".[dev]" + + - name: Build docs + run: uv run poe doc-build + continue-on-error: false + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/_build + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/lint_and_docs.yaml b/.github/workflows/lint_and_docs.yaml deleted file mode 100644 index daef60c8..00000000 --- a/.github/workflows/lint_and_docs.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: Linting, Types and Docs Check - -on: - workflow_dispatch: # Manual trigger only - workflow disabled from auto-running - -jobs: - check: - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - id-token: write - pages: write - actions: write - contents: read - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.11 - - - # use uv package manager - - name: Install uv - run: pip install uv - - uses: actions/cache@v3 - name: Cache dependencies - with: - path: ~/.cache/uv - key: uv-${{ hashFiles('pyproject.toml') }} - - name: Install dependencies - run: | - uv venv - uv pip install -e ".[dev]" - - name: Lint - run: uv run poe lint - - name: Types - run: uv run poe type-check - - name: Docs - run: uv run poe doc-build - - name: Upload artifact - uses: actions/upload-pages-artifact@v2 - with: - path: "docs/_build" - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v2 \ No newline at end of file diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index e0cac399..67f38ffe 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -21,6 +21,18 @@ jobs: python-version: ["3.11"] steps: - uses: actions/checkout@v3 + - name: Free disk space + if: runner.os == 'Linux' + run: | + df -h + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache + sudo apt-get clean + sudo apt-get autoremove -y + docker system prune -af || true + df -h - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -38,6 +50,27 @@ jobs: mkdir -p $HOME/.serena/language_servers/static mkdir -p $HOME/.cache/go-build mkdir -p $HOME/go/bin + - name: Install uv + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + - name: Cache uv virtualenv + id: cache-uv + uses: actions/cache@v3 + with: + path: .venv + key: uv-venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} + - name: Create virtual environment + shell: bash + run: | + if [ ! -d ".venv" ]; then + uv venv + fi + - name: Install dependencies + shell: bash + run: uv pip install -e ".[dev]" + - name: Check formatting + shell: bash + run: uv run poe lint # Add Go bin directory to PATH for this workflow # GITHUB_PATH is a special file that GitHub Actions uses to modify PATH # Writing to this file adds the directory to the PATH for subsequent steps @@ -151,6 +184,47 @@ jobs: shell: bash run: | Rscript -e "install.packages('languageserver', repos='https://cloud.r-project.org')" + - name: Set up Julia + uses: julia-actions/setup-julia@v2 + with: + version: '1.10' + - name: Install Julia LanguageServer + shell: bash + run: julia -e 'using Pkg; Pkg.add("LanguageServer")' + - name: Setup Haskell toolchain + if: runner.os != 'Windows' + uses: haskell/ghcup-setup@v1 + with: + ghc: '9.12.2' + cabal: '3.10.3.0' + hls: '2.11.0.0' + - name: Verify Haskell tools + if: runner.os != 'Windows' + run: | + echo "Verifying installed Haskell tools:" + which ghc && ghc --version + which cabal && cabal --version + # HLS verification - non-blocking in case of version incompatibility + if command -v haskell-language-server-wrapper &>/dev/null; then + echo "Found haskell-language-server-wrapper" + haskell-language-server-wrapper --version || echo "WARNING: HLS wrapper found but version check failed" + elif command -v haskell-language-server &>/dev/null; then + echo "Found haskell-language-server" + haskell-language-server --version || echo "WARNING: HLS found but version check failed" + else + echo "WARNING: HLS not found (may be incompatible with GHC 9.12.2)" + echo "This is not a critical error - tests will use HLS if available at runtime" + fi + shell: bash + - name: Pre-build Haskell test project for HLS + if: runner.os != 'Windows' + run: | + cd test/resources/repos/haskell/test_repo + cabal update + cabal build --only-dependencies + cabal build + echo "Haskell test project built successfully" + shell: bash - name: Install Zig uses: goto-bus-stop/setup-zig@v2 with: @@ -293,15 +367,6 @@ jobs: mv regal.exe "$HOME/bin/" echo "$HOME/bin" >> $GITHUB_PATH fi - - name: Install uv - shell: bash - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - name: Cache uv virtualenv - id: cache-uv - uses: actions/cache@v3 - with: - path: .venv - key: uv-venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} - name: Cache language servers id: cache-language-servers uses: actions/cache@v3 @@ -310,18 +375,14 @@ jobs: key: language-servers-${{ runner.os }}-v1 restore-keys: | language-servers-${{ runner.os }}- - - name: Create virtual environment - shell: bash + - name: Report free disk space + if: runner.os == 'Linux' run: | - if [ ! -d ".venv" ]; then - uv venv - fi - - name: Install dependencies - shell: bash - run: uv pip install -e ".[dev]" - - name: Check formatting - shell: bash - run: uv run poe lint + echo "Free disk space before tests:" + df -h - name: Test with pytest shell: bash run: uv run poe test + - name: Type-checking with mypy + shell: bash + run: uv run poe type-check diff --git a/.gitignore b/.gitignore index 196476e6..78b8833a 100644 --- a/.gitignore +++ b/.gitignore @@ -200,11 +200,7 @@ pylint.html /*.yml !*.template.yml /agent-ui - -# dynamic LS installations -/src/multilspy/language_servers/*/static -/src/solidlsp/language_servers/*/static -/src/solidlsp/language_servers/static +/test/**/.serena # clojure-lsp temporary files .calva/ @@ -212,12 +208,11 @@ pylint.html .cpcache/ .lsp/ -# Cache is in .multilspy -**/.multilspy/** - +# temporary and backup files +*.bak +*.tmp tmp/ -.serena/ .vscode/ # Claude settings @@ -231,6 +226,9 @@ tmp/ # Exception: Don't ignore Nix test repository lib directory (contains source code) !/test/resources/repos/nix/test_repo/lib +# Exception: Don't ignore Julia test repository lib directory (contains source code) +!/test/resources/repos/julia/test_repo/lib + # Swift /test/resources/repos/swift/test_repo/.build /test/resources/repos/swift/test_repo/.swiftpm @@ -238,3 +236,19 @@ tmp/ # Elm /test/resources/repos/elm/test_repo/.elm/ /test/resources/repos/elm/test_repo/elm-stuff/ + +# Scala +.metals/ +.bsp/ +.scala-build/ +.bloop/ +bootstrap +test/resources/repos/scala/.bloop/ + +# Haskell +.stack-work/ +*.cabal +stack.yaml.lock +dist-newstyle/ +cabal.project.local* +.ghc.environment.* \ No newline at end of file diff --git a/.serena/.gitignore b/.serena/.gitignore new file mode 100644 index 00000000..14d86ad6 --- /dev/null +++ b/.serena/.gitignore @@ -0,0 +1 @@ +/cache diff --git a/.serena/memories/adding_new_language_support_guide.md b/.serena/memories/adding_new_language_support_guide.md index 21980531..18d12436 100644 --- a/.serena/memories/adding_new_language_support_guide.md +++ b/.serena/memories/adding_new_language_support_guide.md @@ -21,39 +21,35 @@ Have a look at `intelephense.py` for a reference implementation of a language se dependencies, and on `pyright_server.py` that does not need any additional dependencies because the language server can be installed directly as python package. - ```python from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig -from solidlsp.ls_logger import LanguageServerLogger from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo + class NewLanguageServer(SolidLanguageServer): """ Language server implementation for NewLanguage. """ - - def __init__(self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str): + + def __init__(self, config: LanguageServerConfig, repository_root_path: str): # Determine language server command cmd = self._get_language_server_command() - - super().__init__( - config, - logger, - repository_root_path, + + super().__init__(config, ProcessLaunchInfo(cmd=cmd, cwd=repository_root_path), - "new_language", # Language ID for LSP - ) - + "new_language",, + def _get_language_server_command(self) -> list[str]: """Get the command to start the language server.""" # Example: return ["new-language-server", "--stdio"] pass - + @override def is_ignored_dirname(self, dirname: str) -> bool: """Define language-specific directories to ignore.""" - return super().is_ignored_dirname(dirname) or dirname in ["build", "dist", "target"] + return super().is_ignored_dirname(dirname) or dirname in ["build", + "dist", "target"] ``` ### 1.2 Language Server Discovery and Installation @@ -62,7 +58,7 @@ For languages requiring automatic installation, implement download logic similar ```python @classmethod -def _ensure_server_installed(cls, logger: LanguageServerLogger) -> str: +def _ensure_server_installed(cls) -> str: """Ensure language server is installed and return path.""" # Check system installation first system_server = shutil.which("new-language-server") @@ -70,10 +66,10 @@ def _ensure_server_installed(cls, logger: LanguageServerLogger) -> str: return system_server # Download and install if needed - server_path = cls._download_and_install_server(logger) + server_path = cls._download_and_install_server() return server_path -def _download_and_install_server(cls, logger: LanguageServerLogger) -> str: +def _download_and_install_server(cls) -> str: """Download and install the language server.""" # Implementation specific to your language server pass @@ -129,12 +125,12 @@ In `src/solidlsp/ls.py`, add your language to the `create` method: ```python @classmethod -def create(cls, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str) -> "SolidLanguageServer": +def create(cls, config: LanguageServerConfig, repository_root_path: str) -> "SolidLanguageServer": match config.code_language: # Existing cases... case Language.NEW_LANGUAGE: from solidlsp.language_servers.new_language_server import NewLanguageServer - return NewLanguageServer(config, logger, repository_root_path) + return NewLanguageServer(config, repository_root_path) ``` ## Step 3: Test Repository diff --git a/.serena/project.yml b/.serena/project.yml index a265810e..edba1eeb 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -1,11 +1,5 @@ -# language of the project (csharp, python, rust, java, typescript, javascript, go, cpp, or ruby) -# Special requirements: -# * csharp: Requires the presence of a .sln file in the project folder. -language: python - -# whether to use the project's gitignore file to ignore files -# Added on 2025-04-07 ignore_all_files_in_gitignore: true + # list of additional paths to ignore # same syntax as gitignore, so you can use * and ** # Was previously called `ignored_dirs`, please update your config if you are using that. @@ -64,3 +58,8 @@ excluded_tools: [] initial_prompt: "" project_name: "serena" +languages: +- python +- typescript +included_optional_tools: [] +encoding: utf-8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 912f4b99..d8327d8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,43 @@ # latest Status of the `main` branch. Changes prior to the next official version change will appear here. + +* General: + * Add monorepo/multi-language support + * Project configuration files (`project.yml`) can now define multiple languages. + Auto-detection adds only the most prominent language by default. + * Additional languages can be conveniently added via the Dashboard while a project is already activated. + * Support overloaded symbols in `FindSymbolTool` and related tools + * Name paths of overloaded symbols now include an index (e.g., `myOverloadedFunction[2]`) + * Responses of the Java language server, which handled this in its own way, are now adapted accordingly, + solving several issues related to retrieval problems in Java projects + * Major extensions to the dashboard, which now serves as a central web interface for Serena + * View current configuration + * View the executions, with the possibility to cancel running/scheduled executions + * View tool usage statistics + * View and create memories and edit the serena configuration file + * New two-tier caching of language server document symbols and considerable performance improvements surrounding symbol retrieval/indexing + * Various fixes related to indexing, special paths and determination of ignored paths + * Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations + * Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`) + * Add notion of a "single-project context" (flag `single_project`), allowing user-defined contexts to behave + like the built-in `ide-assistant` context (where the available tools are restricted to ones required by the active + project and project changes are disabled) + +* Client support: + * New mode `oaicompat-agent` and extensions enhancing OpenAI tool compatibility, permitting Serena to work with llama.cpp + +* Tools: + * Added `RenameSymbolTool` for renaming symbols across the codebase (if LS supports this operation). + * Replaced `ReplaceRegexTool` with `ReplaceContentTool`, which supports both plain text and regex-based replacements + (and which requires no escaping in the replacement text, making it more robust) + * Language support: + + * **Add support for Fortran** via fortls language server (requires `pip install fortls`) + * **Add support for Julia** via LanguageServer.jl + * **Add support for Haskell** via Haskell Language Server (HLS) with automatic discovery via ghcup, stack, or system PATH; supports both Stack and Cabal projects + * **Add support for Scala** via Metals language server (requires some [manual setup](docs/03-special-guides/scala_setup_guide_for_serena.md)) * **Add support for F#** via FsAutoComplete/Ionide LSP server. * **Add support for Elm** via @elm-tooling/elm-language-server (automatically downloads if not installed; requires Elm compiler) * **Add support for Perl** via Perl::LanguageServer with LSP integration for .pl, .pm, and .t files @@ -10,19 +46,13 @@ Status of the `main` branch. Changes prior to the next official version change w * **Add support for Zig** via ZLS (cross-file references may not fully work on Windows) * **Add support for Lua** via lua-language-server * **Add support for Nix** requires nixd installation (Windows not supported) + * **Add experimental support for YAML** via yaml-language-server with LSP integration for .yaml and .yml files * **Dart now officially supported**: Dart was always working, but now tests were added, and it is promoted to "officially supported" * **Rust now uses already installed rustup**: The rust-analyzer is no longer bundled with Serena. Instead, it uses the rust-analyzer from your Rust toolchain managed by rustup. This ensures compatibility with your Rust version and eliminates outdated bundled binaries. - * **Kotlin now officially supported**: We now use the official Kotlin LS, tests run through and performance is good, even though the LS is in an early development stage. + * **Kotlin now officially supported**: We now use the official Kotlin LS, tests run through and performance is good, even though the LS is in an early development stage. * **Add support for Erlang** experimental, may hang or be slow, uses the recently archived [erlang_ls](https://github.com/erlang-ls/erlang_ls) * **Ruby dual language server support**: Added ruby-lsp as the modern primary Ruby language server. Solargraph remains available as an experimental legacy option. ruby-lsp supports both .rb and .erb files, while Solargraph supports .rb files only. -* Client support: - * New mode `oaicompat-agent` and extensions in the openai tool compatibility, **permitting Serena to work with llama.cpp** - -* General: - * Various fixes related to indexing, special paths and determation of ignored paths - * Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations - * Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`) # 0.1.4 diff --git a/DOCKER.md b/DOCKER.md index 84f8f746..03432451 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -98,7 +98,7 @@ services: - /path/to/another/project:/workspace/another-project # Add the context for the IDE assistant option: command: - - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant" + - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context claude-code" ``` See the [Docker Merge Compose files documentation](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/) for more details on using merge files. diff --git a/README.md b/README.md index fe217a9f..92970158 100644 --- a/README.md +++ b/README.md @@ -8,46 +8,39 @@ * :wrench: Serena provides essential **semantic code retrieval and editing tools** that are akin to an IDE's capabilities, extracting code entities at the symbol level and exploiting relational structure. When combined with an existing coding agent, these tools greatly enhance (token) efficiency. * :free: Serena is **free & open-source**, enhancing the capabilities of LLMs you already have access to free of charge. -You can think of Serena as providing IDE-like tools to your LLM/coding agent. With it, the agent no longer needs to read entire -files, perform grep-like searches or string replacements to find and edit the right code. Instead, it can use code centered tools like `find_symbol`, `find_referencing_symbols` and `insert_after_symbol`. +You can think of Serena as providing IDE-like tools to your LLM/coding agent. +With it, the agent no longer needs to read entire files, perform grep-like searches or basic string replacements to find the right parts of the code and to edit code. +Instead, it can use code-centric tools like `find_symbol`, `find_referencing_symbols` and `insert_after_symbol`.

Serena is under active development! See the latest updates, upcoming features, and lessons learned to stay up to date.

- - Changelog - - - Roadmap - - - Lessons Learned - + Changelog + Roadmap + Lessons Learned

-### LLM Integration +## LLM Integration -Serena provides the necessary [tools](#list-of-tools) for coding workflows, but an LLM is required to do the actual work, +Serena provides the necessary [tools](https://oraios.github.io/serena/01-about/035_tools.html) for coding workflows, but an LLM is required to do the actual work, orchestrating tool use. -For example, **supercharge the performance of Claude Code** with a [one-line shell command](#claude-code). - In general, Serena can be integrated with an LLM in several ways: * by using the **model context protocol (MCP)**. Serena provides an MCP server which integrates with * Claude Code and Claude Desktop, - * Terminal-based clients like Codex, Gemini-CLI, Qwen3-Coder, rovodev, OpenHands CLI and others, + * terminal-based clients like Codex, Gemini-CLI, Qwen3-Coder, rovodev, OpenHands CLI and others, * IDEs like VSCode, Cursor or IntelliJ, * Extensions like Cline or Roo Code * Local clients like [OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp), [Jan](https://jan.ai/docs/mcp-examples/browser/browserbase#enable-mcp), [Agno](https://docs.agno.com/introduction/playground) and others -* by using [mcpo to connect it to ChatGPT](docs/serena_on_chatgpt.md) or other clients that don't support MCP but do support tool calling via OpenAPI. -* by incorporating Serena's tools into an agent framework of your choice, as illustrated [here](docs/custom_agent.md). +* by using [mcpo to connect it to ChatGPT](docs/03-special-guides/serena_on_chatgpt.md) or other clients that don't support MCP but do support tool calling via OpenAPI. +* by incorporating Serena's tools into an agent framework of your choice, as illustrated [here](docs/03-special-guides/custom_agent.md). Serena's tool implementation is decoupled from the framework-specific code and can thus easily be adapted to any agent framework. -### Serena in Action +## Serena in Action #### Demonstration 1: Efficient Operation in Claude Code @@ -74,44 +67,45 @@ complex projects! So not only is it free and open-source, it frequently achieves than existing solutions that charge a premium. Language servers provide support for a wide range of programming languages. -With Serena, we provide direct, out-of-the-box support for: +With Serena's LSP library, we provide **support for over 30 programming languages**, including +AL, Bash, C#, C/C++, Clojure, Dart, Elixir, Elm, Erlang, Fortran, Go, Haskell, Java, Javascript, Julia, Kotlin, Lua, Markdown, Nix, Perl, PHP, Python, R, Ruby, Rust, Scala, Swift, TypeScript, YAML and Zig. - * Python - * TypeScript/Javascript - * PHP (uses Intelephense LSP; set `INTELEPHENSE_LICENSE_KEY` environment variable for premium features) - * Go (requires installation of gopls) - * R (requires installation of the `languageserver` R package) - * Rust (requires [rustup](https://rustup.rs/) - uses rust-analyzer from your toolchain) - * C/C++ (you may experience issues with finding references, we are working on it) - * Zig (requires installation of ZLS - Zig Language Server) - * C# - * F# (requires .NET SDK 8.0+ and uses FsAutoComplete/Ionide LSP server) - * Ruby (by default, uses [ruby-lsp](https://github.com/Shopify/ruby-lsp), specify ruby_solargraph as your language to use the previous solargraph based implementation) - * Swift - * Kotlin (uses the pre-alpha [official kotlin LS](https://github.com/Kotlin/kotlin-lsp), some issues may appear) - * Java (_Note_: startup is slow, initial startup especially so. There may be issues with java on macos and linux, we are working on it.) - * Clojure - * Dart - * Bash - * Lua (automatically downloads lua-language-server if not installed) - * Nix (requires nixd installation) - * Elixir (requires installation of NextLS and Elixir; **Windows not supported**) - * Elm (automatically downloads elm-language-server if not installed; requires Elm compiler) - * Erlang (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls), experimental, might be slow or hang) - * Perl (requires installation of Perl::LanguageServer) - * AL - * Markdown (automatically downloads marksman if not installed, experimental, must be explicitly specified via `--language markdown` when generating project config - primarily useful for documentation-heavy projects) +> [!IMPORTANT] +> Some languages require additional dependencies to be installed; see the [Language Support](https://oraios.github.io/serena/01-about/020_programming-languages.html) page for details. -Support for further languages can easily be added by providing a shallow adapter for a new language server implementation, -see Serena's [memory on that](.serena/memories/adding_new_language_support_guide.md). +## Quick Start -### Community Feedback +**Prerequisites**. Serena is managed by *uv*. If you don’t already have it, you need to [install uv](https://docs.astral.sh/uv/getting-started/installation/) before proceeding. + +**Starting the MCP Server**. The easiest way to start the Serena MCP server is by running the latest version from GitHub using uvx. +Issue this command to see available options: + +```bash +uvx --from git+https://github.com/oraios/serena serena start-mcp-server --help +``` + +**Configuring Your Client**. To connect Serena to your preferred MCP client, you typically need to [configure a launch command in your client](https://oraios.github.io/serena/02-usage/030_clients.html). +Follow the link for specific instructions on how to set up Serena for Claude Code, Codex, Claude Desktop, MCP-enabled IDEs and other clients (such as local and web-based GUIs). + +> [!TIP] +> While getting started quickly is easy, Serena is a powerful toolkit with many configuration options. +> We highly recommend reading through the [user guide](https://oraios.github.io/serena/02-usage/000_intro.html) to get the most out of Serena. +> +> Specifically, we recommend to read about ... +> * [Serena's project-based workflow](https://oraios.github.io/serena/02-usage/040_workflow.html) and +> * [configuring Serena](https://oraios.github.io/serena/02-usage/050_configuration.html). + +## User Guide + +Please refer to the [user guide](https://oraios.github.io/serena/02-usage/000_intro.html) for detailed instructions on how to use Serena effectively. + +## Community Feedback Most users report that Serena has strong positive effects on the results of their coding agents, even when used within very capable agents like Claude Code. Serena is often described to be a [game changer](https://www.reddit.com/r/ClaudeAI/comments/1lfsdll/try_out_serena_mcp_thank_me_later/), providing an enormous [productivity boost](https://www.reddit.com/r/ClaudeCode/comments/1mguoia/absolutely_insane_improvement_of_claude_code). Serena excels at navigating and manipulating complex codebases, providing tools that support precise code retrieval and editing in the presence of large, strongly structured codebases. -However, when dealing with tasks that involve only very few/small files, you may not benefit from including Serena on top of your existing coding agent. +However, when dealing with tasks that involve only very few/small files, you may not benefit from including Serena on top of your existing coding agent. In particular, when writing code from scratch, Serena will not provide much value initially, as the more complex structures that Serena handles more gracefully than simplistic, file-based approaches are yet to be created. Several videos and blog posts have talked about Serena: @@ -126,666 +120,6 @@ Several videos and blog posts have talked about Serena: * [Serena with Claude Code (in Japanese)](https://blog.lai.so/serena/) * [Turning Claude Code into a Development Powerhouse](https://robertmarshall.dev/blog/turning-claude-code-into-a-development-powerhouse/) -## Table of Contents - - - - - - -- [Quick Start](#quick-start) - * [Running the Serena MCP Server](#running-the-serena-mcp-server) - + [Usage](#usage) - + [Using uvx](#using-uvx) - + [Local Installation](#local-installation) - + [Using Docker (Experimental)](#using-docker-experimental) - + [Using Nix](#using-nix) - + [Streamable HTTP Mode](#streamable-http-mode) - + [Command-Line Arguments](#command-line-arguments) - * [Configuration](#configuration) - * [Project Activation & Indexing](#project-activation--indexing) - * [Claude Code](#claude-code) - * [Codex](#codex) - * [Other Terminal-Based Clients](#other-terminal-based-clients) - * [Claude Desktop](#claude-desktop) - * [MCP Coding Clients (Cline, Roo-Code, Cursor, Windsurf, etc.)](#mcp-coding-clients-cline-roo-code-cursor-windsurf-etc) - * [Local GUIs and Frameworks](#local-guis-and-frameworks) -- [Detailed Usage and Recommendations](#detailed-usage-and-recommendations) - * [Tool Execution](#tool-execution) - + [Shell Execution and Editing Tools](#shell-execution-and-editing-tools) - * [Modes and Contexts](#modes-and-contexts) - + [Contexts](#contexts) - + [Modes](#modes) - + [Customization](#customization) - * [Onboarding and Memories](#onboarding-and-memories) - * [Prepare Your Project](#prepare-your-project) - + [Structure Your Codebase](#structure-your-codebase) - + [Start from a Clean State](#start-from-a-clean-state) - + [Logging, Linting, and Automated Tests](#logging-linting-and-automated-tests) - * [Prompting Strategies](#prompting-strategies) - * [Running Out of Context](#running-out-of-context) - * [Serena's Logs: The Dashboard and GUI Tool](#serenas-logs-the-dashboard-and-gui-tool) -- [Comparison with Other Coding Agents](#comparison-with-other-coding-agents) - * [Subscription-Based Coding Agents](#subscription-based-coding-agents) - * [API-Based Coding Agents](#api-based-coding-agents) - * [Other MCP-Based Coding Agents](#other-mcp-based-coding-agents) -- [Acknowledgements](#acknowledgements) - * [Sponsors](#sponsors) - * [Community Contributions](#community-contributions) - * [Technologies](#technologies) -- [Customizing and Extending Serena](#customizing-and-extending-serena) -- [List of Tools](#list-of-tools) - - - -## Quick Start - -Serena can be used in various ways, below you will find instructions for selected integrations. - -* For coding with Claude, we recommend using Serena through [Claude Code](#claude-code) or [Claude Desktop](#claude-desktop). You can also use Serena in most other [terminal-based clients](#other-terminal-based-clients). -* If you want a GUI experience outside an IDE, you can use one of the many [local GUIs](#local-guis-and-frameworks) that support MCP servers. - You can also connect Serena to many web clients (including ChatGPT) using [mcpo](docs/serena_on_chatgpt.md). -* If you want to use Serena integrated in your IDE, see the section on [other MCP clients](#other-mcp-clients---cline-roo-code-cursor-windsurf-etc). -* You can use Serena as a library for building your own applications. We try to keep the public API stable, but you should still - expect breaking changes and pin Serena to a fixed version if you use it as a dependency. - -Serena is managed by `uv`, so you will need to [install it](https://docs.astral.sh/uv/getting-started/installation/). - -### Running the Serena MCP Server - -You have several options for running the MCP server, which are explained in the subsections below. - -#### Usage - -The typical usage involves the client (Claude Code, Claude Desktop, etc.) running -the MCP server as a subprocess (using stdio communication), -so the client needs to be provided with the command to run the MCP server. -(Alternatively, you can run the MCP server in Streamable HTTP or SSE mode and tell your client -how to connect to it.) - -Note that no matter how you run the MCP server, Serena will, by default, start a small web-based dashboard on localhost that will display logs and allow shutting down the -MCP server (since many clients fail to clean up processes correctly). -This and other settings can be adjusted in the [configuration](#configuration) and/or by providing [command-line arguments](#command-line-arguments). - -#### Using uvx - -`uvx` can be used to run the latest version of Serena directly from the repository, without an explicit local installation. - -```shell -uvx --from git+https://github.com/oraios/serena serena start-mcp-server -``` - -Explore the CLI to see some of the customization options that serena provides (more info on them below). - -#### Local Installation - -1. Clone the repository and change into it. - - ```shell - git clone https://github.com/oraios/serena - cd serena - ``` - -2. Optionally edit the configuration file in your home directory with - - ```shell - uv run serena config edit - ``` - - If you just want the default config, you can skip this part, and a config file will be created when you first run Serena. -3. Run the server with `uv`: - - ```shell - uv run serena start-mcp-server - ``` - - When running from outside the serena installation directory, be sure to pass it, i.e., use - - ```shell - uv run --directory /abs/path/to/serena serena start-mcp-server - ``` - -#### Using Docker (Experimental) - -⚠️ Docker support is currently experimental with several limitations. Please read the [Docker documentation](DOCKER.md) for important caveats before using it. - -You can run the Serena MCP server directly via docker as follows, -assuming that the projects you want to work on are all located in `/path/to/your/projects`: - -```shell -docker run --rm -i --network host -v /path/to/your/projects:/workspaces/projects ghcr.io/oraios/serena:latest serena start-mcp-server --transport stdio -``` - -Replace `/path/to/your/projects` with the absolute path to your projects directory. The Docker approach provides: - -* Better security isolation for shell command execution -* No need to install language servers and dependencies locally -* Consistent environment across different systems - -Alternatively, use docker compose with the `compose.yml` file provided in the repository. - -See the [Docker documentation](DOCKER.md) for detailed setup instructions, configuration options, and known limitations. - -#### Using Nix - -If you are using Nix and [have enabled the `nix-command` and `flakes` features](https://nixos.wiki/wiki/flakes), you can run Serena using the following command: - -```bash -nix run github:oraios/serena -- start-mcp-server --transport stdio -``` - -You can also install Serena by referencing this repo (`github:oraios/serena`) and using it in your Nix flake. The package is exported as `serena`. - -#### Streamable HTTP Mode - -ℹ️ Note that MCP servers which use stdio as a protocol are somewhat unusual as far as client/server architectures go, as the server -necessarily has to be started by the client in order for communication to take place via the server's standard input/output stream. -In other words, you do not need to start the server yourself. The client application (e.g. Claude Desktop) takes care of this and -therefore needs to be configured with a launch command. - -When using instead the *Streamable HTTP* mode, you control the server lifecycle yourself, -i.e. you start the server and provide the client with the URL to connect to it. - -Simply provide `start-mcp-server` with the `--transport streamable-http` option and optionally provide the port. -For example, to run the Serena MCP server in Streamable HTTP mode on port 9121 using a local installation, -you would run this command from the Serena directory, - -```shell -uv run serena start-mcp-server --transport streamable-http --port 9121 -``` - -and then configure your client to connect to `http://localhost:9121/mcp`. - -ℹ️ Note that SSE transport is supported as well, but its use is discouraged. -Use Streamable HTTP instead. - -#### Command-Line Arguments - -The Serena MCP server supports a wide range of additional command-line options, including the option to run in Streamable HTTP or SSE mode -and to adapt Serena to various [contexts and modes of operation](#modes-and-contexts). - -Run with parameter `--help` to get a list of available options. - -### Configuration - -Serena is very flexible in terms of configuration. While for most users, the default configurations will work, -you can fully adjust it to your needs by editing a few yaml files. You can disable tools, change Serena's instructions -(what we denote as the `system_prompt`), adjust the output of tools that just provide a prompt, and even adjust tool descriptions. - -Serena is configured in four places: - -1. The `serena_config.yml` for general settings that apply to all clients and projects. - It is located in your user directory under `.serena/serena_config.yml`. - If you do not explicitly create the file, it will be auto-generated when you first run Serena. - You can edit it directly or use - - ```shell - uvx --from git+https://github.com/oraios/serena serena config edit - ``` - - (or use the `--directory` command version). -2. In the arguments passed to the `start-mcp-server` in your client's config (see below), - which will apply to all sessions started by the respective client. In particular, the [context](#contexts) parameter - should be set appropriately for Serena to be best adjusted to existing tools and capabilities of your client. - See for a detailed explanation. You can override all entries from the `serena_config.yml` through command line arguments. -3. In the `.serena/project.yml` file within your project. This will hold project-level configuration that is used whenever - that project is activated. This file will be autogenerated when you first use Serena on that project, but you can also - generate it explicitly with - - ```shell - uvx --from git+https://github.com/oraios/serena serena project generate-yml - ``` - - (or use the `--directory` command version). -4. Through the context and modes. Explore the [modes and contexts](#modes-and-contexts) section for more details. - -After the initial setup, continue with one of the sections below, depending on how you -want to use Serena. - -### Project Activation & Indexing - -If you are mostly working with the same project, you can configure to always activate it at startup -by passing `--project ` to the `start-mcp-server` command in your client's MCP config. -This is especially useful for clients which configure MCP servers on a per-project basis, like Claude Code. - -Otherwise, the recommended way is to just ask the LLM to activate a project by providing it an absolute path to, or, -in case the project was activated in the past, by its name. The default project name is the directory name. - -* "Activate the project /path/to/my_project" -* "Activate the project my_project" - -All projects that have been activated will be automatically added to your `serena_config.yml`, and for each -project, the file `.serena/project.yml` will be generated. You can adjust the latter, e.g., by changing the name -(which you refer to during the activation) or other options. Make sure to not have two different projects with the -same name. - -ℹ️ For larger projects, we recommend that you index your project to accelerate Serena's tools; otherwise the first -tool application may be very slow. -To do so, run this from the project directory (or pass the path to the project as an argument): - -```shell -uvx --from git+https://github.com/oraios/serena serena project index -``` - -(or use the `--directory` command version). - -### Claude Code - -Serena is a great way to make Claude Code both cheaper and more powerful! - -From your project directory, add serena with a command like this, - -```shell -claude mcp add serena -- --context ide-assistant --project $(pwd) -``` - -where `` is your way of [running the Serena MCP server](#running-the-serena-mcp-server). -For example, when using `uvx`, you would run - -```shell -claude mcp add serena -- uvx --from git+https://github.com/oraios/serena serena start-mcp-server --context ide-assistant --project $(pwd) -``` - -ℹ️ Serena comes with an instruction text, and Claude needs to read it to properly use Serena's tools. - As of version `v1.0.52`, claude code reads the instructions of the MCP server, so this **is handled automatically**. - If you are using an older version, or if Claude fails to read the instructions, you can ask it explicitly - to "read Serena's initial instructions" or run `/mcp__serena__initial_instructions` to load the instruction text. - If you want to make use of that, you will have to enable the corresponding tool explicitly by adding `initial_instructions` to the `included_optional_tools` - in your config. - Note that you may have to make Claude read the instructions when you start a new conversation and after any compacting operation to ensure Claude remains properly configured to use Serena's tools. - -### Codex - -Serena works with OpenAI's Codex CLI out of the box, but you have to use the `codex` context for it to work properly. (The technical reason is that Codex doesn't fully support the MCP specifications, so some massaging of tools is required.). - -Unlike Claude Code, in Codex you add an MCP server globally and not per project. Add the following to -`~/.codex/config.toml` (create the file if it does not exist): - -```toml -[mcp_servers.serena] -command = "uvx" -args = ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server", "--context", "codex"] -``` - -After codex has started, you need to activate the project, which you can do by saying: - -"Activate the current dir as project using serena" - -> If you don't activate the project, you will not be able to use Serena's tools! - -That's it! Have a look at `~/.codex/log/codex-tui.log` to see if any errors occurred. - -The Serena dashboard will run if you have not disabled it in the configuration, but due to Codex's sandboxing the webbrowser -may not open automatically. You can open it manually by going to `http://localhost:24282/dashboard/index.html` (or a higher port, if -that was already taken). - -> Codex will often show the tools as `failed` even though they are successfully executed. This is not a problem, seems to be a bug in Codex. Despite the error message, everything works as expected. - -### Other Terminal-Based Clients - -There are many terminal-based coding assistants that support MCP servers, such as [Codex](https://github.com/openai/codex?tab=readme-ov-file#model-context-protocol-mcp), -[Gemini-CLI](https://github.com/google-gemini/gemini-cli), [Qwen3-Coder](https://github.com/QwenLM/Qwen3-Coder), -[rovodev](https://community.atlassian.com/forums/Rovo-for-Software-Teams-Beta/Introducing-Rovo-Dev-CLI-AI-Powered-Development-in-your-terminal/ba-p/3043623), -the [OpenHands CLI](https://docs.all-hands.dev/usage/how-to/cli-mode) and [opencode](https://github.com/sst/opencode). - -They generally benefit from the symbolic tools provided by Serena. You might want to customize some aspects of Serena -by writing your own context, modes or prompts to adjust it to your workflow, to other MCP servers you are using, and to -the client's internal capabilities. - -### Claude Desktop - -For [Claude Desktop](https://claude.ai/download) (available for Windows and macOS), go to File / Settings / Developer / MCP Servers / Edit Config, -which will let you open the JSON file `claude_desktop_config.json`. -Add the `serena` MCP server configuration, using a [run command](#running-the-serena-mcp-server) depending on your setup. - -* local installation: - - ```json - { - "mcpServers": { - "serena": { - "command": "/abs/path/to/uv", - "args": ["run", "--directory", "/abs/path/to/serena", "serena", "start-mcp-server"] - } - } - } - ``` - -* uvx: - - ```json - { - "mcpServers": { - "serena": { - "command": "/abs/path/to/uvx", - "args": ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server"] - } - } - } - ``` - -* docker: - - ```json - { - "mcpServers": { - "serena": { - "command": "docker", - "args": ["run", "--rm", "-i", "--network", "host", "-v", "/path/to/your/projects:/workspaces/projects", "ghcr.io/oraios/serena:latest", "serena", "start-mcp-server", "--transport", "stdio"] - } - } - } - ``` - -If you are using paths containing backslashes for paths on Windows -(note that you can also just use forward slashes), be sure to escape them correctly (`\\`). - -That's it! Save the config and then restart Claude Desktop. You are ready for activating your first project. - -ℹ️ You can further customize the run command using additional arguments (see [above](#command-line-arguments)). - -Note: on Windows and macOS there are official Claude Desktop applications by Anthropic, for Linux there is an [open-source -community version](https://github.com/aaddrick/claude-desktop-debian). - -⚠️ Be sure to fully quit the Claude Desktop application, as closing Claude will just minimize it to the system tray – at least on Windows. - -⚠️ Some clients may leave behind zombie processes. You will have to find and terminate them manually then. - With Serena, you can activate the [dashboard](#serenas-logs-the-dashboard-and-gui-tool) to prevent unnoted processes and also use the dashboard - for shutting down Serena. - -After restarting, you should see Serena's tools in your chat interface (notice the small hammer icon). - -For more information on MCP servers with Claude Desktop, see [the official quick start guide](https://modelcontextprotocol.io/quickstart/user). - -### MCP Coding Clients (Cline, Roo-Code, Cursor, Windsurf, etc.) - -Being an MCP Server, Serena can be included in any MCP Client. The same configuration as above, -perhaps with small client-specific modifications, should work. Most of the popular -existing coding assistants (IDE extensions or VSCode-like IDEs) support connections -to MCP Servers. It is **recommended to use the `ide-assistant` context** for these integrations by adding `"--context", "ide-assistant"` to the `args` in your MCP client's configuration. Including Serena generally boosts their performance -by providing them tools for symbolic operations. - -In this case, the billing for the usage continues to be controlled by the client of your choice -(unlike with the Claude Desktop client). But you may still want to use Serena through such an approach, -e.g., for one of the following reasons: - -1. You are already using a coding assistant (say Cline or Cursor) and just want to make it more powerful. -2. You are on Linux and don't want to use the [community-created Claude Desktop](https://github.com/aaddrick/claude-desktop-debian). -3. You want tighter integration of Serena into your IDE and don't mind paying for that. - -### Local GUIs and Frameworks - -Over the last months, several technologies have emerged that allow you to run a powerful local GUI -and connect it to an MCP server. They will work with Serena out of the box. -Some of the leading open source GUI technologies offering this are -[Jan](https://jan.ai/docs/mcp), [OpenHands](https://github.com/All-Hands-AI/OpenHands/), -[OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp) and [Agno](https://docs.agno.com/introduction/playground). -They allow combining Serena with almost any LLM (including locally running ones) and offer various other integrations. - -## Detailed Usage and Recommendations - -### Tool Execution - -Serena combines tools for semantic code retrieval with editing capabilities and shell execution. -Serena's behavior can be further customized through [Modes and Contexts](#modes-and-contexts). -Find the complete list of tools [below](#full-list-of-tools). - -The use of all tools is generally recommended, as this allows Serena to provide the most value: -Only by executing shell commands (in particular, tests) can Serena identify and correct mistakes -autonomously. - -#### Shell Execution and Editing Tools - -Many clients have their own shell execution tool, and by default Serena's shell tool will be disabled in them -(e.g., when using the `ide-assistant` or `codex` context). However, when using Serena through something like -Claude Desktop or ChatGPT, it is recommended to enable Serena's `execute_shell_command` tool to allow -agentic behavior. - -It should be noted that the `execute_shell_command` tool allows for arbitrary code execution. -When using Serena as an MCP Server, clients will typically ask the user for permission -before executing a tool, so as long as the user inspects execution parameters beforehand, -this should not be a problem. -However, if you have concerns, you can choose to disable certain commands in your project's configuration file. -If you only want to use Serena purely for analyzing code and suggesting implementations -without modifying the codebase, you can enable read-only mode by setting `read_only: true` in your project configuration file. -This will automatically disable all editing tools and prevent any modifications to your codebase while still -allowing all analysis and exploration capabilities. - -In general, be sure to back up your work and use a version control system in order to avoid -losing any work. - -### Modes and Contexts - -Serena's behavior and toolset can be adjusted using contexts and modes. -These allow for a high degree of customization to best suit your workflow and the environment Serena is operating in. - -#### Contexts - -A context defines the general environment in which Serena is operating. -It influences the initial system prompt and the set of available tools. -A context is set at startup when launching Serena (e.g., via CLI options for an MCP server or in the agent script) and cannot be changed during an active session. - -Serena comes with pre-defined contexts: - -* `desktop-app`: Tailored for use with desktop applications like Claude Desktop. This is the default. -* `agent`: Designed for scenarios where Serena acts as a more autonomous agent, for example, when used with Agno. -* `ide-assistant`: Optimized for integration into IDEs like VSCode, Cursor, or Cline, focusing on in-editor coding assistance. -Choose the context that best matches the type of integration you are using. - -When launching Serena, specify the context using `--context `. -Note that for cases where parameter lists are specified (e.g. Claude Desktop), you must add two parameters to the list. - -If you are using a local server (such as Llama.cpp) which requires you to use OpenAI-compatible tool descriptions, use context `oaicompat-agent` instead of `agent`. - -#### Modes - -Modes further refine Serena's behavior for specific types of tasks or interaction styles. Multiple modes can be active simultaneously, allowing you to combine their effects. Modes influence the system prompt and can also alter the set of available tools by excluding certain ones. - -Examples of built-in modes include: - -* `planning`: Focuses Serena on planning and analysis tasks. -* `editing`: Optimizes Serena for direct code modification tasks. -* `interactive`: Suitable for a conversational, back-and-forth interaction style. -* `one-shot`: Configures Serena for tasks that should be completed in a single response, often used with `planning` for generating reports or initial plans. -* `no-onboarding`: Skips the initial onboarding process if it's not needed for a particular session. -* `onboarding`: (Usually triggered automatically) Focuses on the project onboarding process. - -Modes can be set at startup (similar to contexts) but can also be _switched dynamically_ during a session. You can instruct the LLM to use the `switch_modes` tool to activate a different set of modes (e.g., "switch to planning and one-shot modes"). - -When launching Serena, specify modes using `--mode `; multiple modes can be specified, e.g. `--mode planning --mode no-onboarding`. - -:warning: **Mode Compatibility**: While you can combine modes, some may be semantically incompatible (e.g., `interactive` and `one-shot`). Serena currently does not prevent incompatible combinations; it is up to the user to choose sensible mode configurations. - -#### Customization - -You can create your own contexts and modes to precisely tailor Serena to your needs in two ways: - -* You can use Serena's CLI to manage modes and contexts. Check out - - ```shell - uvx --from git+https://github.com/oraios/serena serena mode --help - ``` - - and - - ```shell - uvx --from git+https://github.com/oraios/serena serena context --help - ``` - - _NOTE_: Custom contexts/modes are simply YAML files in `/.serena`, they are automatically registered and available for use by their name (filename without the `.yml` extension). If you don't want to use Serena's CLI, you can create and manage them in any way you see fit. -* **Using external YAML files**: When starting Serena, you can also provide an absolute path to a custom `.yml` file for a context or mode. - -This customization allows for deep integration and adaptation of Serena to specific project requirements or personal preferences. - -### Onboarding and Memories - -By default, Serena will perform an **onboarding process** when -it is started for the first time for a project. -The goal of the onboarding is for Serena to get familiar with the project -and to store memories, which it can then draw upon in future interactions. -If an LLM should fail to complete the onboarding and does not actually write the -respective memories to disk, you may need to ask it to do so explicitly. - -The onboarding will usually read a lot of content from the project, thus filling -up the context. It can therefore be advisable to switch to another conversation -once the onboarding is complete. -After the onboarding, we recommend that you have a quick look at the memories and, -if necessary, edit them or add additional ones. - -**Memories** are files stored in `.serena/memories/` in the project directory, -which the agent can choose to read in subsequent interactions. -Feel free to read and adjust them as needed; you can also add new ones manually. -Every file in the `.serena/memories/` directory is a memory file. -Whenever Serena starts working on a project, the list of memories is -provided, and the agent can decide to read them. -We found that memories can significantly improve the user experience with Serena. - -### Prepare Your Project - -#### Structure Your Codebase - -Serena uses the code structure for finding, reading and editing code. This means that it will -work well with well-structured code but may perform poorly on fully unstructured one (like a "God class" -with enormous, non-modular functions). -Furthermore, for languages that are not statically typed, type annotations are highly beneficial. - -#### Start from a Clean State - -It is best to start a code generation task from a clean git state. Not only will -this make it easier for you to inspect the changes, but also the model itself will -have a chance of seeing what it has changed by calling `git diff` and thereby -correct itself or continue working in a followup conversation if needed. - -:warning: **Important**: since Serena will write to files using the system-native line endings -and it might want to look at the git diff, it is important to -set `git config core.autocrlf` to `true` on Windows. -With `git config core.autocrlf` set to `false` on Windows, you may end up with huge diffs -only due to line endings. It is generally a good idea to globally enable this git setting on Windows: - -```shell -git config --global core.autocrlf true -``` - -#### Logging, Linting, and Automated Tests - -Serena can successfully complete tasks in an _agent loop_, where it iteratively -acquires information, performs actions, and reflects on the results. -However, Serena cannot use a debugger; it must rely on the results of program executions, -linting results, and test results to assess the correctness of its actions. -Therefore, software that is designed to meaningful interpretable outputs (e.g. log messages) -and that has a good test coverage is much easier to work with for Serena. - -We generally recommend to start an editing task from a state where all linting checks and tests pass. - -### Prompting Strategies - -We found that it is often a good idea to spend some time conceptualizing and planning a task -before actually implementing it, especially for non-trivial task. This helps both in achieving -better results and in increasing the feeling of control and staying in the loop. You can -make a detailed plan in one session, where Serena may read a lot of your code to build up the context, -and then continue with the implementation in another (potentially after creating suitable memories). - -### Running Out of Context - -For long and complicated tasks, or tasks where Serena has read a lot of content, you -may come close to the limits of context tokens. In that case, it is often a good idea to continue -in a new conversation. Serena has a dedicated tool to create a summary of the current state -of the progress and all relevant info for continuing it. You can request to create this summary and -write it to a memory. Then, in a new conversation, you can just ask Serena to read the memory and -continue with the task. In our experience, this worked really well. On the up-side, since in a -single session there is no summarization involved, Serena does not usually get lost (unlike some -other agents that summarize under the hood), and it is also instructed to occasionally check whether -it's on the right track. - -Moreover, Serena is instructed to be frugal with context -(e.g., to not read bodies of code symbols unnecessarily), -but we found that Claude is not always very good in being frugal (Gemini seemed better at it). -You can explicitly instruct it to not read the bodies if you know that it's not needed. - -### Serena's Logs: The Dashboard and GUI Tool - -Serena provides two convenient ways of accessing the logs of the current session: - -* via the **web-based dashboard** (enabled by default) - - This is supported on all platforms. - By default, it will be accessible at `http://localhost:24282/dashboard/index.html`, - but a higher port may be used if the default port is unavailable/multiple instances are running. - -* via the **GUI tool** (disabled by default) - - This is mainly supported on Windows, but it may also work on Linux; macOS is unsupported. - -Both can be enabled, configured or disabled in Serena's configuration file (`serena_config.yml`, see above). -If enabled, they will automatically be opened as soon as the Serena agent/MCP server is started. -The web dashboard will display usage statistics of Serena's tools if you set `record_tool_usage_stats: True` in your config. - -In addition to viewing logs, both tools allow to shut down the Serena agent. -This function is provided, because clients like Claude Desktop may fail to terminate the MCP server subprocess -when they themselves are closed. - -## Comparison with Other Coding Agents - -To our knowledge, Serena is the first fully-featured coding agent where the -entire functionality -is available through an MCP server, thus not requiring API keys or -subscriptions. - -### Subscription-Based Coding Agents - -Many prominent subscription-based coding agents are parts of IDEs like -Windsurf, Cursor and VSCode. -Serena's functionality is similar to Cursor's Agent, Windsurf's Cascade or -VSCode's agent mode. - -Serena has the advantage of not requiring a subscription. -A potential disadvantage is that it -is not directly integrated into an IDE, so the inspection of newly written code -is not as seamless. - -More technical differences are: - -* Serena is not bound to a specific IDE or CLI. - Serena's MCP server can be used with any MCP client (including some IDEs), - and the Agno-based agent provides additional ways of applying its functionality. -* Serena is not bound to a specific large language model or API. -* Serena navigates and edits code using a language server, so it has a symbolic - understanding of the code. - IDE-based tools often use a RAG-based or purely text-based approach, which is often - less powerful, especially for large codebases. -* Serena is open-source and has a small codebase, so it can be easily extended - and modified. - -### API-Based Coding Agents - -An alternative to subscription-based agents are API-based agents like Claude -Code, Cline, Aider, Roo Code and others, where the usage costs map directly -to the API costs of the underlying LLM. -Some of them (like Cline) can even be included in IDEs as an extension. -They are often very powerful and their main downside are the (potentially very -high) API costs. - -Serena itself can be used as an API-based agent (see the section on Agno above). -We have not yet written a CLI tool or a -dedicated IDE extension for Serena (and there is probably no need for the latter, as -Serena can already be used with any IDE that supports MCP servers). -If there is demand for a Serena as a CLI tool like Claude Code, we will -consider writing one. - -The main difference between Serena and other API-based agents is that Serena can -also be used as an MCP server, thus not requiring -an API key and bypassing the API costs. This is a unique feature of Serena. - -### Other MCP-Based Coding Agents - -There are other MCP servers designed for coding, like [DesktopCommander](https://github.com/wonderwhy-er/DesktopCommanderMCP) and -[codemcp](https://github.com/ezyang/codemcp). -However, to the best of our knowledge, none of them provide semantic code -retrieval and editing tools; they rely purely on text-based analysis. -It is the integration of language servers and the MCP that makes Serena unique -and so powerful for challenging coding tasks, especially in the context of -larger codebases. - ## Acknowledgements ### Sponsors @@ -815,16 +149,12 @@ what it is today. We built Serena on top of multiple existing open-source technologies, the most important ones being: 1. [multilspy](https://github.com/microsoft/multilspy). - A library which wraps language server implementations and adapts them for interaction via Python - and which provided the basis for our library Solid-LSP (src/solidlsp). + A library which wraps language server implementations and adapts them for interaction via Python. + It provided the basis for our library Solid-LSP (`src/solidlsp`). Solid-LSP provides pure synchronous LSP calls and extends the original library with the symbolic logic that Serena required. 2. [Python MCP SDK](https://github.com/modelcontextprotocol/python-sdk) -3. [Agno](https://github.com/agno-agi/agno) and - the associated [agent-ui](https://github.com/agno-agi/agent-ui), - which we use to allow Serena to work with any model, beyond the ones - supporting the MCP. -4. All the language servers that we use through Solid-LSP. +3. All the language servers that we use through Solid-LSP. Without these projects, Serena would not have been possible (or would have been significantly more difficult to build). @@ -840,52 +170,3 @@ It is also relatively straightforward to add [support for a new programming lang We look forward to seeing what the community will come up with! For details on contributing, see [contributing guidelines](/CONTRIBUTING.md). - -## List of Tools - -Here is the list of Serena's default tools with a short description (output of `uv run serena tools list`): - -* `activate_project`: Activates a project by name. -* `check_onboarding_performed`: Checks whether project onboarding was already performed. -* `create_text_file`: Creates/overwrites a file in the project directory. -* `delete_memory`: Deletes a memory from Serena's project-specific memory store. -* `execute_shell_command`: Executes a shell command. -* `find_file`: Finds files in the given relative paths -* `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type). -* `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). -* `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. -* `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. -* `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. -* `list_dir`: Lists files and directories in the given directory (optionally with recursion). -* `list_memories`: Lists memories in Serena's project-specific memory store. -* `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). -* `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context). -* `read_file`: Reads a file within the project directory. -* `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. -* `replace_regex`: Replaces content in a file by using regular expressions. -* `replace_symbol_body`: Replaces the full definition of a symbol. -* `search_for_pattern`: Performs a search for a pattern in the project. -* `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. -* `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. -* `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed. -* `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. - -There are several tools that are disabled by default, and have to be enabled explicitly, e.g., through the context or modes. -Note that several of our default contexts do enable some of these tools. For example, the `desktop-app` context enables the `execute_shell_command` tool. - -The full list of optional tools is (output of `uv run serena tools list --only-optional`): - -* `delete_lines`: Deletes a range of lines within a file. -* `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. -* `initial_instructions`: Gets the initial instructions for the current project. - Should only be used in settings where the system prompt cannot be set, - e.g. in clients you have no control over, like Claude Desktop. -* `insert_at_line`: Inserts content at a given line in a file. -* `jet_brains_find_referencing_symbols`: Finds symbols that reference the given symbol -* `jet_brains_find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). -* `jet_brains_get_symbols_overview`: Retrieves an overview of the top-level symbols within a specified file -* `remove_project`: Removes a project from the Serena configuration. -* `replace_lines`: Replaces a range of lines within a file with new content. -* `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. -* `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. -* `switch_modes`: Activates modes by providing a list of their names diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..d86fc2c2 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,4 @@ +/_toc.yml +/jupyter_execute +/conf.py +/_build diff --git a/docs/01-about/000_intro.md b/docs/01-about/000_intro.md new file mode 100644 index 00000000..3fbd8b64 --- /dev/null +++ b/docs/01-about/000_intro.md @@ -0,0 +1,14 @@ +# About Serena + +* Serena is a powerful **coding agent toolkit** capable of turning an LLM into a fully-featured agent that works + **directly on your codebase**. + Unlike most other tools, it is not tied to an LLM, framework or an interface, making it easy to use it in a variety of ways. +* Serena provides essential **semantic code retrieval and editing tools** that are akin to an IDE's capabilities, + extracting code entities at the symbol level and exploiting relational structure. + When combined with an existing coding agent, these tools greatly enhance (token) efficiency. +* Serena is **free & open-source**, enhancing the capabilities of LLMs you already have access to free of charge. + +Therefore, you can think of Serena as providing IDE-like tools to your LLM/coding agent. +With it, the agent no longer needs to read entire files, perform grep-like searches or string replacements to find and +edit the right code. +Instead, it can use code-centred tools like `find_symbol`, `find_referencing_symbols` and `insert_after_symbol`. diff --git a/docs/01-about/010_llm-integration.md b/docs/01-about/010_llm-integration.md new file mode 100644 index 00000000..0f9c75aa --- /dev/null +++ b/docs/01-about/010_llm-integration.md @@ -0,0 +1,17 @@ +# LLM Integration + +Serena provides the necessary [tools](035_tools) for coding workflows, but an LLM is required to do the actual work, +orchestrating tool use. + +In general, Serena can be integrated with an LLM in several ways: + +* by using the **model context protocol (MCP)**. + Serena provides an MCP server which integrates with + * Claude Code and Claude Desktop, + * Terminal-based clients like Codex, Gemini-CLI, Qwen3-Coder, rovodev, OpenHands CLI and others, + * IDEs like VSCode, Cursor or IntelliJ, + * Extensions like Cline or Roo Code + * Local clients like [OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp), [Jan](https://jan.ai/docs/mcp-examples/browser/browserbase#enable-mcp), [Agno](https://docs.agno.com/introduction/playground) and others +* by using [mcpo to connect it to ChatGPT](../03-special-guides/serena_on_chatgpt.md) or other clients that don't support MCP but do support tool calling via OpenAPI. +* by incorporating Serena's tools into an agent framework of your choice, as illustrated [here](../03-special-guides/custom_agent). + Serena's tool implementation is decoupled from the framework-specific code and can thus easily be adapted to any agent framework. diff --git a/docs/01-about/020_programming-languages.md b/docs/01-about/020_programming-languages.md new file mode 100644 index 00000000..622be251 --- /dev/null +++ b/docs/01-about/020_programming-languages.md @@ -0,0 +1,65 @@ +# Language Support + +Serena's semantic code analysis capabilities build on **language servers** using the widely implemented +language server protocol (LSP). The LSP provides a set of versatile code querying +and editing functionalities based on symbolic understanding of the code. +Equipped with these capabilities, Serena discovers and edits code just like a seasoned developer +making use of an IDE's capabilities would. +Serena can efficiently find the right context and do the right thing even in very large and +complex projects! So not only is it free and open-source, it frequently achieves better results +than existing solutions that charge a premium. + +Language servers provide support for a wide range of programming languages. +With Serena, we provide direct, out-of-the-box support for: + +* **AL** +* **Bash** +* **C#** +* **C/C++** + (you may experience issues with finding references, we are working on it) +* **Clojure** +* **Dart** +* **Elixir** + (requires installation of NextLS and Elixir; Windows not supported) +* **Elm** + (requires Elm compiler) +* **Erlang** + (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls); experimental, might be slow or hang) +* **F#** + (requires .NET SDK 8.0+ and uses FsAutoComplete/Ionide LSP server) +* **Fortran** + (requires installation of fortls: `pip install fortls`) +* **Go** + (requires installation of `gopls`) +* **Haskell** + (automatically locates HLS via ghcup, stack, or system PATH; supports Stack and Cabal projects) +* **Java** +* **JavaScript** +* **Julia** +* **Kotlin** + (uses the pre-alpha [official kotlin LS](https://github.com/Kotlin/kotlin-lsp), some issues may appear) +* **Lua** +* **Markdown** + (must be explicitly specified via `--language markdown` when generating project config, primarily useful for documentation-heavy projects) +* **Nix** + (requires nixd installation) +* **Perl** + (requires installation of Perl::LanguageServer) +* **PHP** + (uses Intelephense LSP; set `INTELEPHENSE_LICENSE_KEY` environment variable for premium features) +* **Python** +* **R** + (requires installation of the `languageserver` R package) +* **Ruby** + (by default, uses [ruby-lsp](https://github.com/Shopify/ruby-lsp), specify ruby_solargraph as your language to use the previous solargraph based implementation) +* **Rust** + (requires [rustup](https://rustup.rs/) - uses rust-analyzer from your toolchain) +* **Scala** + (requires some [manual setup](../03-special-guides/scala_setup_guide_for_serena); uses Metals LSP) +* **Swift** +* **TypeScript** +* **Zig** + (requires installation of ZLS - Zig Language Server) + +Support for further languages can easily be added by providing a shallow adapter for a new language server implementation, +see Serena's [memory on that](https://github.com/oraios/serena/blob/main/.serena/memories/adding_new_language_support_guide.md). diff --git a/docs/01-about/030_serena-in-action.md b/docs/01-about/030_serena-in-action.md new file mode 100644 index 00000000..ec3212bd --- /dev/null +++ b/docs/01-about/030_serena-in-action.md @@ -0,0 +1,24 @@ +# Serena in Action + +## Demonstration 1: Efficient Operation in Claude Code + +A demonstration of Serena efficiently retrieving and editing code within Claude Code, thereby saving tokens and time. Efficient operations are not only useful for saving costs, but also for generally improving the generated code's quality. This effect may be less pronounced in very small projects, but often becomes of crucial importance in larger ones. + + + +## Demonstration 2: Serena in Claude Desktop + +A demonstration of Serena implementing a small feature for itself (a better log GUI) with Claude Desktop. +Note how Serena's tools enable Claude to find and edit the right symbols. + + diff --git a/docs/01-about/035_tools.md b/docs/01-about/035_tools.md new file mode 100644 index 00000000..55ff8bf2 --- /dev/null +++ b/docs/01-about/035_tools.md @@ -0,0 +1,47 @@ +# List of Tools + +Find the full list of Serena's tools below (output of ` tools list --all`). + +Note that in most configurations, only a subset of these tools will be enabled simultaneously (see the section on [configuration](../02-usage/050_configuration) for details). + +* `activate_project`: Activates a project based on the project name or path. +* `check_onboarding_performed`: Checks whether project onboarding was already performed. +* `create_text_file`: Creates/overwrites a file in the project directory. +* `delete_lines`: Deletes a range of lines within a file. +* `delete_memory`: Deletes a memory from Serena's project-specific memory store. +* `execute_shell_command`: Executes a shell command. +* `find_file`: Finds files in the given relative paths +* `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type). +* `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filter + ed by type). +* `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, + contexts, and modes. +* `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. +* `initial_instructions`: Provides instructions on how to use the Serena toolbox. +* `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. +* `insert_at_line`: Inserts content at a given line in a file. +* `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. +* `jet_brains_find_referencing_symbols`: Finds symbols that reference the given symbol +* `jet_brains_find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (option + ally filtered by type). +* `jet_brains_get_symbols_overview`: Retrieves an overview of the top-level symbols within a specified file +* `list_dir`: Lists files and directories in the given directory (optionally with recursion). +* `list_memories`: Lists memories in Serena's project-specific memory store. +* `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). +* `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with th + e necessary context). +* `read_file`: Reads a file within the project directory. +* `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. +* `remove_project`: Removes a project from the Serena configuration. +* `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities. +* `replace_lines`: Replaces a range of lines within a file with new content. +* `replace_content`: Replaces content in a file (optionally using regular expressions). +* `replace_symbol_body`: Replaces the full definition of a symbol. +* `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. +* `search_for_pattern`: Performs a search for a pattern in the project. +* `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. +* `switch_modes`: Activates modes by providing a list of their names +* `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. +* `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. +* `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed. +* `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. diff --git a/docs/01-about/040_comparison-to-other-agents.md b/docs/01-about/040_comparison-to-other-agents.md new file mode 100644 index 00000000..df89149e --- /dev/null +++ b/docs/01-about/040_comparison-to-other-agents.md @@ -0,0 +1,51 @@ +# Comparison with Other Coding Agents + +To our knowledge, Serena is the first fully-featured coding agent where the +entire functionality is made available through an MCP server, +thus not requiring additional API keys or subscriptions if access to an LLM +is already available through an MCP-compatible client. + +## Subscription-Based Coding Agents + +Many prominent subscription-based coding agents are parts of IDEs like +Windsurf, Cursor and VSCode. +Serena's functionality is similar to Cursor's Agent, Windsurf's Cascade or +VSCode's agent mode. + +Serena has the advantage of not requiring a subscription. + +More technical differences are: + +* Serena navigates and edits code using a language server, so it has a symbolic + understanding of the code. + IDE-based tools often use a text search-based or purely text file-based approach, which is often + less powerful, especially for large codebases. +* Serena is not bound to a specific interface (IDE or CLI). + Serena's MCP server can be used with any MCP client (including some IDEs). +* Serena is not bound to a specific large language model or API. +* Serena is open-source and has a small codebase, so it can be easily extended + and modified. + +## API-Based Coding Agents + +An alternative to subscription-based agents are API-based agents like Claude +Code, Cline, Aider, Roo Code and others, where the usage costs map directly +to the API costs of the underlying LLM. +Some of them (like Cline) can even be included in IDEs as an extension. +They are often very powerful and their main downside are the (potentially very +high) API costs. +Serena itself can be used as an API-based agent (see the [section on Agno](../03-special-guides/custom_agent.md)). + +The main difference between Serena and other API-based agents is that Serena can +also be used as an MCP server, thus not requiring +an API key and bypassing the API costs. + +## Other MCP-Based Coding Agents + +There are other MCP servers designed for coding, like [DesktopCommander](https://github.com/wonderwhy-er/DesktopCommanderMCP) and +[codemcp](https://github.com/ezyang/codemcp). +However, to the best of our knowledge, none of them provide semantic code +retrieval and editing tools; they rely purely on text-based analysis. +It is the integration of language servers and the MCP that makes Serena unique +and so powerful for challenging coding tasks, especially in the context of +larger codebases. \ No newline at end of file diff --git a/docs/01-about/050_acknowledgements.md b/docs/01-about/050_acknowledgements.md new file mode 100644 index 00000000..334a4e6e --- /dev/null +++ b/docs/01-about/050_acknowledgements.md @@ -0,0 +1,34 @@ +# Acknowledgements + +## Sponsors + +We are very grateful to our [sponsors](https://github.com/sponsors/oraios), who help us drive Serena's development. +The core team (the founders of [Oraios AI](https://oraios-ai.de/)) put in a lot of work in order to turn Serena into a useful open source project. +So far, there is no business model behind this project, and sponsors are our only source of income from it. + +Sponsors help us dedicate more time to the project, managing contributions, and working on larger features (like better tooling based on more advanced +LSP features, VSCode integration, debugging via the DAP, and several others). +If you find this project useful to your work, or would like to accelerate the development of Serena, consider becoming a sponsor. + +We are proud to announce that the Visual Studio Code team, together with Microsoft’s Open Source Programs Office and GitHub Open Source +have decided to sponsor Serena with a one-time contribution! + +## Community Contributions + +A significant part of Serena, especially support for various languages, was contributed by the open source community. +We are very grateful for the many contributors who made this possible and who played an important role in making Serena +what it is today. + +## Technologies + +We built Serena on top of multiple existing open-source technologies, the most important ones being: + +1. [multilspy](https://github.com/microsoft/multilspy). + A library which wraps language server implementations and adapts them for interaction via Python + and which provided the basis for our library Solid-LSP (src/solidlsp). + Solid-LSP provides pure synchronous LSP calls and extends the original library with the symbolic logic + that Serena required. +2. [Python MCP SDK](https://github.com/modelcontextprotocol/python-sdk) +3. All the language servers that we use through Solid-LSP. + +Without these projects, Serena would not have been possible (or would have been significantly more difficult to build). diff --git a/docs/02-usage/000_intro.md b/docs/02-usage/000_intro.md new file mode 100644 index 00000000..a2f1fb05 --- /dev/null +++ b/docs/02-usage/000_intro.md @@ -0,0 +1,6 @@ +# Usage + +Serena can be used in various ways and supports coding workflows through a project-based approach. +Its configuration is flexible and allows tailoring it to your specific needs. + +In this section, you will find general usage instructions as well as concrete instructions for selected integrations. diff --git a/docs/02-usage/010_prerequisites.md b/docs/02-usage/010_prerequisites.md new file mode 100644 index 00000000..b408664e --- /dev/null +++ b/docs/02-usage/010_prerequisites.md @@ -0,0 +1,12 @@ +# Prerequisites + +## Package Manager: uv + +Serena is managed by `uv`. +If you do not have it yet, install it following the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). + +## Language-Specific Requirements + +Depending on the programming language you intend to use with Serena, you may need to install additional tools or SDKs if you +intend to use the language server backend of Serena. +See the [language support documentation](../01-about/020_programming-languages) for details. diff --git a/docs/02-usage/020_running.md b/docs/02-usage/020_running.md new file mode 100644 index 00000000..ede78ec2 --- /dev/null +++ b/docs/02-usage/020_running.md @@ -0,0 +1,228 @@ +# Running Serena + +Serena is a command-line tool with a variety of sub-commands. +This section describes + * varies ways of running Serena + * how to run and configure the most important command, i.e. starting the MCP server + * other useful commands. + +## Ways of Running Serena + +In the following, we will refer to the command used to run Serena as ``, +which you should replace with the appropriate command based on your chosen method. + +### Using uvx + +`uvx` is part of `uv`. It can be used to run the latest version of Serena directly from the repository, without an explicit local installation. + + uvx --from git+https://github.com/oraios/serena serena + +Explore the CLI to see some of the customization options that serena provides (more info on them below). + +### Local Installation + +1. Clone the repository and change into it. + + ```shell + git clone https://github.com/oraios/serena + cd serena + ``` + +2. Run Serena via + + ```shell + uv run serena + ``` + + when within the serena installation directory. + From other directories, run it with the `--directory` option, i.e. + + ```shell + uv run --directory /abs/path/to/serena serena + ``` + +:::{note} +Adding the `--directory` option results in the working directory being set to the Serena directory. +As a consequence, you will need to specify paths when using CLI commands that would otherwise operate on the current directory. +::: + +(docker)= +### Using Docker (Experimental) + +:::{warning} +Docker support is currently experimental with several limitations. +::: + +You can run the Serena MCP server directly via docker as follows, +assuming that the projects you want to work on are all located in `/path/to/your/projects`: + +```shell +docker run --rm -i --network host -v /path/to/your/projects:/workspaces/projects ghcr.io/oraios/serena:latest serena +``` + +Replace `/path/to/your/projects` with the absolute path to your projects directory. The Docker approach provides: + +* Better security isolation for shell command execution +* No need to install language servers and dependencies locally +* Consistent environment across different systems + +Alternatively, use docker compose with the `compose.yml` file provided in the repository. + +See our [Docker Setup](https://github.com/oraios/serena/blob/main/DOCKER.md) documentation for more detailed setup instructions, configuration options, and known limitations. + +### Using Nix + +If you are using Nix and [have enabled the `nix-command` and `flakes` features](https://nixos.wiki/wiki/flakes), you can run Serena using the following command: + +```bash +nix run github:oraios/serena -- [options] +``` + +You can also install Serena by referencing this repo (`github:oraios/serena`) and using it in your Nix flake. The package is exported as `serena`. + +(start-mcp-server)= +## Running the MCP Server + +Given your preferred method of running Serena, you can start the MCP server using the `start-mcp-server` command: + + start-mcp-server [options] + +Note that no matter how you run the MCP server, Serena will, by default, start a web-based dashboard on localhost that will allow you to inspect +the server's operations, logs, and configuration. + +### Standard I/O Mode + +The typical usage involves the client (e.g. Claude Code, Codex or Cursor) running +the MCP server as a subprocess and using the process' stdin/stdout streams to communicate with it. +In order to launch the server, the client thus needs to be provided with the command to run the MCP server. + +:::{note} +MCP servers which use stdio as a protocol are somewhat unusual as far as client/server architectures go, as the server +necessarily has to be started by the client in order for communication to take place via the server's standard input/output streams. +In other words, you do not need to start the server yourself. The client application (e.g. Claude Desktop) takes care of this and +therefore needs to be configured with a launch command. +::: + +Communication over stdio is the default for the Serena MCP server, so in the simplest +case, you can simply run the `start-mcp-server` command without any additional options. + + start-mcp-server + +For example, to run the server in stdio mode via `uvx`, you would run: + + uvx --from git+https://github.com/oraios/serena serena start-mcp-server + +See the section ["Configuring Your MCP Client"](030_clients) for specific information on how to configure your MCP client (e.g. Claude Code, Codex, Cursor, etc.) +to use such a launch command. + +(streamable-http)= +### Streamable HTTP Mode + +When using instead the *Streamable HTTP* mode, you control the server lifecycle yourself, +i.e. you start the server and provide the client with the URL to connect to it. + +Simply provide `start-mcp-server` with the `--transport streamable-http` option and optionally provide the desired port +via the `--port` option. + + start-mcp-server --transport streamable-http --port + +For example, to run the Serena MCP server in streamable HTTP mode on port 9121 using uvx, +you would run + + uvx --from git+https://github.com/oraios/serena serena start-mcp-server --transport streamable-http --port 9121 + +and then configure your client to connect to `http://localhost:9121/mcp`. + +Note that while the legacy SSE transport is also supported (via `--transport sse` with corresponding /sse endpoint), its use is discouraged. + +(mcp-args)= +### MCP Server Command-Line Arguments + +The Serena MCP server supports a wide range of additional command-line options. +Use the command + + start-mcp-server --help + +to get a list of all available options. + +Some useful options include: + + * `--project `: specify the project to work on by name or path. + * `--context `: specify the operation [context](contexts) in which Serena shall operate + * `--mode `: specify one or more [modes](modes) to enable (can be passed several times) + * `--enable-web-dashboard `: enable or disable the web dashboard (enabled by default) + +## Other Commands + +Serena provides several other commands in addition to `start-mcp-server`, +most of which are related to project setup and configuration. + +To get a list of available commands, run: + + --help + +To get help on a specific command, run: + + --help + +In general, add `--help` to any command or sub-command to get information about its usage and available options. + +Here are some examples of commands you might find useful: + +```bash +# get help about a sub-command + tools list --help + +# list all available tools + tools list --all + +# get detailed description of a specific tool + tools description find_symbol + +# creating a new Serena project in the current directory + project create + +# creating and immediately indexing a project + project create --index + +# indexing the project in the current directory (auto-creates if needed) + project index + +# run a health check on the project in the current directory + project health-check + +# check if a path is ignored by the project + project is_ignored_path path/to/check + +# edit Serena's configuration file + config edit + +# list available contexts + context list + +# create a new context + context create my-custom-context + +# edit a custom context + context edit my-custom-context + +# list available modes + mode list + +# create a new mode + mode create my-custom-mode + +# edit a custom mode + mode edit my-custom-mode + +# list available prompt definitions + prompts list + +# create an override for internal prompts + prompts create-override prompt-name + +# edit a prompt override + prompts edit-override prompt-name +``` + +Explore the full set of commands and options using the CLI itself! \ No newline at end of file diff --git a/docs/02-usage/030_clients.md b/docs/02-usage/030_clients.md new file mode 100644 index 00000000..79574dce --- /dev/null +++ b/docs/02-usage/030_clients.md @@ -0,0 +1,169 @@ +# Connecting Your MCP Client + +In the following, we provide default configurations for popular MCP-enabled clients. + +Depending on your needs, you might want to customize Serena's behaviour by + * [adding command-line arguments](mcp-args) + * [adjusting configuration](050_configuration). + +## Claude Code + +Serena is a great way to make Claude Code both cheaper and more powerful! + +From your project directory, add serena with a command like this, + +```shell +claude mcp add serena -- start-mcp-server --context claude-code --project "$(pwd)" +``` + +where `` is [your way of running Serena](020_running). +For example, when using `uvx`, the above command becomes + +```shell +claude mcp add serena -- uvx --from git+https://github.com/oraios/serena serena start-mcp-server --context claude-code --project "$(pwd)" +``` + +Note: + * We use the `ide-assistant` context to disable unnecessary tools (avoiding duplication + with Claude Code's built-in capabilities). + * We specify the current directory as the project directory with `--project "$(pwd)"`, such + that Serena is configured to work on the current project from the get-go, following + Claude Code's mode of operation. + +Be sure to use at least `v1.0.52` of Claude Code (as earlier versions do not read MCP server system prompts upon startup). + +## Codex + +Serena works with OpenAI's Codex CLI out of the box, but you have to use the `codex` context for it to work properly. (The technical reason is that Codex doesn't fully support the MCP specifications, so some massaging of tools is required.). + +Add a [run command](020_running) to `~/.codex/config.toml` to configure Serena for all Codex sessions; +create the file if it does not exist. +For example, when using `uvx`, add the following section: + +```toml +[mcp_servers.serena] +command = "uvx" +args = ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server", "--context", "codex"] +``` +After codex has started, you need to activate the project, which you can do by saying: + +"Activate the current dir as project using serena" + +> If you don't activate the project, you will not be able to use Serena's tools! + +That's it! Have a look at `~/.codex/log/codex-tui.log` to see if any errors occurred. + +Serena's dashboard will run if you have not disabled it in the configuration, but due to Codex's sandboxing, the web browser +may not open automatically. You can open it manually by going to `http://localhost:24282/dashboard/index.html` (or a higher port, if +that was already taken). + +> Codex will often show the tools as `failed` even though they are successfully executed. This is not a problem, seems to be a bug in Codex. Despite the error message, everything works as expected. + +## Claude Desktop + +On Windows and macOS there are official [Claude Desktop applications by Anthropic](https://claude.ai/download), for Linux there is an [open-source +community version](https://github.com/aaddrick/claude-desktop-debian). + +To configure MCP server settings, go to File / Settings / Developer / MCP Servers / Edit Config, +which will let you open the JSON file `claude_desktop_config.json`. + +Add the `serena` MCP server configuration, using a [run command](020_running.md) depending on your setup. + +* local installation: + + ```json + { + "mcpServers": { + "serena": { + "command": "/abs/path/to/uv", + "args": ["run", "--directory", "/abs/path/to/serena", "serena", "start-mcp-server"] + } + } + } + ``` + +* uvx: + + ```json + { + "mcpServers": { + "serena": { + "command": "/abs/path/to/uvx", + "args": ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server"] + } + } + } + ``` + +* docker: + + ```json + { + "mcpServers": { + "serena": { + "command": "docker", + "args": ["run", "--rm", "-i", "--network", "host", "-v", "/path/to/your/projects:/workspaces/projects", "ghcr.io/oraios/serena:latest", "serena", "start-mcp-server", "--transport", "stdio"] + } + } + } + ``` + +If you are using paths containing backslashes for paths on Windows +(note that you can also just use forward slashes), be sure to escape them correctly (`\\`). + +That's it! Save the config and then restart Claude Desktop. You are ready for activating your first project. + +:::{attention} +Be sure to fully quit the Claude Desktop application via File / Exit, as regularly closing the application will just minimize it. +::: + +After restarting, you should see Serena's tools in your chat interface (notice the small hammer icon). + +For more information on MCP servers with Claude Desktop, see [the official quick start guide](https://modelcontextprotocol.io/quickstart/user). + +## Other Clients + +In general, Serena can be used with any MCP-enabled client. +To connect Serena to your favourite client, simply + +1. determine how to add a custom MCP server to your client (refer to the client's documentation). +2. add a new MCP server entry by specifying either + * a [run command](start-mcp-server) that allows the client to start the MCP server in stdio mode as a subprocess, or + * the URL of the HTTP/SSE endpoint, having started the [Serena MCP server in HTTP/SSE mode](streamable-http) beforehand. + +Below, we list some popular categories of clients that Serena can be used with. + +### Terminal-Based Clients + +There are many terminal-based coding assistants that support MCP servers, such as + + * [Gemini-CLI](https://github.com/google-gemini/gemini-cli), + * [Qwen3-Coder](https://github.com/QwenLM/Qwen3-Coder), + * [rovodev](https://community.atlassian.com/forums/Rovo-for-Software-Teams-Beta/Introducing-Rovo-Dev-CLI-AI-Powered-Development-in-your-terminal/ba-p/3043623), + * [OpenHands CLI](https://docs.all-hands.dev/usage/how-to/cli-mode) and + * [opencode](https://github.com/sst/opencode). + +They generally benefit from the symbolic tools provided by Serena. You might want to customize some aspects of Serena +by writing your own context, modes or prompts to adjust it to the client's respective internal capabilities (and your general workflow). + +### MCP-Enabled IDEs and Coding Clients (Cline, Roo-Code, Cursor, Windsurf, etc.) + +Most of the popular existing coding assistants (e.g. IDE extensions) and AI-enabled IDEs themselves support connections +to MCP Servers. Serena generally boosts performance by providing efficient tools for symbolic operations. + +We generally **recommend to use the `ide-assistant` context** for these integrations by adding the arguments `--context ide-assistant` +in order to reduce tool duplication. + +### Local GUIs and Agent Frameworks + +Over the last months, several technologies have emerged that allow you to run a local GUI client +and connect it to an MCP server. The respective applications will typically work with Serena out of the box. +Some of the leading open source GUI applications are + + * [Jan](https://jan.ai/docs/mcp), + * [OpenHands](https://github.com/All-Hands-AI/OpenHands/), + * [OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp) and + * [Agno](https://docs.agno.com/introduction/playground). + +These applications allow to combine Serena with almost any LLM (including locally running ones) +and offer various other integrations. diff --git a/docs/02-usage/040_workflow.md b/docs/02-usage/040_workflow.md new file mode 100644 index 00000000..35afb55e --- /dev/null +++ b/docs/02-usage/040_workflow.md @@ -0,0 +1,138 @@ +# The Project Workflow + +Serena uses a project-based workflow. +A **project** is simply a directory on your filesystem that contains code and other files +that you want Serena to work with. + +Assuming that you have project you want to work with (which may initially be empty), +setting up a project with Serena typically involves the following steps: + +1. **Project creation**: Configuring project settings for Serena (and indexing the project, if desired) +2. **Project activation**: Making Serena aware of the project you want to work with +3. **Onboarding**: Getting Serena familiar with the project (creating memories) +4. **Working on coding tasks**: Using Serena to help you with actual coding tasks in the project + +(project-creation-indexing)= +## Project Creation & Indexing + +You can create a project either + * implicitly, by just activating a directory as a project while already in a conversation; this will use default settings for your project (skip to the next section). + * explicitly, using the project creation command, or + +### Explicit Project Creation + +To explicitly create a project, use the following command while in the project directory: + + project create [options] + +For instance, when using `uvx`, run + + uvx --from git+https://github.com/oraios/serena serena project create [options] + + * For an empty project, you will need to specify the programming language + (e.g., `--language python`). + * For an existing project, the main programming language will be detected automatically, + but you can choose to explicitly specify multiple languages by passing the `--language` parameter + multiple times (e.g. `--language python --language typescript`). + * You can optionally specify a custom project name with `--name "My Project"`. + * You can immediately index the project after creation with `--index`. + +After creation, you can adjust the project settings in the generated `.serena/project.yml` file. + +(indexing)= +### Indexing + +Especially for larger project, it is advisable to index the project after creation (in order to avoid +delays during MCP server startup or the first tool application): + +While in the project directory, run this command: + + project index + +Indexing has to be called only once. During regular usage, Serena will automatically update the index whenever files change. + +## Project Activation + +Project activation makes Serena aware of the project you want to work with. +You can either choose to do this + * while in a conversation, by telling the model to activate a project, e.g., + + * "Activate the project /path/to/my_project" (for first-time activation with auto-creation) + * "Activate the project my_project" + + Note that this option requires the `activate_project` tool to be active, + which it isn't in the (default version) of context `ide-assistant` if a project is provided at startup. + (The tool is deactivated because we assume that in the ide-assistant context the user will only work on the open project and have + no need to switch it.) + + * when the MCP server starts, by passing the project path or name as a command-line argument + (e.g. when working on a fixed project in `ide-assistant` mode): `--project ` + + +## Onboarding & Memories + +By default, Serena will perform an **onboarding process** when +it is started for the first time for a project. +The goal of the onboarding is for Serena to get familiar with the project +and to store memories, which it can then draw upon in future interactions. +If an LLM should fail to complete the onboarding and does not actually write the +respective memories to disk, you may need to ask it to do so explicitly. + +The onboarding will usually read a lot of content from the project, thus filling +up the context. It can therefore be advisable to switch to another conversation +once the onboarding is complete. +After the onboarding, we recommend that you have a quick look at the memories and, +if necessary, edit them or add additional ones. + +**Memories** are files stored in `.serena/memories/` in the project directory, +which the agent can choose to read in subsequent interactions. +Feel free to read and adjust them as needed; you can also add new ones manually. +Every file in the `.serena/memories/` directory is a memory file. +Whenever Serena starts working on a project, the list of memories is +provided, and the agent can decide to read them. +We found that memories can significantly improve the user experience with Serena. + + +## Preparing Your Project + +When using Serena to work on your project, it can be helpful to follow a few best practices. + +### Structure Your Codebase + +Serena uses the code structure for finding, reading and editing code. This means that it will +work well with well-structured code but may perform poorly on fully unstructured one (like a "God class" +with enormous, non-modular functions). + +Furthermore, for languages that are not statically typed, the use of type annotations (if supported) +are highly beneficial. + +### Start from a Clean State + +It is best to start a code generation task from a clean git state. Not only will +this make it easier for you to inspect the changes, but also the model itself will +have a chance of seeing what it has changed by calling `git diff` and thereby +correct itself or continue working in a followup conversation if needed. + +### Use Platform-Native Line Endings + +**Important**: since Serena will write to files using the system-native line endings +and it might want to look at the git diff, it is important to +set `git config core.autocrlf` to `true` on Windows. +With `git config core.autocrlf` set to `false` on Windows, you may end up with huge diffs +due to line endings only. +It is generally a good idea to globally enable this git setting on Windows: + +```shell +git config --global core.autocrlf true +``` + +### Logging, Linting, and Automated Tests + +Serena can successfully complete tasks in an _agent loop_, where it iteratively +acquires information, performs actions, and reflects on the results. +However, Serena cannot use a debugger; it must rely on the results of program executions, +linting results, and test results to assess the correctness of its actions. +Therefore, software that is designed to meaningful interpretable outputs (e.g. log messages) +and that has a good test coverage is much easier to work with for Serena. + +We generally recommend to start an editing task from a state where all linting checks and tests pass. \ No newline at end of file diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md new file mode 100644 index 00000000..ce3de259 --- /dev/null +++ b/docs/02-usage/050_configuration.md @@ -0,0 +1,135 @@ +# Configuration + +Serena is very flexible in terms of configuration. While for most users, the default configurations will work, +you can fully adjust it to your needs by editing a few yaml files. You can disable tools, change Serena's instructions +(what we denote as the `system_prompt`), adjust the output of tools that just provide a prompt, and even adjust tool descriptions. + +Serena is configured in four places: + +1. The `serena_config.yml` for general settings that apply to all clients and projects. + It is located in your user directory under `.serena/serena_config.yml`. + If you do not explicitly create the file, it will be auto-generated when you first run Serena. + You can edit it directly or use + + ```shell + uvx --from git+https://github.com/oraios/serena serena config edit + ``` + + (or use the `--directory` command version). +2. In the arguments passed to the `start-mcp-server` in your client's config (see below), + which will apply to all sessions started by the respective client. In particular, the [context](contexts) parameter + should be set appropriately for Serena to be best adjusted to existing tools and capabilities of your client. + See for a detailed explanation. You can override all entries from the `serena_config.yml` through command line arguments. +3. In the `.serena/project.yml` file within your project. This will hold project-level configuration that is used whenever + that project is activated. This file will be autogenerated when you first use Serena on that project, but you can also + create it explicitly with `serena project create [options]` (see the [](project-creation-indexing) + for details on available options). +4. Through the context and modes (see below). + +After the initial setup, continue with one of the sections below, depending on how you +want to use Serena. + +## Modes and Contexts + +Serena's behavior and toolset can be adjusted using contexts and modes. +These allow for a high degree of customization to best suit your workflow and the environment Serena is operating in. + +(contexts)= +### Contexts + +A **context** defines the general environment in which Serena is operating. +It influences the initial system prompt and the set of available tools. +A context is set at startup when launching Serena (e.g., via CLI options for an MCP server or in the agent script) and cannot be changed during an active session. + +Serena comes with pre-defined contexts: + +* `desktop-app`: Tailored for use with desktop applications like Claude Desktop. This is the default. + The full set of Serena's tools is provided, as the application is assumed to have no prior coding-specific capabilities. +* `claude-code`: Optimized for use with Claude Code, it disables tools that would duplicate Claude Code's built-in capabilities. +* `codex`: Optimized for use with OpenAI Codex. +* `ide`: Generic context for IDE assistants/coding agents, e.g. VSCode, Cursor, or Cline, focusing on augmenting existing capabilities. + Basic file operations and shell execution are assumed to be handled by the assistant's own capabilities. +* `agent`: Designed for scenarios where Serena acts as a more autonomous agent, for example, when used with Agno. + +* Choose the context that best matches the type of integration you are using. + +Find the concrete definitions of these modes [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/contexts). + +When launching Serena, specify the context using `--context `. +Note that for cases where parameter lists are specified (e.g. Claude Desktop), you must add two parameters to the list. + +If you are using a local server (such as Llama.cpp) which requires you to use OpenAI-compatible tool descriptions, use context `oaicompat-agent` instead of `agent`. + +You can manage contexts using the `context` command, + + context --help + context list + context create + context edit + context delete + +where `` is [your way of running Serena](020_running). + +(modes)= +### Modes + +Modes further refine Serena's behavior for specific types of tasks or interaction styles. Multiple modes can be active simultaneously, allowing you to combine their effects. Modes influence the system prompt and can also alter the set of available tools by excluding certain ones. + +Examples of built-in modes include: + +* `planning`: Focuses Serena on planning and analysis tasks. +* `editing`: Optimizes Serena for direct code modification tasks. +* `interactive`: Suitable for a conversational, back-and-forth interaction style. +* `one-shot`: Configures Serena for tasks that should be completed in a single response, often used with `planning` for generating reports or initial plans. +* `no-onboarding`: Skips the initial onboarding process if it's not needed for a particular session but retains the memory tools (assuming initial memories were created externally). +* `onboarding`: Focuses on the project onboarding process. +* `no-memories`: Disables all memory tools (and tools building on memories such as onboarding tools) + +Find the concrete definitions of these modes [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/modes). + +Modes can be set at startup (similar to contexts) but can also be _switched dynamically_ during a session. +You can instruct the LLM to use the `switch_modes` tool to activate a different set of modes (e.g., "Switch to planning and one-shot modes"). + +When launching Serena, specify modes using `--mode `; multiple modes can be specified, e.g. `--mode planning --mode no-onboarding`. + +:::{attention} +**Mode Compatibility**: While you can combine modes, some may be semantically incompatible (e.g., `interactive` and `one-shot`). +Serena currently does not prevent incompatible combinations; it is up to the user to choose sensible mode configurations. +::: + +You can manage modes using the `mode` command, + + mode --help + mode list + mode create + mode edit + mode delete + +where `` is [your way of running Serena](020_running). + +## Advanced Configuration + +For advanced users, Serena's configuration can be further customized. + +### Serena Data Directory + +The Serena user data directory (where configuration, language server files, logs, etc. are stored) defaults to `~/.serena`. +You can change this location by setting the `SERENA_HOME` environment variable to your desired path. + +### Custom Prompts + +All of Serena's prompts can be fully customized. +We define prompt as jinja templates in yaml files, and you can inspect our default prompts [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/prompt_templates). + +To override a prompt, simply add a .yml file to the `prompt_templates` folder in your Serena data directory +which defines the prompt with the same name as the default prompt you want to override. +For example, to override the `system_prompt`, you could create a file `~/.serena/prompt_templates/system_prompt.yml` (assuming default Serena data folder location) +with content like: + +```yaml +prompts: + system_prompt: | + Whatever you want ... +``` + +It is advisable to use the default prompt as a starting point and modify it to suit your needs. diff --git a/docs/02-usage/060_dashboard.md b/docs/02-usage/060_dashboard.md new file mode 100644 index 00000000..73376926 --- /dev/null +++ b/docs/02-usage/060_dashboard.md @@ -0,0 +1,22 @@ +# The Dashboard and GUI Tool + +Serena comes with built-in tools for monitoring and managing the current session: + +* the **web-based dashboard** (enabled by default) + + The dashboard provides detailed information on your Serena session, the current configuration and provides access to logs. + Some settings (e.g. the current set of active programming languages) can also be directly modified through the dashboard. + + The dashboard is supported on all platforms. + + By default, it will be accessible at `http://localhost:24282/dashboard/index.html`, + but a higher port may be used if the default port is unavailable/multiple instances are running. + +* the **GUI tool** (disabled by default) + + The GUI tool is a native application window which displays logs. + + This is mainly supported on Windows, but it may also work on Linux; macOS is unsupported. + +Both can be configured in Serena's [configuration](050_configuration) file (`serena_config.yml`). +If enabled, they will automatically be opened as soon as the Serena agent/MCP server is started. diff --git a/docs/02-usage/070_security.md b/docs/02-usage/070_security.md new file mode 100644 index 00000000..12faabdf --- /dev/null +++ b/docs/02-usage/070_security.md @@ -0,0 +1,12 @@ +# Security Considerations + +As fundamental abilities for a coding agent, Serena contains tools for executing shell commands and modifying files. +Therefore, if the respective tool calls are not monitored or restricted (and execution takes place in a sensitive environment), +there is a risk of unintended consequences. + +Therefore, to reduce the risk of unintended consequences when using Serena, it is recommended to + * back up your work regularly (e.g. use a version control system like Git), + * monitor tool executions carefully (e.g. via your MCP client, provided that it supports it), + * consider enabling read-only mode for your project (set `read_only: True` in project.yml) if you only want to analyze code without modifying it, + * restrict the set of allowed tools via the [configuration](050_configuration), + * use a sandboxed environment for running Serena (e.g. by [using Docker](docker)). diff --git a/docs/02-usage/999_additional-usage.md b/docs/02-usage/999_additional-usage.md new file mode 100644 index 00000000..275fd190 --- /dev/null +++ b/docs/02-usage/999_additional-usage.md @@ -0,0 +1,32 @@ +# Additional Usage Pointers + +## Prompting Strategies + +We found that it is often a good idea to spend some time conceptualizing and planning a task +before actually implementing it, especially for non-trivial task. This helps both in achieving +better results and in increasing the feeling of control and staying in the loop. You can +make a detailed plan in one session, where Serena may read a lot of your code to build up the context, +and then continue with the implementation in another (potentially after creating suitable memories). + +## Running Out of Context + +For long and complicated tasks, or tasks where Serena has read a lot of content, you +may come close to the limits of context tokens. In that case, it is often a good idea to continue +in a new conversation. Serena has a dedicated tool to create a summary of the current state +of the progress and all relevant info for continuing it. You can request to create this summary and +write it to a memory. Then, in a new conversation, you can just ask Serena to read the memory and +continue with the task. In our experience, this worked really well. On the up-side, since in a +single session there is no summarization involved, Serena does not usually get lost (unlike some +other agents that summarize under the hood), and it is also instructed to occasionally check whether +it's on the right track. + +Serena instructs the LLM to be economical in general, so the problem of running out of context +should not occur too often, unless the task is very large or complicated. + +## Serena and Git Worktrees + +[git-worktree](https://git-scm.com/docs/git-worktree) can be an excellent way to parallelize your work. More on this in [Anthropic: Run parallel Claude Code sessions with Git worktrees](https://docs.claude.com/en/docs/claude-code/common-workflows#run-parallel-claude-code-sessions-with-git-worktrees). + +When it comes to serena AND git-worktree AND larger projects (that take longer to index), +the recommended way is to COPY your `$ORIG_PROJECT/.serena/cache` to `$GIT_WORKTREE/.serena/cache`. +Perform [pre-indexing of your project](indexing) to avoid having to re-index per each worktree you create. diff --git a/docs/03-special-guides/000_intro.md b/docs/03-special-guides/000_intro.md new file mode 100644 index 00000000..d2d34716 --- /dev/null +++ b/docs/03-special-guides/000_intro.md @@ -0,0 +1,3 @@ +# Special Guides + +This section contains special guides for certain topics that require more in-depth explanations. \ No newline at end of file diff --git a/docs/custom_agent.md b/docs/03-special-guides/custom_agent.md similarity index 96% rename from docs/custom_agent.md rename to docs/03-special-guides/custom_agent.md index d1d7afde..add268dd 100644 --- a/docs/custom_agent.md +++ b/docs/03-special-guides/custom_agent.md @@ -63,5 +63,5 @@ configuration file (`.yml`). It should be straightforward to incorporate Serena into any agent framework (like [pydantic-ai](https://ai.pydantic.dev/), [langgraph](https://langchain-ai.github.io/langgraph/tutorials/introduction/) or others). Typically, you need only to write an adapter for Serena's tools to the tool representation in the framework of your choice, -as was done by us for Agno with [SerenaAgnoToolkit](/src/serena/agno.py). +as was done by us for Agno with `SerenaAgnoToolkit` (see `/src/serena/agno.py`). diff --git a/docs/03-special-guides/scala_setup_guide_for_serena.md b/docs/03-special-guides/scala_setup_guide_for_serena.md new file mode 100644 index 00000000..2c4b1cc8 --- /dev/null +++ b/docs/03-special-guides/scala_setup_guide_for_serena.md @@ -0,0 +1,67 @@ +# Scala Setup Guide for Serena + +This guide explains how to prepare a Scala project so that Serena can provide reliable code intelligence via Metals (Scala LSP) and how to run Scala tests manually. + +Serena automatically bootstraps the Metals language server using Coursier when needed. Your project, however, must be importable by a build server (BSP) — typically via Bloop or sbt’s built‑in BSP — so that Metals can compile and index your code. + +--- +## Prerequisites + +Install the following on your system and ensure they are available on `PATH`: + +- Java Development Kit (JDK). A modern LTS (e.g., 17 or 21) is recommended. +- `sbt` +- Coursier command (`cs`) or the legacy `coursier` launcher + - Serena uses `cs` if available; if only `coursier` exists, it will attempt to install `cs`. If neither is present, install Coursier first. + +--- +## Quick Start (Recommended: VS Code + Metals auto‑import) + +1. Open your Scala project in VS Code. +2. When prompted by Metals, accept “Import build”. Wait until the import and initial compile/indexing finish. +3. Run the “Connect to build server” command (id: `build.connect`). +4. Once the import completes, start Serena in your project root and use it. + +This flow ensures the `.bloop/` and (if applicable) `.metals/` directories are created and your build is known to the build server that Metals uses. + +--- +## Manual Setup (No VS Code) + +Follow these steps if you prefer a manual setup or you are not using VS Code: + +These instructions cover the setup for projects that use sbt as the build tool, with Bloop as the BSP server. + + +1. Add Bloop to `project/plugins.sbt` in your Scala project: + ```scala + // project/plugins.sbt + addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "") + ``` + Replace `` with an appropriate current version from the Metals documentation. + +2. Export Bloop configuration with sources: + ```bash + sbt -Dbloop.export-jar-classifiers=sources bloopInstall + ``` + This creates a `.bloop/` directory containing your project’s build metadata for the BSP server. + +3. Compile from sbt to verify the build: + ```bash + sbt compile + ``` + +4. Start Serena in your project root. Serena will bootstrap Metals (if not already present) and connect to the build server using the configuration exported above. + +--- +## Using Serena with Scala + +- Serena automatically detects Scala files (`*.scala`, `*.sbt`) and will start a Metals process per project when needed. +- On first run, you may see messages like “Bootstrapping metals…” in the Serena logs — this is expected. +- Optimal results require that your project compiles successfully via the build server (BSP). If compilation fails, fix build errors in `sbt` first. + + +Notes: +- Ensure you completed the manual or auto‑import steps so that the build is compiled and indexed; otherwise, code navigation and references may be incomplete until the first successful compile. + +## Reference +- Metals + sbt: [https://scalameta.org/metals/docs/build-tools/sbt](https://scalameta.org/metals/docs/build-tools/sbt) \ No newline at end of file diff --git a/docs/serena_on_chatgpt.md b/docs/03-special-guides/serena_on_chatgpt.md similarity index 100% rename from docs/serena_on_chatgpt.md rename to docs/03-special-guides/serena_on_chatgpt.md diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 00000000..dbea0794 --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,154 @@ +# Book settings +# Learn more at https://jupyterbook.org/customize/config.html + +####################################################################################### +# A default configuration that will be loaded for all jupyter books +# Users are expected to override these values in their own `_config.yml` file. +# This is also the "master list" of all allowed keys and values. + +####################################################################################### +# Book settings +title : Serena Documentation # The title of the book. Will be placed in the left navbar. +author : Oraios AI & Oraios Software # The author of the book +copyright : "2025 by Serena contributors" # Copyright year to be placed in the footer +# Patterns to skip when building the book. Can be glob-style (e.g. "*skip.ipynb") +exclude_patterns : ['**.ipynb_checkpoints', '.DS_Store', 'Thumbs.db', '_build', 'jupyter_execute', '.jupyter_cache', '.pytest_cache', 'docs/autogen_rst.py', 'docs/create_toc.py'] +# Auto-exclude files not in the toc +only_build_toc_files : true + +####################################################################################### +# Execution settings +execute: + # NOTE: Notebooks are not executed, because test_notebooks.py executes them and stores them with outputs in the docs/ folder + # NOTE: If changed, repeat below in `nb_execution_mode`. + execute_notebooks : "off" # Whether to execute notebooks at build time. Must be one of ("auto", "force", "cache", "off") + cache : "" # A path to the jupyter cache that will be used to store execution artifacts. Defaults to `_build/.jupyter_cache/` + exclude_patterns : [] # A list of patterns to *skip* in execution (e.g. a notebook that takes a really long time) + timeout : 1000 # The maximum time (in seconds) each notebook cell is allowed to run. + run_in_temp : false # If `True`, then a temporary directory will be created and used as the command working directory (cwd), + # otherwise the notebook's parent directory will be the cwd. + allow_errors : true # If `False`, when a code cell raises an error the execution is stopped, otherwise all cells are always run. + stderr_output : show # One of 'show', 'remove', 'remove-warn', 'warn', 'error', 'severe' + +####################################################################################### +# Parse and render settings +parse: + myst_enable_extensions: # default extensions to enable in the myst parser. See https://myst-parser.readthedocs.io/en/latest/using/syntax-optional.html + - amsmath + - colon_fence + # - deflist + - dollarmath + # - html_admonition + # - html_image + - linkify + # - replacements + # - smartquotes + - substitution + - tasklist + - html_admonition + - html_image + myst_url_schemes: [ mailto, http, https ] # URI schemes that will be recognised as external URLs in Markdown links + myst_dmath_double_inline: true # Allow display math ($$) within an inline context + +####################################################################################### +# HTML-specific settings +html: + #favicon : # A path to a favicon image + use_edit_page_button : false # Whether to add an "edit this page" button to pages. If `true`, repository information in repository: must be filled in + use_repository_button : false # Whether to add a link to your repository button + use_issues_button : false # Whether to add an "open an issue" button + use_multitoc_numbering : true # Continuous numbering across parts/chapters + use_darkmode_button : false + extra_footer : "" + home_page_in_navbar : true # Whether to include your home page in the left Navigation Bar + baseurl : "https://oraios.github.io/serena/" + comments: + hypothesis : false + utterances : false + announcement : "" # A banner announcement at the top of the site. + +####################################################################################### +# LaTeX-specific settings +latex: + latex_engine : pdflatex # one of 'pdflatex', 'xelatex' (recommended for unicode), 'luatex', 'platex', 'uplatex' + use_jupyterbook_latex : true # use sphinx-jupyterbook-latex for pdf builds as default + targetname : book.tex +# Add a bibtex file so that we can create citations +#bibtex_bibfiles: +# - refs.bib + +####################################################################################### +# Launch button settings +launch_buttons: + notebook_interface : classic # The interface interactive links will activate ["classic", "jupyterlab"] + binderhub_url : "" # The URL of the BinderHub (e.g., https://mybinder.org) + jupyterhub_url : "" # The URL of the JupyterHub (e.g., https://datahub.berkeley.edu) + thebe : false # Add a thebe button to pages (requires the repository to run on Binder) + colab_url : "https://colab.research.google.com" + +repository: + url : https://github.com/oraios/serena # The URL to your book's repository + path_to_book : docs # A path to your book's folder, relative to the repository root. + branch : main # Which branch of the repository should be used when creating links + +####################################################################################### +# Advanced and power-user settings +sphinx: + extra_extensions : + - sphinx.ext.autodoc + - sphinx.ext.viewcode + - sphinx_toolbox.more_autodoc.sourcelink + #- sphinxcontrib.spelling + local_extensions : # A list of local extensions to load by sphinx specified by "name: path" items + recursive_update : false # A boolean indicating whether to overwrite the Sphinx config (true) or recursively update (false) + config : # key-value pairs to directly over-ride the Sphinx configuration + master_doc: "01-about/000_intro.md" + html_theme_options: + logo: + image_light: ../resources/serena-logo.svg + image_dark: ../resources/serena-logo-dark-mode.svg + autodoc_typehints_format: "short" + autodoc_member_order: "bysource" + autoclass_content: "both" + autodoc_default_options: + show-inheritance: True + autodoc_show_sourcelink: True + add_module_names: False + github_username: oraios + github_repository: serena + nb_execution_mode: "off" + nb_merge_streams: True # This is important for cell outputs to appear as single blocks rather than one block per line + python_use_unqualified_type_names: True + nb_mime_priority_overrides: [ + [ 'html', 'application/vnd.jupyter.widget-view+json', 10 ], + [ 'html', 'application/javascript', 20 ], + [ 'html', 'text/html', 30 ], + [ 'html', 'text/latex', 40 ], + [ 'html', 'image/svg+xml', 50 ], + [ 'html', 'image/png', 60 ], + [ 'html', 'image/jpeg', 70 ], + [ 'html', 'text/markdown', 80 ], + [ 'html', 'text/plain', 90 ], + [ 'spelling', 'application/vnd.jupyter.widget-view+json', 10 ], + [ 'spelling', 'application/javascript', 20 ], + [ 'spelling', 'text/html', 30 ], + [ 'spelling', 'text/latex', 40 ], + [ 'spelling', 'image/svg+xml', 50 ], + [ 'spelling', 'image/png', 60 ], + [ 'spelling', 'image/jpeg', 70 ], + [ 'spelling', 'text/markdown', 80 ], + [ 'spelling', 'text/plain', 90 ], + ] + mathjax_path: https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js + mathjax3_config: + loader: { load: [ '[tex]/configmacros' ] } + tex: + packages: { '[+]': [ 'configmacros' ] } + macros: + vect: ["{\\mathbf{\\boldsymbol{#1}} }", 1] + E: "{\\mathbb{E}}" + P: "{\\mathbb{P}}" + R: "{\\mathbb{R}}" + abs: ["{\\left| #1 \\right|}", 1] + simpl: ["{\\Delta^{#1} }", 1] + amax: "{\\text{argmax}}" diff --git a/docs/autogen_rst.py b/docs/autogen_rst.py new file mode 100644 index 00000000..e59c2f70 --- /dev/null +++ b/docs/autogen_rst.py @@ -0,0 +1,155 @@ +import logging +import os +import shutil +from pathlib import Path +from typing import Optional, List + +log = logging.getLogger(os.path.basename(__file__)) + +TOP_LEVEL_PACKAGE = "serena" +PROJECT_NAME = "Serena" + +def module_template(module_qualname: str): + module_name = module_qualname.split(".")[-1] + title = module_name.replace("_", r"\_") + return f"""{title} +{"=" * len(title)} + +.. automodule:: {module_qualname} + :members: + :undoc-members: +""" + + +def index_template(package_name: str, doc_references: Optional[List[str]] = None, text_prefix=""): + doc_references = doc_references or "" + if doc_references: + doc_references = "\n" + "\n".join(f"* :doc:`{ref}`" for ref in doc_references) + "\n" + + dirname = package_name.split(".")[-1] + title = dirname.replace("_", r"\_") + if title == TOP_LEVEL_PACKAGE: + title = "API Reference" + return f"{title}\n{'=' * len(title)}" + text_prefix + doc_references + + +def write_to_file(content: str, path: str): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(content) + os.chmod(path, 0o666) + + +_SUBTITLE = ( + f"\n Here is the autogenerated documentation of the {PROJECT_NAME} API. \n \n " + "The Table of Contents to the left has the same structure as the " + "repository's package code. The links at each page point to the submodules and subpackages. \n" +) + + +def make_rst(src_root, rst_root, clean=False, overwrite=False, package_prefix=""): + """Creates/updates documentation in form of rst files for modules and packages. + + Does not delete any existing rst files. Thus, rst files for packages or modules that have been removed or renamed + should be deleted by hand. + + This method should be executed from the project's top-level directory + + :param src_root: path to library base directory, typically "src/" + :param rst_root: path to the root directory to which .rst files will be written + :param clean: whether to completely clean the target directory beforehand, removing any existing .rst files + :param overwrite: whether to overwrite existing rst files. This should be used with caution as it will delete + all manual changes to documentation files + :package_prefix: a prefix to prepend to each module (for the case where the src_root is not the base package), + which, if not empty, should end with a "." + :return: + """ + rst_root = os.path.abspath(rst_root) + + if clean and os.path.isdir(rst_root): + shutil.rmtree(rst_root) + + base_package_name = package_prefix + os.path.basename(src_root) + + # TODO: reduce duplication with same logic for subpackages below + files_in_dir = os.listdir(src_root) + module_names = [f[:-3] for f in files_in_dir if f.endswith(".py") and not f.startswith("_")] + subdir_refs = [ + f"{f}/index" + for f in files_in_dir + if os.path.isdir(os.path.join(src_root, f)) + and not f.startswith("_") + and not f.startswith(".") + ] + package_index_rst_path = os.path.join( + rst_root, + "index.rst", + ) + log.info(f"Writing {package_index_rst_path}") + write_to_file( + index_template( + base_package_name, + doc_references=module_names + subdir_refs, + text_prefix=_SUBTITLE, + ), + package_index_rst_path, + ) + + for root, dirnames, filenames in os.walk(src_root): + if os.path.basename(root).startswith("_"): + continue + base_package_relpath = os.path.relpath(root, start=src_root) + base_package_qualname = package_prefix + os.path.relpath( + root, + start=os.path.dirname(src_root), + ).replace(os.path.sep, ".") + + for dirname in dirnames: + if dirname.startswith("_"): + log.debug(f"Skipping {dirname}") + continue + files_in_dir = os.listdir(os.path.join(root, dirname)) + module_names = [ + f[:-3] for f in files_in_dir if f.endswith(".py") and not f.startswith("_") + ] + subdir_refs = [ + f"{f}/index" + for f in files_in_dir + if os.path.isdir(os.path.join(root, dirname, f)) and not f.startswith("_") + ] + package_qualname = f"{base_package_qualname}.{dirname}" + package_index_rst_path = os.path.join( + rst_root, + base_package_relpath, + dirname, + "index.rst", + ) + log.info(f"Writing {package_index_rst_path}") + write_to_file( + index_template(package_qualname, doc_references=module_names + subdir_refs), + package_index_rst_path, + ) + + for filename in filenames: + base_name, ext = os.path.splitext(filename) + if ext == ".py" and not filename.startswith("_"): + module_qualname = f"{base_package_qualname}.{filename[:-3]}" + + module_rst_path = os.path.join(rst_root, base_package_relpath, f"{base_name}.rst") + if os.path.exists(module_rst_path) and not overwrite: + log.debug(f"{module_rst_path} already exists, skipping it") + + log.info(f"Writing module documentation to {module_rst_path}") + write_to_file(module_template(module_qualname), module_rst_path) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + docs_root = Path(__file__).parent + enable_module_docs = False + if enable_module_docs: + make_rst( + docs_root / ".." / "src" / "serena", + docs_root / "serena", + clean=True, + ) diff --git a/docs/create_toc.py b/docs/create_toc.py new file mode 100644 index 00000000..39490e88 --- /dev/null +++ b/docs/create_toc.py @@ -0,0 +1,9 @@ +import os +from pathlib import Path + +# This script provides a platform-independent way of making the jupyter-book call (used in pyproject.toml) +folder = Path(__file__).parent +toc_file = folder / "_toc.yml" +cmd = f"jupyter-book toc from-project docs -e .rst -e .md -e .ipynb >{toc_file}" +print(cmd) +os.system(cmd) diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..8b9e5dc0 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,2 @@ + +If you are not redirected automatically, [click here](01-about/000_intro.html). diff --git a/flake.nix b/flake.nix index 7529365b..e311aa0d 100644 --- a/flake.nix +++ b/flake.nix @@ -76,7 +76,11 @@ formatter = pkgs.alejandra; packages = { - serena = pythonSet.mkVirtualEnv "serena" workspace.deps.default; + serena-env = pythonSet.mkVirtualEnv "serena-env" workspace.deps.default; + serena = pkgs.runCommand "serena" {} '' + mkdir -p $out/bin + ln -s ${packages.serena-env}/bin/serena $out/bin/serena + ''; default = packages.serena; }; diff --git a/pyproject.toml b/pyproject.toml index aac2131a..196f99ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ classifiers = [ dependencies = [ "requests>=2.32.3,<3", "pyright>=1.1.396,<2", + "fortls>=3.2.2", "overrides>=7.7.0,<8", "python-dotenv>=1.0.0, <2", "mcp==1.12.3", @@ -64,8 +65,15 @@ dev = [ "types-pyyaml>=6.0.12.20241230", "syrupy>=4.9.1", "types-requests>=2.32.4.20241230", + # docs + "sphinx>=7,<8", + "sphinx_rtd_theme>=0.5.1", + "sphinx-toolbox==3.7.0", + "jupyter-book>=1,<2", + "nbsphinx", + "pyinstrument", ] -agno = ["agno>=1.2.6", "sqlalchemy>=2.0.40"] +agno = ["agno>=2.2.1", "sqlalchemy>=2.0.40"] google = ["google-genai>=1.8.0"] [project.urls] @@ -113,15 +121,23 @@ PYDEVD_DISABLE_FILE_VALIDATION = "1" # Uses PYTEST_MARKERS env var for default markers # For custom markers, one can either adjust the env var or just use -m option in the command line, # as the second -m option will override the first one. -test = "pytest test -vv -m \"${PYTEST_MARKERS:-not java and not rust and not erlang}\"" +test = "pytest test -vv" _black_check = "black --check src scripts test" _ruff_check = "ruff check src scripts test" _black_format = "black src scripts test" _ruff_format = "ruff check --fix src scripts test" lint = ["_black_check", "_ruff_check"] format = ["_ruff_format", "_black_format"] -_mypy = "mypy src/serena" +_mypy = "mypy src/serena src/solidlsp" type-check = ["_mypy"] +# docs +_autogen_rst = "python docs/autogen_rst.py" +_sphinx_build = "sphinx-build -b html docs docs/_build -W --keep-going" +_jb_generate_toc = "python docs/create_toc.py" +_jb_generate_config = "jupyter-book config sphinx docs/" +doc-clean = "rm -rf docs/_build docs/03_api" +doc-generate-files = ["_autogen_rst", "_jb_generate_toc", "_jb_generate_config"] +doc-build = ["doc-clean", "doc-generate-files", "_sphinx_build"] [tool.ruff] target-version = "py311" @@ -210,7 +226,6 @@ ignore = [ "COM812", "RET503", "RET504", - "UP038", "F403", "F405", "C401", @@ -238,6 +253,7 @@ ignore = [ "C400", # wants to unnecessarily force use of list comprehension "UP037", # can incorrectly (!) convert quoted type to unquoted type, causing an error "UP045", # imposes T | None instead of Optional[T] + "UP031", # forbids % operator to format strings ] unfixable = ["F841", "F601", "F602", "B018"] extend-fixable = ["F401", "B905", "W291"] @@ -275,10 +291,15 @@ markers = [ "nix: language server running for Nix", "dart: language server running for Dart", "erlang: language server running for Erlang", + "scala: language server running for Scala", "al: language server running for AL (Microsoft Dynamics 365 Business Central)", "fsharp: language server running for F#", "rego: language server running for Rego", "markdown: language server running for Markdown", + "julia: Julia language server tests", + "fortran: language server running for Fortran", + "haskell: Haskell language server tests", + "yaml: language server running for YAML", ] [tool.codespell] diff --git a/repo_dir_sync.py b/repo_dir_sync.py new file mode 100644 index 00000000..874a362f --- /dev/null +++ b/repo_dir_sync.py @@ -0,0 +1,363 @@ +# -*- coding: utf-8 -*- +import glob +import os +import shutil +from subprocess import Popen, PIPE +import re +import sys +from typing import List, Optional, Sequence +import platform + + +def popen(cmd): + shell = platform.system() != "Windows" + p = Popen(cmd, shell=shell, stdin=PIPE, stdout=PIPE) + return p + + +def call(cmd): + p = popen(cmd) + return p.stdout.read().decode("utf-8") + + +def execute(cmd, exceptionOnError=True): + """ + :param cmd: the command to execute + :param exceptionOnError: if True, raise on exception on error (return code not 0); if False return + whether the call was successful + :return: True if the call was successful, False otherwise (if exceptionOnError==False) + """ + p = popen(cmd) + p.wait() + success = p.returncode == 0 + if exceptionOnError: + if not success: + raise Exception("Command failed: %s" % cmd) + else: + return success + + +def gitLog(path, arg): + oldPath = os.getcwd() + os.chdir(path) + lg = call("git log --no-merges " + arg) + os.chdir(oldPath) + return lg + + +def gitCommit(msg): + with open(COMMIT_MSG_FILENAME, "wb") as f: + f.write(msg.encode("utf-8")) + gitCommitWithMessageFromFile(COMMIT_MSG_FILENAME) + + +def gitCommitWithMessageFromFile(commitMsgFilename): + if not os.path.exists(commitMsgFilename): + raise FileNotFoundError(f"{commitMsgFilename} not found in {os.path.abspath(os.getcwd())}") + os.system(f"git commit --file={commitMsgFilename}") + os.unlink(commitMsgFilename) + + +COMMIT_MSG_FILENAME = "commitmsg.txt" + + +class OtherRepo: + SYNC_COMMIT_ID_FILE_LIB_REPO = ".syncCommitId.remote" + SYNC_COMMIT_ID_FILE_THIS_REPO = ".syncCommitId.this" + SYNC_COMMIT_MESSAGE = f"Updated %s sync commit identifiers" + SYNC_BACKUP_DIR = ".syncBackup" + + def __init__(self, name, branch, pathToLib): + self.pathToLibInThisRepo = os.path.abspath(pathToLib) + if not os.path.exists(self.pathToLibInThisRepo): + raise ValueError(f"Repository directory '{self.pathToLibInThisRepo}' does not exist") + self.name = name + self.branch = branch + self.libRepo: Optional[LibRepo] = None + + def isSyncEstablished(self): + return os.path.exists(os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_LIB_REPO)) + + def lastSyncIdThisRepo(self): + with open(os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_THIS_REPO), "r") as f: + commitId = f.read().strip() + return commitId + + def lastSyncIdLibRepo(self): + with open(os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_LIB_REPO), "r") as f: + commitId = f.read().strip() + return commitId + + def gitLogThisRepoSinceLastSync(self): + lg = gitLog(self.pathToLibInThisRepo, '--name-only HEAD "^%s" .' % self.lastSyncIdThisRepo()) + lg = re.sub(r'commit [0-9a-z]{8,40}\n.*\n.*\n\s*\n.*\n\s*(\n.*\.syncCommitId\.(this|remote))+', r"", lg, flags=re.MULTILINE) # remove commits with sync commit id update + indent = " " + lg = indent + lg.replace("\n", "\n" + indent) + return lg + + def gitLogLibRepoSinceLastSync(self, libRepo: "LibRepo"): + syncIdFile = os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_LIB_REPO) + if not os.path.exists(syncIdFile): + return "" + with open(syncIdFile, "r") as f: + syncId = f.read().strip() + lg = gitLog(libRepo.libPath, '--name-only HEAD "^%s" .' % syncId) + lg = re.sub(r"Sync (\w+)\n\s*\n", r"Sync\n\n", lg, flags=re.MULTILINE) + indent = " " + lg = indent + lg.replace("\n", "\n" + indent) + return "\n\n" + lg + + def _userInputYesNo(self, question) -> bool: + result = None + while result not in ("y", "n"): + result = input(question + " [y|n]: ").strip() + return result == "y" + + def pull(self, libRepo: "LibRepo"): + """ + Pulls in changes from this repository into the lib repo + """ + # switch to branch in lib repo + os.chdir(libRepo.rootPath) + execute("git checkout %s" % self.branch) + + # check if the branch contains the commit that is referenced as the remote commit + remoteCommitId = self.lastSyncIdLibRepo() + remoteCommitExists = execute("git rev-list HEAD..%s" % remoteCommitId, exceptionOnError=False) + if not remoteCommitExists: + if not self._userInputYesNo(f"\nWARNING: The referenced remote commit {remoteCommitId} does not exist " + f"in your {self.libRepo.name} branch '{self.branch}'!\nSomeone else may have " + f"pulled/pushed in the meantime.\nIt is recommended that you do not continue. " + f"Continue?"): + return + + # check if this branch is clean + lgLib = self.gitLogLibRepoSinceLastSync(libRepo).strip() + if lgLib != "": + print(f"The following changes have been added to this branch in the library:\n\n{lgLib}\n\n") + print(f"ERROR: You must push these changes before you can pull or reset this branch to {remoteCommitId}") + sys.exit(1) + + # get log with relevant commits in this repo that are to be pulled + lg = self.gitLogThisRepoSinceLastSync() + + os.chdir(libRepo.rootPath) + + # create commit message file + commitMsg = f"Sync {self.name}\n\n" + lg + with open(COMMIT_MSG_FILENAME, "w") as f: + f.write(commitMsg) + + # ask whether to commit these changes + print("Relevant commits:\n\n" + lg + "\n\n") + if not self._userInputYesNo(f"The above changes will be pulled from {self.name}.\n" + f"You may change the commit message by editing {os.path.abspath(COMMIT_MSG_FILENAME)}.\n" + "Continue?"): + os.unlink(COMMIT_MSG_FILENAME) + return + + # prepare restoration of ignored files + self.prepare_restoration_of_ignored_files(libRepo.rootPath) + + # remove library tree in lib repo + shutil.rmtree(self.libRepo.libDirectory) + + # copy tree from this repo to lib repo (but drop the sync commit id files) + shutil.copytree(self.pathToLibInThisRepo, self.libRepo.libDirectory) + for fn in (self.SYNC_COMMIT_ID_FILE_LIB_REPO, self.SYNC_COMMIT_ID_FILE_THIS_REPO): + p = os.path.join(self.libRepo.libDirectory, fn) + if os.path.exists(p): + os.unlink(p) + + # restore ignored directories/files + self.restore_ignored_files(libRepo.rootPath) + + # make commit in lib repo + os.system("git add %s" % self.libRepo.libDirectory) + gitCommitWithMessageFromFile(COMMIT_MSG_FILENAME) + newSyncCommitIdLibRepo = call("git rev-parse HEAD").strip() + + # update commit ids in this repo + os.chdir(self.pathToLibInThisRepo) + newSyncCommitIdThisRepo = call("git rev-parse HEAD").strip() + with open(self.SYNC_COMMIT_ID_FILE_LIB_REPO, "w") as f: + f.write(newSyncCommitIdLibRepo) + with open(self.SYNC_COMMIT_ID_FILE_THIS_REPO, "w") as f: + f.write(newSyncCommitIdThisRepo) + execute('git add %s %s' % (self.SYNC_COMMIT_ID_FILE_LIB_REPO, self.SYNC_COMMIT_ID_FILE_THIS_REPO)) + execute(f'git commit -m "{self.SYNC_COMMIT_MESSAGE % self.libRepo.name} (pull)"') + + print(f"\n\nIf everything was successful, you should now push your changes to branch " + f"'{self.branch}'\nand get your branch merged into develop (issuing a pull request where appropriate)") + + def push(self, libRepo: "LibRepo"): + """ + Pushes changes from the lib repo to this repo + """ + os.chdir(libRepo.rootPath) + + # switch to the source repo branch + execute(f"git checkout {self.branch}") + + if self.isSyncEstablished(): + + # check if there are any commits that have not yet been pulled + unpulledCommits = self.gitLogThisRepoSinceLastSync().strip() + if unpulledCommits != "": + print(f"\n{unpulledCommits}\n\n") + if not self._userInputYesNo(f"WARNING: The above changes in repository '{self.name}' have not" + f" yet been pulled.\nYou might want to pull them.\n" + f"If you continue with the push, they will be lost. Continue?"): + return + + # get change log in lib repo since last sync + libLogSinceLastSync = self.gitLogLibRepoSinceLastSync(libRepo) + + print("Relevant commits:\n\n" + libLogSinceLastSync + "\n\n") + if not self._userInputYesNo("The above changes will be pushed. Continue?"): + return + print() + else: + libLogSinceLastSync = "" + + # prepare restoration of ignored files in target repo + base_dir_this_repo = os.path.join(self.pathToLibInThisRepo, "..") + self.prepare_restoration_of_ignored_files(base_dir_this_repo) + + # remove the target repo tree and update it with the tree from the source repo + shutil.rmtree(self.pathToLibInThisRepo) + shutil.copytree(libRepo.libPath, self.pathToLibInThisRepo) + + # get the commit id of the source repo we just copied + commitId = call("git rev-parse HEAD").strip() + + # restore ignored directories and files + self.restore_ignored_files(base_dir_this_repo) + + # go to the target repo + os.chdir(self.pathToLibInThisRepo) + + # commit new version in this repo + execute("git add .") + with open(self.SYNC_COMMIT_ID_FILE_LIB_REPO, "w") as f: + f.write(commitId) + execute("git add %s" % self.SYNC_COMMIT_ID_FILE_LIB_REPO) + gitCommit(f"{self.libRepo.name} {commitId}" + libLogSinceLastSync) + commitId = call("git rev-parse HEAD").strip() + + # update information on the commit id we just added + with open(self.SYNC_COMMIT_ID_FILE_THIS_REPO, "w") as f: + f.write(commitId) + execute("git add %s" % self.SYNC_COMMIT_ID_FILE_THIS_REPO) + execute(f'git commit -m "{self.SYNC_COMMIT_MESSAGE % self.libRepo.name} (push)"') + + os.chdir(libRepo.rootPath) + + print(f"\n\nIf everything was successful, you should now update the remote branch:\ngit push") + + def prepare_restoration_of_ignored_files(self, base_dir: str): + """ + :param base_dir: the directory containing the lib directory, to which ignored paths are relative + """ + cwd = os.getcwd() + os.chdir(base_dir) + + # ensure backup dir exists and is empty + if os.path.exists(self.SYNC_BACKUP_DIR): + shutil.rmtree(self.SYNC_BACKUP_DIR) + os.mkdir(self.SYNC_BACKUP_DIR) + + # backup ignored, unversioned directories + for d in self.libRepo.fullyIgnoredUnversionedDirectories: + if os.path.exists(d): + shutil.copytree(d, os.path.join(self.SYNC_BACKUP_DIR, d)) + + os.chdir(cwd) + + def restore_ignored_files(self, base_dir: str): + """ + :param base_dir: the directory containing the lib directory, to which ignored paths are relative + """ + cwd = os.getcwd() + os.chdir(base_dir) + + # remove fully ignored directories that were overwritten by the sync + for d in self.libRepo.fullyIgnoredVersionedDirectories + self.libRepo.fullyIgnoredUnversionedDirectories: + if os.path.exists(d): + print("Removing overwritten content: %s" % d) + shutil.rmtree(d) + + # restore directories and files that can be restored via git + for d in self.libRepo.ignoredDirectories + self.libRepo.fullyIgnoredVersionedDirectories: + restoration_cmd = "git checkout %s" % d + print("Restoring: %s" % restoration_cmd) + os.system(restoration_cmd) + for pattern in self.libRepo.ignoredFileGlobPatterns: + for path in glob.glob(pattern, recursive=True): + print("Restoring via git: %s" % path) + os.system("git checkout %s" % path) + + # restore directories that were backed up + for d in self.libRepo.fullyIgnoredUnversionedDirectories: + if os.path.exists(os.path.join(self.SYNC_BACKUP_DIR, d)): + print("Restoring from backup: %s" % d) + shutil.copytree(os.path.join(self.SYNC_BACKUP_DIR, d), d) + + # remove backup dir + shutil.rmtree(self.SYNC_BACKUP_DIR) + + os.chdir(cwd) + + +class LibRepo: + def __init__(self, name: str, libDirectory: str, + ignoredDirectories: Sequence[str] = (), + fullyIgnoredVersionedDirectories: Sequence[str] = (), + fullyIgnoredUnversionedDirectories: Sequence[str] = (), + ignoredFileGlobPatterns: Sequence[str] = () + ): + """ + :param name: name of the library being synced + :param libDirectory: relative path to the library directory within this repo + :param ignoredDirectories: ignored directories; existing files in ignored directories will be restored + via 'git checkout' on pull/push, but new files will be added. + This is useful for configuration-like files, where users may have local changes that should not + be overwritten, but new files should still be added. + :param fullyIgnoredVersionedDirectories: + fully ignored versioned directories will be restored to original state after push/pull via git checkout + :param fullyIgnoredUnversionedDirectories: + fully ignored unversioned directories will be backed up and restored to original state after push/pull + :param ignoredFileGlobPatterns: files matching ignored glob patterns will be restored via 'git checkout' + on pull/push + """ + self.name = name + self.rootPath = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + self.libDirectory = libDirectory + self.libPath = os.path.join(self.rootPath, self.libDirectory) + self.ignoredDirectories: List[str] = list(ignoredDirectories) + self.fullyIgnoredVersionedDirectories: List[str] = list(fullyIgnoredVersionedDirectories) + self.fullyIgnoredUnversionedDirectories: List[str] = list(fullyIgnoredUnversionedDirectories) + self.ignoredFileGlobPatterns: List[str] = list(ignoredFileGlobPatterns) + self.otherRepos: List[OtherRepo] = [] + + def add(self, repo: OtherRepo): + repo.libRepo = self + self.otherRepos.append(repo) + + def runMain(self): + repos = self.otherRepos + args = sys.argv[1:] + if len(args) != 2: + print(f"usage: sync.py <{'|'.join([repo.name for repo in repos])}> ") + else: + repo = [r for r in repos if r.name == args[0]] + if len(repo) != 1: + raise ValueError(f"Unknown repo '{args[0]}'") + repo = repo[0] + + if args[1] == "push": + repo.push(self) + elif args[1] == "pull": + repo.pull(self) + else: + raise ValueError(f"Unknown command '{args[1]}'") diff --git a/scripts/agno_agent.py b/scripts/agno_agent.py index 5c8e50a4..53473fd2 100644 --- a/scripts/agno_agent.py +++ b/scripts/agno_agent.py @@ -1,7 +1,6 @@ from agno.models.anthropic.claude import Claude from agno.models.google.gemini import Gemini -from agno.playground.playground import Playground -from agno.playground.serve import serve_playground_app +from agno.os import AgentOS from sensai.util import logging from sensai.util.helper import mark_used @@ -9,16 +8,26 @@ from serena.agno import SerenaAgnoAgentProvider mark_used(Gemini, Claude) -# initialize logging (Note: since this module is reimported by serve_playground_app and the logging configuration -# is extended by SerenaAgentProvider, we must handle this here conditionally) +# initialize logging if __name__ == "__main__": logging.configure(level=logging.INFO) # Define the model to use (see Agno documentation for supported models; these are just examples) # model = Claude(id="claude-3-7-sonnet-20250219") -model = Gemini(id="gemini-2.5-pro-exp-03-25") +model = Gemini(id="gemini-2.5-pro") -app = Playground(agents=[SerenaAgnoAgentProvider.get_agent(model)]).get_app() +# Create the Serena agent using the existing provider +serena_agent = SerenaAgnoAgentProvider.get_agent(model) + +# Create AgentOS app with the Serena agent +agent_os = AgentOS( + description="Serena coding assistant powered by AgentOS", + id="serena-agentos", + agents=[serena_agent], +) + +app = agent_os.get_app() if __name__ == "__main__": - serve_playground_app("agno_agent:app", reload=False, log_config=None) + # Start the AgentOS server + agent_os.serve(app="agno_agent:app", reload=False) diff --git a/scripts/demo_run_tools.py b/scripts/demo_run_tools.py index d52cfb00..8cbaae79 100644 --- a/scripts/demo_run_tools.py +++ b/scripts/demo_run_tools.py @@ -7,20 +7,21 @@ import json from pprint import pprint from serena.agent import SerenaAgent -from serena.config.serena_config import SerenaConfig from serena.constants import REPO_ROOT -from serena.tools import FindFileTool, FindReferencingSymbolsTool, GetSymbolsOverviewTool, SearchForPatternTool +from serena.tools import FindFileTool, FindReferencingSymbolsTool, GetSymbolsOverviewTool, JetBrainsFindSymbolTool, SearchForPatternTool if __name__ == "__main__": - agent = SerenaAgent(project=REPO_ROOT, serena_config=SerenaConfig(gui_log_window_enabled=False, web_dashboard=False)) + agent = SerenaAgent(project=REPO_ROOT) # apply a tool + find_symbol_tool = agent.get_tool(JetBrainsFindSymbolTool) find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) find_file_tool = agent.get_tool(FindFileTool) search_pattern_tool = agent.get_tool(SearchForPatternTool) overview_tool = agent.get_tool(GetSymbolsOverviewTool) result = agent.execute_task( - lambda: overview_tool.apply("src/solidlsp/ls.py"), + lambda: find_symbol_tool.apply("displayBasicStats"), ) pprint(json.loads(result)) + # input("Press Enter to continue...") diff --git a/scripts/profile_tool_call.py b/scripts/profile_tool_call.py new file mode 100644 index 00000000..e0e69981 --- /dev/null +++ b/scripts/profile_tool_call.py @@ -0,0 +1,53 @@ +import cProfile +from pathlib import Path +from typing import Literal + +from sensai.util import logging +from sensai.util.logging import LogTime +from sensai.util.profiling import profiled + +from serena.agent import SerenaAgent +from serena.config.serena_config import SerenaConfig +from serena.tools import FindSymbolTool + +log = logging.getLogger(__name__) + + +if __name__ == "__main__": + logging.configure() + + # The profiler to use: + # Use pyinstrument for hierarchical profiling output + # Use cProfile to determine which functions take the most time overall (and use snakeviz to visualize) + profiler: Literal["pyinstrument", "cprofile"] = "cprofile" + + project_path = Path(__file__).parent.parent # Serena root + + serena_config = SerenaConfig.from_config_file() + serena_config.log_level = logging.INFO + serena_config.jetbrains = False + serena_config.gui_log_window_enabled = False + serena_config.web_dashboard = False + + agent = SerenaAgent(str(project_path), serena_config=serena_config) + + # wait for language server to be ready + agent.execute_task(lambda: log.info("Language server is ready.")) + + def tool_call(): + """This is the function we want to profile.""" + # NOTE: We use apply (not apply_ex) to run the tool call directly on the main thread + with LogTime("Tool call"): + result = agent.get_tool(FindSymbolTool).apply(name_path="DQN") + log.info("Tool result:\n%s", result) + + if profiler == "pyinstrument": + + @profiled(log_to_file=True) + def profiled_tool_call(): + tool_call() + + profiled_tool_call() + + elif profiler == "cprofile": + cProfile.run("tool_call()", "tool_call.pstat") diff --git a/src/serena/agent.py b/src/serena/agent.py index d8a491dd..c02e1051 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -6,13 +6,10 @@ import multiprocessing import os import platform import sys -import threading import webbrowser from collections.abc import Callable -from concurrent.futures import Future, ThreadPoolExecutor from logging import Logger -from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, TypeVar +from typing import TYPE_CHECKING, Optional, TypeVar from sensai.util import logging from sensai.util.logging import LogTime @@ -20,15 +17,17 @@ from sensai.util.logging import LogTime from interprompt.jinja_template import JinjaTemplate from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats -from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, ToolSet, get_serena_managed_in_project_dir +from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode +from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition from serena.dashboard import SerenaDashboardAPI +from serena.ls_manager import LanguageServerManager from serena.project import Project from serena.prompt_factory import SerenaPromptFactory -from serena.tools import ActivateProjectTool, GetCurrentConfigTool, Tool, ToolMarker, ToolRegistry +from serena.task_executor import TaskExecutor +from serena.tools import ActivateProjectTool, GetCurrentConfigTool, ReplaceContentTool, Tool, ToolMarker, ToolRegistry from serena.util.inspection import iter_subclasses from serena.util.logging import MemoryLogHandler -from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language if TYPE_CHECKING: from serena.gui_log_viewer import GuiLogViewer @@ -43,40 +42,11 @@ class ProjectNotFoundError(Exception): pass -class MemoriesManager: - def __init__(self, project_root: str): - self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" - self._memory_dir.mkdir(parents=True, exist_ok=True) - - def _get_memory_file_path(self, name: str) -> Path: - # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not. - name = name.replace(".md", "") - filename = f"{name}.md" - return self._memory_dir / filename - - def load_memory(self, name: str) -> str: - memory_file_path = self._get_memory_file_path(name) - if not memory_file_path.exists(): - return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it." - with open(memory_file_path, encoding="utf-8") as f: - return f.read() - - def save_memory(self, name: str, content: str) -> str: - memory_file_path = self._get_memory_file_path(name) - with open(memory_file_path, "w", encoding="utf-8") as f: - f.write(content) - return f"Memory {name} written." - - def list_memories(self) -> list[str]: - return [f.name.replace(".md", "") for f in self._memory_dir.iterdir() if f.is_file()] - - def delete_memory(self, name: str) -> str: - memory_file_path = self._get_memory_file_path(name) - memory_file_path.unlink() - return f"Memory {name} deleted." - - class AvailableTools: + """ + Represents the set of available/exposed tools of a SerenaAgent. + """ + def __init__(self, tools: list[Tool]): """ :param tools: the list of available tools @@ -93,6 +63,94 @@ class AvailableTools: return len(self.tools) +class ToolSet: + """ + Represents a set of tools by their names. + """ + + LEGACY_TOOL_NAME_MAPPING = {"replace_regex": ReplaceContentTool.get_name_from_cls()} + """ + maps legacy tool names to their new names for backward compatibility + """ + + def __init__(self, tool_names: set[str]) -> None: + self._tool_names = tool_names + + @classmethod + def default(cls) -> "ToolSet": + """ + :return: the default tool set, which contains all tools that are enabled by default + """ + from serena.tools import ToolRegistry + + return cls(set(ToolRegistry().get_tool_names_default_enabled())) + + def apply(self, *tool_inclusion_definitions: "ToolInclusionDefinition") -> "ToolSet": + """ + Applies one or more tool inclusion definitions to this tool set, + resulting in a new tool set. + + :param tool_inclusion_definitions: the definitions to apply + :return: a new tool set with the definitions applied + """ + from serena.tools import ToolRegistry + + def get_updated_tool_name(tool_name: str) -> str: + """Retrieves the updated tool name if the provided tool name is deprecated, logging a warning.""" + if tool_name in self.LEGACY_TOOL_NAME_MAPPING: + new_tool_name = self.LEGACY_TOOL_NAME_MAPPING[tool_name] + log.warning("Tool name '%s' is deprecated, please use '%s' instead", tool_name, new_tool_name) + return new_tool_name + return tool_name + + registry = ToolRegistry() + tool_names = set(self._tool_names) + for definition in tool_inclusion_definitions: + included_tools = [] + excluded_tools = [] + for included_tool in definition.included_optional_tools: + included_tool = get_updated_tool_name(included_tool) + if not registry.is_valid_tool_name(included_tool): + raise ValueError(f"Invalid tool name '{included_tool}' provided for inclusion") + if included_tool not in tool_names: + tool_names.add(included_tool) + included_tools.append(included_tool) + for excluded_tool in definition.excluded_tools: + excluded_tool = get_updated_tool_name(excluded_tool) + if not registry.is_valid_tool_name(excluded_tool): + raise ValueError(f"Invalid tool name '{excluded_tool}' provided for exclusion") + if excluded_tool in tool_names: + tool_names.remove(excluded_tool) + excluded_tools.append(excluded_tool) + if included_tools: + log.info(f"{definition} included {len(included_tools)} tools: {', '.join(included_tools)}") + if excluded_tools: + log.info(f"{definition} excluded {len(excluded_tools)} tools: {', '.join(excluded_tools)}") + return ToolSet(tool_names) + + def without_editing_tools(self) -> "ToolSet": + """ + :return: a new tool set that excludes all tools that can edit + """ + from serena.tools import ToolRegistry + + registry = ToolRegistry() + tool_names = set(self._tool_names) + for tool_name in self._tool_names: + if registry.get_tool_class_by_name(tool_name).can_edit(): + tool_names.remove(tool_name) + return ToolSet(tool_names) + + def get_tool_names(self) -> set[str]: + """ + Returns the names of the tools that are currently included in the tool set. + """ + return self._tool_names + + def includes_name(self, tool_name: str) -> bool: + return tool_name in self._tool_names + + class SerenaAgent: def __init__( self, @@ -120,12 +178,10 @@ class SerenaAgent: # project-specific instances, which will be initialized upon project activation self._active_project: Project | None = None - self.language_server: SolidLanguageServer | None = None - self.memories_manager: MemoriesManager | None = None # adjust log level serena_log_level = self.serena_config.log_level - if Logger.root.level > serena_log_level: + if Logger.root.level != serena_log_level: log.info(f"Changing the root logger level to {serena_log_level}") Logger.root.setLevel(serena_log_level) @@ -162,25 +218,9 @@ class SerenaAgent: if self._gui_log_viewer is not None: self._gui_log_viewer.set_tool_names(tool_names) - self._tool_usage_stats: ToolUsageStats | None = None - if self.serena_config.record_tool_usage_stats: - token_count_estimator = RegisteredTokenCountEstimator[self.serena_config.token_count_estimator] - log.info(f"Tool usage statistics recording is enabled with token count estimator: {token_count_estimator.name}.") - self._tool_usage_stats = ToolUsageStats(token_count_estimator) - - # start the dashboard (web frontend), registering its log handler - if self.serena_config.web_dashboard: - self._dashboard_thread, port = SerenaDashboardAPI( - get_memory_log_handler(), tool_names, agent=self, tool_usage_stats=self._tool_usage_stats - ).run_in_thread() - dashboard_url = f"http://127.0.0.1:{port}/dashboard/index.html" - log.info("Serena web dashboard started at %s", dashboard_url) - if self.serena_config.web_dashboard_open_on_launch: - # open the dashboard URL in the default web browser (using a separate process to control - # output redirection) - process = multiprocessing.Process(target=self._open_dashboard, args=(dashboard_url,)) - process.start() - process.join(timeout=1) + token_count_estimator = RegisteredTokenCountEstimator[self.serena_config.token_count_estimator] + log.info(f"Will record tool usage statistics with token count estimator: {token_count_estimator.name}.") + self._tool_usage_stats = ToolUsageStats(token_count_estimator) # log fundamental information log.info(f"Starting Serena server (version={serena_version()}, process id={os.getpid()}, parent process id={os.getppid()})") @@ -193,8 +233,8 @@ class SerenaAgent: # determine the base toolset defining the set of exposed tools (which e.g. the MCP shall see), # limited by the Serena config, the context (which is fixed for the session) and JetBrains mode tool_inclusion_definitions: list[ToolInclusionDefinition] = [self.serena_config, self._context] - if self._context.name == RegisteredContext.IDE_ASSISTANT.value: - tool_inclusion_definitions.extend(self._ide_assistant_context_tool_inclusion_definitions(project)) + if self._context.single_project: + tool_inclusion_definitions.extend(self._single_project_context_tool_inclusion_definitions(project)) if self.serena_config.jetbrains: tool_inclusion_definitions.append(SerenaAgentMode.from_name_internal("jetbrains")) @@ -203,10 +243,8 @@ class SerenaAgent: log.info(f"Number of exposed tools: {len(self._exposed_tools)}") # create executor for starting the language server and running tools in another thread - # This executor is used to achieve linear task execution, so it is important to use a single-threaded executor. - self._task_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="SerenaAgentExecutor") - self._task_executor_lock = threading.Lock() - self._task_executor_task_index = 1 + # This executor is used to achieve linear task execution + self._task_executor = TaskExecutor("SerenaAgentTaskExecutor") # Initialize the prompt factory self.prompt_factory = SerenaPromptFactory() @@ -227,6 +265,57 @@ class SerenaAgent: except Exception as e: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) + # start the dashboard (web frontend), registering its log handler + # should be the last thing to happen in the initialization since the dashboard + # may access various parts of the agent + if self.serena_config.web_dashboard: + self._dashboard_thread, port = SerenaDashboardAPI( + get_memory_log_handler(), tool_names, agent=self, tool_usage_stats=self._tool_usage_stats + ).run_in_thread() + dashboard_url = f"http://127.0.0.1:{port}/dashboard/index.html" + log.info("Serena web dashboard started at %s", dashboard_url) + if self.serena_config.web_dashboard_open_on_launch: + # open the dashboard URL in the default web browser (using a separate process to control + # output redirection) + process = multiprocessing.Process(target=self._open_dashboard, args=(dashboard_url,)) + process.start() + process.join(timeout=1) + # inform the GUI window (if any) + if self._gui_log_viewer is not None: + self._gui_log_viewer.set_dashboard_url(dashboard_url) + + def get_current_tasks(self) -> list[TaskExecutor.TaskInfo]: + """ + Gets the list of tasks currently running or queued for execution. + The function returns a list of thread-safe TaskInfo objects (specifically created for the caller). + + :return: the list of tasks in the execution order (running task first) + """ + return self._task_executor.get_current_tasks() + + def get_last_executed_task(self) -> TaskExecutor.TaskInfo | None: + """ + Gets the last executed task. + + :return: the last executed task info or None if no task has been executed yet + """ + return self._task_executor.get_last_executed_task() + + def get_language_server_manager(self) -> LanguageServerManager | None: + if self._active_project is not None: + return self._active_project.language_server_manager + return None + + def get_language_server_manager_or_raise(self) -> LanguageServerManager: + language_server_manager = self.get_language_server_manager() + if language_server_manager is None: + raise Exception( + "The language server manager is not initialized, indicating a problem during project activation. " + "Inform the user, telling them to inspect Serena's logs in order to determine the issue. " + "IMPORTANT: Wait for further instructions before you continue!" + ) + return language_server_manager + def get_context(self) -> SerenaAgentContext: return self._context @@ -243,7 +332,7 @@ class SerenaAgent: os.environ["COMSPEC"] = "" # force use of default shell log.info("Adjusting COMSPEC environment variable to use the default shell instead of '%s'", comspec) - def _ide_assistant_context_tool_inclusion_definitions(self, project_root_or_name: str | None) -> list[ToolInclusionDefinition]: + def _single_project_context_tool_inclusion_definitions(self, project_root_or_name: str | None) -> list[ToolInclusionDefinition]: """ In the IDE assistant context, the agent is assumed to work on a single project, and we thus want to apply that project's tool exclusions/inclusions from the get-go, limiting the set @@ -261,6 +350,9 @@ class SerenaAgent: # and provide responses to the client immediately. project = self.load_project_from_path_or_name(project_root_or_name, autogenerate=False) if project is not None: + log.info( + "Applying tool inclusion/exclusion definitions for single-project context based on project '%s'", project.project_name + ) tool_inclusion_definitions.append( ToolInclusionDefinition( excluded_tools=[ActivateProjectTool.get_name_from_cls(), GetCurrentConfigTool.get_name_from_cls()] @@ -269,18 +361,15 @@ class SerenaAgent: tool_inclusion_definitions.append(project.project_config) return tool_inclusion_definitions - def record_tool_usage_if_enabled(self, input_kwargs: dict, tool_result: str | dict, tool: Tool) -> None: + def record_tool_usage(self, input_kwargs: dict, tool_result: str | dict, tool: Tool) -> None: """ Record the usage of a tool with the given input and output strings if tool usage statistics recording is enabled. """ tool_name = tool.get_name() - if self._tool_usage_stats is not None: - input_str = str(input_kwargs) - output_str = str(tool_result) - log.debug(f"Recording tool usage for tool '{tool_name}'") - self._tool_usage_stats.record_tool_usage(tool_name, input_str, output_str) - else: - log.debug(f"Tool usage statistics recording is disabled, not recording usage of '{tool_name}'.") + input_str = str(input_kwargs) + output_str = str(tool_result) + log.debug(f"Recording tool usage for tool '{tool_name}'") + self._tool_usage_stats.record_tool_usage(tool_name, input_str, output_str) @staticmethod def _open_dashboard(url: str) -> None: @@ -359,6 +448,11 @@ class SerenaAgent: available_tools=self._exposed_tools.tool_names, available_markers=available_markers, ) + + # If a project is active at startup, append its activation message + if self._active_project is not None: + system_prompt += "\n\n" + self._active_project.get_activation_message() + log.info("System prompt:\n%s", system_prompt) return system_prompt @@ -382,36 +476,33 @@ class SerenaAgent: log.info(f"Active tools ({len(self._active_tools)}): {', '.join(self.get_active_tool_names())}") - def issue_task(self, task: Callable[[], Any], name: str | None = None) -> Future: + def issue_task( + self, task: Callable[[], T], name: str | None = None, logged: bool = True, timeout: float | None = None + ) -> TaskExecutor.Task[T]: """ Issue a task to the executor for asynchronous execution. It is ensured that tasks are executed in the order they are issued, one after another. :param task: the task to execute :param name: the name of the task for logging purposes; if None, use the task function's name - :return: a Future object representing the execution of the task + :param logged: whether to log management of the task; if False, only errors will be logged + :param timeout: the maximum time to wait for task completion in seconds, or None to wait indefinitely + :return: the task object, through which the task's future result can be accessed """ - with self._task_executor_lock: - task_name = f"Task-{self._task_executor_task_index}[{name or task.__name__}]" - self._task_executor_task_index += 1 + return self._task_executor.issue_task(task, name=name, logged=logged, timeout=timeout) - def task_execution_wrapper() -> Any: - with LogTime(task_name, logger=log): - return task() - - log.info(f"Scheduling {task_name}") - return self._task_executor.submit(task_execution_wrapper) - - def execute_task(self, task: Callable[[], T]) -> T: + def execute_task(self, task: Callable[[], T], name: str | None = None, logged: bool = True, timeout: float | None = None) -> T: """ Executes the given task synchronously via the agent's task executor. This is useful for tasks that need to be executed immediately and whose results are needed right away. :param task: the task to execute + :param name: the name of the task for logging purposes; if None, use the task function's name + :param logged: whether to log management of the task; if False, only errors will be logged + :param timeout: the maximum time to wait for task completion in seconds, or None to wait indefinitely :return: the result of the task execution """ - future = self.issue_task(task) - return future.result() + return self._task_executor.execute_task(task, name=name, logged=logged, timeout=timeout) def is_using_language_server(self) -> bool: """ @@ -424,18 +515,14 @@ class SerenaAgent: self._active_project = project self._update_active_tools() - # initialize project-specific instances which do not depend on the language server - self.memories_manager = MemoriesManager(project.project_root) - - def init_language_server() -> None: + def init_language_server_manager() -> None: # start the language server with LogTime("Language server initialization", logger=log): - self.reset_language_server() - assert self.language_server is not None + self.reset_language_server_manager() # initialize the language server in the background (if in language server mode) if self.is_using_language_server(): - self.issue_task(init_language_server) + self.issue_task(init_language_server_manager) if self._project_activation_callback is not None: self._project_activation_callback() @@ -542,12 +629,9 @@ class SerenaAgent: return result_str - def is_language_server_running(self) -> bool: - return self.language_server is not None and self.language_server.is_running() - - def reset_language_server(self) -> None: + def reset_language_server_manager(self) -> None: """ - Starts/resets the language server for the current project + Starts/resets the language server manager for the current project """ tool_timeout = self.serena_config.tool_timeout if tool_timeout is None or tool_timeout < 0: @@ -557,27 +641,32 @@ class SerenaAgent: raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds") ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout - # stop the language server if it is running - if self.is_language_server_running(): - assert self.language_server is not None - log.info(f"Stopping the current language server at {self.language_server.repository_root_path} ...") - self.language_server.stop() - self.language_server = None - - # instantiate and start the language server - assert self._active_project is not None - self.language_server = self._active_project.create_language_server( + # instantiate and start the necessary language servers + self.get_active_project_or_raise().create_language_server_manager( log_level=self.serena_config.log_level, ls_timeout=ls_timeout, trace_lsp_communication=self.serena_config.trace_lsp_communication, ls_specific_settings=self.serena_config.ls_specific_settings, ) - log.info(f"Starting the language server for {self._active_project.project_name}") - self.language_server.start() - if not self.language_server.is_running(): - raise RuntimeError( - f"Failed to start the language server for {self._active_project.project_name} at {self._active_project.project_root}" - ) + + def add_language(self, language: Language) -> None: + """ + Adds a new language to the active project, spawning the respective language server and updating the project configuration. + The addition is scheduled via the agent's task executor and executed synchronously, i.e. the method returns + when the addition is complete. + + :param language: the language to add + """ + self.execute_task(lambda: self.get_active_project_or_raise().add_language(language), name=f"AddLanguage:{language.value}") + + def remove_language(self, language: Language) -> None: + """ + Removes a language from the active project, shutting down the respective language server and updating the project configuration. + The removal is scheduled via the agent's task executor and executed asynchronously. + + :param language: the language to remove + """ + self.issue_task(lambda: self.get_active_project_or_raise().remove_language(language), name=f"RemoveLanguage:{language.value}") def get_tool(self, tool_class: type[TTool]) -> TTool: return self._all_tools[tool_class] # type: ignore @@ -586,20 +675,22 @@ class SerenaAgent: ToolRegistry().print_tool_overview(self._active_tools.values()) def __del__(self) -> None: + self.shutdown() + + def shutdown(self, timeout: float = 2.0) -> None: """ - Destructor to clean up the language server instance and GUI logger + Shuts down the agent, freeing resources and stopping background tasks. """ if not hasattr(self, "_is_initialized"): return log.info("SerenaAgent is shutting down ...") - if self.is_language_server_running(): - log.info("Stopping the language server ...") - assert self.language_server is not None - self.language_server.save_cache() - self.language_server.stop() + if self._active_project is not None: + self._active_project.shutdown(timeout=timeout) + self._active_project = None if self._gui_log_viewer: log.info("Stopping the GUI log window ...") self._gui_log_viewer.stop() + self._gui_log_viewer = None def get_tool_by_name(self, tool_name: str) -> Tool: tool_class = ToolRegistry().get_tool_class_by_name(tool_name) diff --git a/src/serena/agno.py b/src/serena/agno.py index 07ab7371..253c4a49 100644 --- a/src/serena/agno.py +++ b/src/serena/agno.py @@ -6,9 +6,9 @@ from pathlib import Path from typing import Any from agno.agent import Agent -from agno.memory import AgentMemory +from agno.db.sqlite import SqliteDb +from agno.memory import MemoryManager from agno.models.base import Model -from agno.storage.sqlite import SqliteStorage from agno.tools.function import Function from agno.tools.toolkit import Toolkit from dotenv import load_dotenv @@ -111,10 +111,9 @@ class SerenaAgnoAgentProvider: raise # Even though we don't want to keep history between sessions, - # for agno-ui to work as a conversation, we use a persistent storage on disk. - # This storage should be deleted between sessions. + # for agno-ui to work as a conversation, we use a persistent database on disk. + # This database should be deleted between sessions. # Note that this might collide with custom options for the agent, like adding vector-search based tools. - # See here for an explanation: https://www.reddit.com/r/agno/comments/1jk6qea/regarding_the_built_in_memory/ sql_db_path = (Path("temp") / "agno_agent_storage.db").absolute() sql_db_path.parent.mkdir(exist_ok=True) # delete the db file if it exists @@ -125,19 +124,18 @@ class SerenaAgnoAgentProvider: agno_agent = Agent( name="Serena", model=model, - # See explanation above on why storage is needed - storage=SqliteStorage(table_name="serena_agent_sessions", db_file=str(sql_db_path)), + # See explanation above on why database is needed + db=SqliteDb(db_file=str(sql_db_path)), description="A fully-featured coding assistant", tools=[SerenaAgnoToolkit(serena_agent)], - # The tool calls will be shown in the UI anyway since whether to show them is configurable per tool + # Tool calls will be shown in the UI since that's configurable per tool # To see detailed logs, you should use the serena logger (configure it in the project file path) - show_tool_calls=False, markdown=True, system_message=serena_agent.create_system_prompt(), telemetry=False, - memory=AgentMemory(), - add_history_to_messages=True, - num_history_responses=100, # you might want to adjust this (expense vs. history awareness) + memory_manager=MemoryManager(), + add_history_to_context=True, + num_history_runs=100, # you might want to adjust this (expense vs. history awareness) ) cls._agent = agno_agent log.info(f"Agent instantiated: {agno_agent}") diff --git a/src/serena/analytics.py b/src/serena/analytics.py index c773fc08..add6fd09 100644 --- a/src/serena/analytics.py +++ b/src/serena/analytics.py @@ -69,12 +69,26 @@ class AnthropicTokenCount(TokenCountEstimator): return self._send_count_tokens_request(text).input_tokens +class CharCountEstimator(TokenCountEstimator): + """ + A naive character count estimator that estimates tokens based on character count. + """ + + def __init__(self, avg_chars_per_token: int = 4): + self._avg_chars_per_token = avg_chars_per_token + + def estimate_token_count(self, text: str) -> int: + # Assuming an average of 4 characters per token + return len(text) // self._avg_chars_per_token + + _registered_token_estimator_instances_cache: dict[RegisteredTokenCountEstimator, TokenCountEstimator] = {} class RegisteredTokenCountEstimator(Enum): TIKTOKEN_GPT4O = "TIKTOKEN_GPT4O" ANTHROPIC_CLAUDE_SONNET_4 = "ANTHROPIC_CLAUDE_SONNET_4" + CHAR_COUNT = "CHAR_COUNT" @classmethod def get_valid_names(cls) -> list[str]: @@ -89,8 +103,10 @@ class RegisteredTokenCountEstimator(Enum): return TiktokenCountEstimator(model_name="gpt-4o") case RegisteredTokenCountEstimator.ANTHROPIC_CLAUDE_SONNET_4: return AnthropicTokenCount(model_name="claude-sonnet-4-20250514") + case RegisteredTokenCountEstimator.CHAR_COUNT: + return CharCountEstimator(avg_chars_per_token=4) case _: - raise ValueError(f"Unknown token count estimator: {self.value}") + raise ValueError(f"Unknown token count estimator: {self}") def load_estimator(self) -> TokenCountEstimator: estimator_instance = _registered_token_estimator_instances_cache.get(self) diff --git a/src/serena/cli.py b/src/serena/cli.py index 0db76046..e7cca2be 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -1,3 +1,4 @@ +import collections import glob import json import os @@ -11,6 +12,7 @@ from typing import Any, Literal import click from sensai.util import logging from sensai.util.logging import FileLoggerContext, datetime_tag +from sensai.util.string import dict_string from tqdm import tqdm from serena.agent import SerenaAgent @@ -19,14 +21,10 @@ from serena.config.serena_config import ProjectConfig, SerenaConfig, SerenaPaths from serena.constants import ( DEFAULT_CONTEXT, DEFAULT_MODES, - PROMPT_TEMPLATES_DIR_IN_USER_HOME, PROMPT_TEMPLATES_DIR_INTERNAL, SERENA_LOG_FORMAT, - SERENA_MANAGED_DIR_IN_HOME, SERENAS_OWN_CONTEXT_YAMLS_DIR, SERENAS_OWN_MODE_YAMLS_DIR, - USER_CONTEXT_YAMLS_DIR, - USER_MODE_YAMLS_DIR, ) from serena.mcp import SerenaMCPFactory, SerenaMCPFactorySingleProcess from serena.project import Project @@ -268,7 +266,7 @@ class ModeCommands(AutoRegisteringGroup): if not (name or from_internal): raise click.UsageError("Provide at least one of --name or --from-internal.") mode_name = name or from_internal - dest = os.path.join(USER_MODE_YAMLS_DIR, f"{mode_name}.yml") + dest = os.path.join(SerenaPaths().user_modes_dir, f"{mode_name}.yml") src = ( os.path.join(SERENAS_OWN_MODE_YAMLS_DIR, f"{from_internal}.yml") if from_internal @@ -287,7 +285,7 @@ class ModeCommands(AutoRegisteringGroup): @click.command("edit", help="Edit a custom mode YAML file.") @click.argument("mode_name") def edit(mode_name: str) -> None: - path = os.path.join(USER_MODE_YAMLS_DIR, f"{mode_name}.yml") + path = os.path.join(SerenaPaths().user_modes_dir, f"{mode_name}.yml") if not os.path.exists(path): if mode_name in SerenaAgentMode.list_registered_mode_names(include_user_modes=False): click.echo( @@ -303,7 +301,7 @@ class ModeCommands(AutoRegisteringGroup): @click.command("delete", help="Delete a custom mode file.") @click.argument("mode_name") def delete(mode_name: str) -> None: - path = os.path.join(USER_MODE_YAMLS_DIR, f"{mode_name}.yml") + path = os.path.join(SerenaPaths().user_modes_dir, f"{mode_name}.yml") if not os.path.exists(path): click.echo(f"Custom mode '{mode_name}' not found.") return @@ -345,7 +343,7 @@ class ContextCommands(AutoRegisteringGroup): if not (name or from_internal): raise click.UsageError("Provide at least one of --name or --from-internal.") ctx_name = name or from_internal - dest = os.path.join(USER_CONTEXT_YAMLS_DIR, f"{ctx_name}.yml") + dest = os.path.join(SerenaPaths().user_contexts_dir, f"{ctx_name}.yml") src = ( os.path.join(SERENAS_OWN_CONTEXT_YAMLS_DIR, f"{from_internal}.yml") if from_internal @@ -364,7 +362,7 @@ class ContextCommands(AutoRegisteringGroup): @click.command("edit", help="Edit a custom context YAML file.") @click.argument("context_name") def edit(context_name: str) -> None: - path = os.path.join(USER_CONTEXT_YAMLS_DIR, f"{context_name}.yml") + path = os.path.join(SerenaPaths().user_contexts_dir, f"{context_name}.yml") if not os.path.exists(path): if context_name in SerenaAgentContext.list_registered_context_names(include_user_contexts=False): click.echo( @@ -380,7 +378,7 @@ class ContextCommands(AutoRegisteringGroup): @click.command("delete", help="Delete a custom context file.") @click.argument("context_name") def delete(context_name: str) -> None: - path = os.path.join(USER_CONTEXT_YAMLS_DIR, f"{context_name}.yml") + path = os.path.join(SerenaPaths().user_contexts_dir, f"{context_name}.yml") if not os.path.exists(path): click.echo(f"Custom context '{context_name}' not found.") return @@ -399,10 +397,9 @@ class SerenaConfigCommands(AutoRegisteringGroup): "edit", help="Edit serena_config.yml in your default editor. Will create a config file from the template if no config is found." ) def edit() -> None: - config_path = os.path.join(SERENA_MANAGED_DIR_IN_HOME, "serena_config.yml") - if not os.path.exists(config_path): - SerenaConfig.generate_config_file(config_path) - _open_in_editor(config_path) + serena_config = SerenaConfig.from_config_file() + assert serena_config.config_file_path is not None + _open_in_editor(serena_config.config_file_path) class ProjectCommands(AutoRegisteringGroup): @@ -414,26 +411,74 @@ class ProjectCommands(AutoRegisteringGroup): ) @staticmethod - @click.command("generate-yml", help="Generate a project.yml file.") - @click.argument("project_path", type=click.Path(exists=True, file_okay=False), default=os.getcwd()) - @click.option("--language", type=str, default=None, help="Programming language; inferred if not specified.") - def generate_yml(project_path: str, language: str | None = None) -> None: + def _create_project(project_path: str, name: str | None, language: tuple[str, ...]) -> ProjectConfig: + """ + Helper method to create a project configuration file. + + :param project_path: Path to the project directory + :param name: Optional project name (defaults to directory name if not specified) + :param language: Tuple of language names + :return: The generated ProjectConfig instance + :raises FileExistsError: If project.yml already exists + :raises ValueError: If an unsupported language is specified + """ yml_path = os.path.join(project_path, ProjectConfig.rel_path_to_project_yml()) if os.path.exists(yml_path): raise FileExistsError(f"Project file {yml_path} already exists.") - lang_inst = None + + languages: list[Language] = [] if language: - try: - lang_inst = Language[language.upper()] - except KeyError: - all_langs = [l.name.lower() for l in Language.iter_all(include_experimental=True)] - raise ValueError(f"Unknown language '{language}'. Supported: {all_langs}") - generated_conf = ProjectConfig.autogenerate(project_root=project_path, project_language=lang_inst) - print(f"Generated project.yml with language {generated_conf.language.value} at {yml_path}.") + for lang in language: + try: + languages.append(Language(lang.lower())) + except ValueError: + all_langs = [l.value for l in Language] + raise ValueError(f"Unknown language '{lang}'. Supported: {all_langs}") + + generated_conf = ProjectConfig.autogenerate( + project_root=project_path, project_name=name, languages=languages if languages else None, interactive=True + ) + yml_path = ProjectConfig.path_to_project_yml(project_path) + languages_str = ", ".join([lang.value for lang in generated_conf.languages]) if generated_conf.languages else "N/A" + click.echo(f"Generated project with languages {{{languages_str}}} at {yml_path}.") + return generated_conf @staticmethod - @click.command("index", help="Index a project by saving symbols to the LSP cache.") + @click.command("create", help="Create a new Serena project configuration.") + @click.argument("project_path", type=click.Path(exists=True, file_okay=False), default=os.getcwd()) + @click.option("--name", type=str, default=None, help="Project name; defaults to directory name if not specified.") + @click.option( + "--language", type=str, multiple=True, help="Programming language(s); inferred if not specified. Can be passed multiple times." + ) + @click.option("--index", is_flag=True, help="Index the project after creation.") + @click.option( + "--log-level", + type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), + default="WARNING", + help="Log level for indexing (only used if --index is set).", + ) + @click.option("--timeout", type=float, default=10, help="Timeout for indexing a single file (only used if --index is set).") + def create(project_path: str, name: str | None, language: tuple[str, ...], index: bool, log_level: str, timeout: float) -> None: + try: + ProjectCommands._create_project(project_path, name, language) + if index: + click.echo("Indexing project...") + ProjectCommands._index_project(project_path, log_level, timeout=timeout) + except FileExistsError as e: + raise click.ClickException(f"Project already exists: {e}\nUse 'serena project index' to index an existing project.") + except ValueError as e: + raise click.ClickException(str(e)) + + @staticmethod + @click.command("index", help="Index a project by saving symbols to the LSP cache. Auto-creates project.yml if it doesn't exist.") @click.argument("project", type=click.Path(exists=True), default=os.getcwd(), required=False) + @click.option("--name", type=str, default=None, help="Project name (only used if auto-creating project.yml).") + @click.option( + "--language", + type=str, + multiple=True, + help="Programming language(s) (only used if auto-creating project.yml). Inferred if not specified.", + ) @click.option( "--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), @@ -441,7 +486,19 @@ class ProjectCommands(AutoRegisteringGroup): help="Log level for indexing.", ) @click.option("--timeout", type=float, default=10, help="Timeout for indexing a single file.") - def index(project: str, log_level: str, timeout: float) -> None: + def index(project: str, name: str | None, language: tuple[str, ...], log_level: str, timeout: float) -> None: + # Check if project.yml exists, if not auto-create it + yml_path = os.path.join(project, ProjectConfig.rel_path_to_project_yml()) + if not os.path.exists(yml_path): + click.echo(f"Project configuration not found at {yml_path}. Auto-creating...") + try: + ProjectCommands._create_project(project, name, language) + except FileExistsError: + # Race condition - file was created between check and creation + pass + except ValueError as e: + raise click.ClickException(str(e)) + ProjectCommands._index_project(project, log_level, timeout=timeout) @staticmethod @@ -459,33 +516,42 @@ class ProjectCommands(AutoRegisteringGroup): logging.configure(level=lvl) serena_config = SerenaConfig.from_config_file() proj = Project.load(os.path.abspath(project)) - click.echo(f"Indexing symbols in project {project}…") - ls = proj.create_language_server(log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings) - log_file = os.path.join(project, ".serena", "logs", "indexing.txt") + click.echo(f"Indexing symbols in project {project} …") + ls_mgr = proj.create_language_server_manager( + log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings + ) + try: + log_file = os.path.join(project, ".serena", "logs", "indexing.txt") - collected_exceptions: list[Exception] = [] - files_failed = [] - with ls.start_server(): files = proj.gather_source_files() + + collected_exceptions: list[Exception] = [] + files_failed = [] + language_file_counts: dict[Language, int] = collections.defaultdict(lambda: 0) for i, f in enumerate(tqdm(files, desc="Indexing")): try: - ls.request_document_symbols(f, include_body=False) - ls.request_document_symbols(f, include_body=True) + ls = ls_mgr.get_language_server(f) + ls.request_document_symbols(f) + language_file_counts[ls.language] += 1 except Exception as e: log.error(f"Failed to index {f}, continuing.") collected_exceptions.append(e) files_failed.append(f) if (i + 1) % 10 == 0: - ls.save_cache() - ls.save_cache() - click.echo(f"Symbols saved to {ls.cache_path}") - if len(files_failed) > 0: - os.makedirs(os.path.dirname(log_file), exist_ok=True) - with open(log_file, "w") as f: - for file, exception in zip(files_failed, collected_exceptions, strict=True): - f.write(f"{file}\n") - f.write(f"{exception}\n") - click.echo(f"Failed to index {len(files_failed)} files, see:\n{log_file}") + ls_mgr.save_all_caches() + reported_language_file_counts = {k.value: v for k, v in language_file_counts.items()} + click.echo(f"Indexed files per language: {dict_string(reported_language_file_counts, brackets=None)}") + ls_mgr.save_all_caches() + + if len(files_failed) > 0: + os.makedirs(os.path.dirname(log_file), exist_ok=True) + with open(log_file, "w") as f: + for file, exception in zip(files_failed, collected_exceptions, strict=True): + f.write(f"{file}\n") + f.write(f"{exception}\n") + click.echo(f"Failed to index {len(files_failed)} files, see:\n{log_file}") + finally: + ls_mgr.stop_all() @staticmethod @click.command("is_ignored_path", help="Check if a path is ignored by the project configuration.") @@ -522,16 +588,20 @@ class ProjectCommands(AutoRegisteringGroup): if proj.is_ignored_path(file, ignore_non_source_files=True): click.echo(f"'{file}' is ignored or declared as non-code file by the project configuration, won't index.") exit(1) - ls = proj.create_language_server() - with ls.start_server(): - symbols, _ = ls.request_document_symbols(file, include_body=False) - ls.request_document_symbols(file, include_body=True) - if verbose: - click.echo(f"Symbols in file '{file}':") - for symbol in symbols: - click.echo(f" - {symbol['name']} at line {symbol['selectionRange']['start']['line']} of kind {symbol['kind']}") - ls.save_cache() - click.echo(f"Successfully indexed file '{file}', {len(symbols)} symbols saved to {ls.cache_path}.") + ls_mgr = proj.create_language_server_manager() + try: + for ls in ls_mgr.iter_language_servers(): + click.echo(f"Indexing for language {ls.language.value} …") + document_symbols = ls.request_document_symbols(file) + symbols, _ = document_symbols.get_all_symbols_and_roots() + if verbose: + click.echo(f"Symbols in file '{file}':") + for symbol in symbols: + click.echo(f" - {symbol['name']} at line {symbol['selectionRange']['start']['line']} of kind {symbol['kind']}") + ls.save_cache() + click.echo(f"Successfully indexed file '{file}', {len(symbols)} symbols saved to cache in {ls.cache_dir}.") + finally: + ls_mgr.stop_all() @staticmethod @click.command("health-check", help="Perform a comprehensive health check of the project's tools and language server.") @@ -736,8 +806,9 @@ class PromptCommands(AutoRegisteringGroup): @staticmethod def _get_user_prompt_yaml_path(prompt_yaml_name: str) -> str: - os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True) - return os.path.join(PROMPT_TEMPLATES_DIR_IN_USER_HOME, prompt_yaml_name) + templates_dir = SerenaPaths().user_prompt_templates_dir + os.makedirs(templates_dir, exist_ok=True) + return os.path.join(templates_dir, prompt_yaml_name) @staticmethod @click.command("list", help="Lists yamls that are used for defining prompts.") @@ -788,9 +859,10 @@ class PromptCommands(AutoRegisteringGroup): @staticmethod @click.command("list-overrides", help="List existing prompt override files") def list_overrides() -> None: - os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True) + user_templates_dir = SerenaPaths().user_prompt_templates_dir + os.makedirs(user_templates_dir, exist_ok=True) serena_prompt_yaml_names = [os.path.basename(f) for f in glob.glob(PROMPT_TEMPLATES_DIR_INTERNAL + "/*.yml")] - override_files = glob.glob(os.path.join(PROMPT_TEMPLATES_DIR_IN_USER_HOME, "*.yml")) + override_files = glob.glob(os.path.join(user_templates_dir, "*.yml")) for file_path in override_files: if os.path.basename(file_path) in serena_prompt_yaml_names: click.echo(file_path) diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index 6fe41da1..6e9f65b1 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -4,13 +4,15 @@ import os from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator, Reversible from contextlib import contextmanager -from typing import TYPE_CHECKING, Generic, Optional, TypeVar +from typing import TYPE_CHECKING, Generic, Optional, TypeVar, cast from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol -from solidlsp import SolidLanguageServer +from solidlsp import SolidLanguageServer, ls_types from solidlsp.ls import LSPFileBuffer -from solidlsp.ls_utils import TextUtils +from solidlsp.ls_types import extract_text_edits +from solidlsp.ls_utils import PathUtils, TextUtils +from .constants import DEFAULT_SOURCE_FILE_ENCODING from .project import Project from .tools.jetbrains_plugin_client import JetBrainsPluginClient @@ -27,6 +29,14 @@ class CodeEditor(Generic[TSymbol], ABC): self.project_root = project_root self.agent = agent + # set encoding based on active project, if available + encoding = DEFAULT_SOURCE_FILE_ENCODING + if agent is not None: + project = agent.get_active_project() + if project is not None: + encoding = project.project_config.encoding + self.encoding = encoding + class EditedFile(ABC): @abstractmethod def get_contents(self) -> str: @@ -58,7 +68,7 @@ class CodeEditor(Generic[TSymbol], ABC): yield edited_file # save the file abs_path = os.path.join(self.project_root, relative_path) - with open(abs_path, "w", encoding="utf-8") as f: + with open(abs_path, "w", encoding=self.encoding) as f: f.write(edited_file.get_contents()) @abstractmethod @@ -207,15 +217,25 @@ class CodeEditor(Generic[TSymbol], ABC): with self._edited_file_context(relative_file_path) as edited_file: edited_file.delete_text_between_positions(start_pos, end_pos) + @abstractmethod + def rename_symbol(self, name_path: str, relative_file_path: str, new_name: str) -> str: + """ + Renames the symbol with the given name throughout the codebase. + + :param name_path: the name path of the symbol to rename + :param relative_file_path: the relative path of the file containing the symbol + :param new_name: the new name for the symbol + :return: a status message + """ + class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, agent: Optional["SerenaAgent"] = None): - super().__init__(project_root=symbol_retriever.get_language_server().repository_root_path, agent=agent) + super().__init__(project_root=symbol_retriever.get_root_path(), agent=agent) self._symbol_retriever = symbol_retriever - @property - def _lang_server(self) -> SolidLanguageServer: - return self._symbol_retriever.get_language_server() + def _get_language_server(self, relative_path: str) -> SolidLanguageServer: + return self._symbol_retriever.get_language_server(relative_path) class EditedFile(CodeEditor.EditedFile): def __init__(self, lang_server: SolidLanguageServer, relative_path: str, file_buffer: LSPFileBuffer): @@ -232,14 +252,19 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): def insert_text_at_position(self, pos: PositionInFile, text: str) -> None: self._lang_server.insert_text_at_position(self._relative_path, pos.line, pos.col, text) + def apply_text_edits(self, text_edits: list[ls_types.TextEdit]) -> None: + return self._lang_server.apply_text_edits_to_file(self._relative_path, text_edits) + @contextmanager def _open_file_context(self, relative_path: str) -> Iterator["CodeEditor.EditedFile"]: - with self._lang_server.open_file(relative_path) as file_buffer: - yield self.EditedFile(self._lang_server, relative_path, file_buffer) + lang_server = self._get_language_server(relative_path) + with lang_server.open_file(relative_path) as file_buffer: + yield self.EditedFile(lang_server, relative_path, file_buffer) def _get_code_file_content(self, relative_path: str) -> str: """Get the content of a file using the language server.""" - return self._lang_server.language_server.retrieve_full_file_content(relative_path) + lang_server = self._get_language_server(relative_path) + return lang_server.language_server.retrieve_full_file_content(relative_path) def _find_unique_symbol(self, name_path: str, relative_file_path: str) -> LanguageServerSymbol: symbol_candidates = self._symbol_retriever.find_by_name(name_path, within_relative_path=relative_file_path) @@ -252,6 +277,48 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): ) return symbol_candidates[0] + def _apply_workspace_edit(self, workspace_edit: ls_types.WorkspaceEdit) -> list[str]: + """ + Apply a WorkspaceEdit by making the changes to files. + + :param workspace_edit: The WorkspaceEdit containing the changes to apply + :return: List of relative file paths that were modified + """ + uri_to_edits = extract_text_edits(workspace_edit) + modified_relative_paths = [] + + # Handle the 'changes' format (URI -> list of TextEdits) + for uri, edits in uri_to_edits.items(): + file_path = PathUtils.uri_to_path(uri) + relative_path = os.path.relpath(file_path, self._symbol_retriever.get_root_path()) + modified_relative_paths.append(relative_path) + with self._edited_file_context(relative_path) as edited_file: + edited_file = cast(LanguageServerCodeEditor.EditedFile, edited_file) + edited_file.apply_text_edits(edits) + return modified_relative_paths + + def rename_symbol(self, name_path: str, relative_file_path: str, new_name: str) -> str: + symbol = self._find_unique_symbol(name_path, relative_file_path) + if not symbol.location.has_position_in_file(): + raise ValueError(f"Symbol '{name_path}' does not have a valid position in file for renaming") + + # After has_position_in_file check, line and column are guaranteed to be non-None + assert symbol.location.line is not None + assert symbol.location.column is not None + + lang_server = self._get_language_server(relative_file_path) + rename_result = lang_server.request_rename_symbol_edit( + relative_file_path=relative_file_path, line=symbol.location.line, column=symbol.location.column, new_name=new_name + ) + if rename_result is None: + raise ValueError( + f"Language server for {lang_server.language_id} returned no rename edits for symbol '{name_path}'. " + f"The symbol might not support renaming." + ) + modified_files = self._apply_workspace_edit(rename_result) + msg = f"Successfully renamed '{name_path}' to '{new_name}' in {len(modified_files)} file(s)" + return msg + class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]): def __init__(self, project: Project, agent: Optional["SerenaAgent"] = None) -> None: @@ -292,3 +359,14 @@ class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]): "Their locations are: \n " + json.dumps([s["location"] for s in symbols], indent=2) ) return JetBrainsSymbol(symbols[0], self._project) + + def rename_symbol(self, name_path: str, relative_file_path: str, new_name: str) -> str: + with JetBrainsPluginClient.from_project(self._project) as client: + client.rename_symbol( + name_path=name_path, + relative_path=relative_file_path, + new_name=new_name, + rename_in_comments=False, + rename_in_text_occurrences=False, + ) + return "Success" diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index 44cd42fb..f1f34fd9 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -4,7 +4,6 @@ Context and Mode configuration loader import os from dataclasses import dataclass, field -from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Self @@ -12,15 +11,14 @@ import yaml from sensai.util import logging from sensai.util.string import ToStringMixin -from serena.config.serena_config import ToolInclusionDefinition +from serena.config.serena_config import SerenaPaths, ToolInclusionDefinition from serena.constants import ( DEFAULT_CONTEXT, DEFAULT_MODES, INTERNAL_MODE_YAMLS_DIR, + SERENA_FILE_ENCODING, SERENAS_OWN_CONTEXT_YAMLS_DIR, SERENAS_OWN_MODE_YAMLS_DIR, - USER_CONTEXT_YAMLS_DIR, - USER_MODE_YAMLS_DIR, ) if TYPE_CHECKING: @@ -56,7 +54,7 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a mode from a YAML file.""" - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) return cls(name=name, **data) @@ -65,14 +63,14 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): def get_path(cls, name: str) -> str: """Get the path to the YAML file for a mode.""" fname = f"{name}.yml" - custom_mode_path = os.path.join(USER_MODE_YAMLS_DIR, fname) + custom_mode_path = os.path.join(SerenaPaths().user_modes_dir, fname) if os.path.exists(custom_mode_path): return custom_mode_path own_yaml_path = os.path.join(SERENAS_OWN_MODE_YAMLS_DIR, fname) if not os.path.exists(own_yaml_path): raise FileNotFoundError( - f"Mode {name} not found in {USER_MODE_YAMLS_DIR} or in {SERENAS_OWN_MODE_YAMLS_DIR}." + f"Mode {name} not found in {SerenaPaths().user_modes_dir} or in {SERENAS_OWN_MODE_YAMLS_DIR}." f"Available modes:\n{cls.list_registered_mode_names()}" ) return own_yaml_path @@ -102,7 +100,7 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): @classmethod def list_custom_mode_names(cls) -> list[str]: """Names of all custom modes defined by the user.""" - return [f.stem for f in Path(USER_MODE_YAMLS_DIR).glob("*.yml")] + return [f.stem for f in Path(SerenaPaths().user_modes_dir).glob("*.yml")] @classmethod def load_default_modes(cls) -> list[Self]: @@ -124,14 +122,29 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): """ name: str + """the name of the context""" + prompt: str """ a Jinja2 template for the generation of the system prompt. It is formatted by the agent (see SerenaAgent._format_prompt()). """ + description: str = "" + tool_description_overrides: dict[str, str] = field(default_factory=dict) - """Maps tool names to custom descriptions, default descriptions are extracted from the tool docstrings.""" + """ + maps tool names to custom descriptions, default descriptions are extracted from the tool docstrings. + """ + + single_project: bool = False + """ + whether to assume that Serena shall only work on a single project in this context (provided that a project is given + when Serena is started). + If set to true and a project is provided at startup, the set of tools is limited to those required by the project's + concrete configuration, and other tools are excluded completely, allowing the set of tools to be minimal. + The `activate_project` tool will, therefore, be disabled in this case, as project switching is not allowed. + """ def _tostring_includes(self) -> list[str]: return ["name"] @@ -139,7 +152,7 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a context from a YAML file.""" - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) # Ensure backwards compatibility for tool_description_overrides @@ -151,14 +164,14 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): def get_path(cls, name: str) -> str: """Get the path to the YAML file for a context.""" fname = f"{name}.yml" - custom_context_path = os.path.join(USER_CONTEXT_YAMLS_DIR, fname) + custom_context_path = os.path.join(SerenaPaths().user_contexts_dir, fname) if os.path.exists(custom_context_path): return custom_context_path own_yaml_path = os.path.join(SERENAS_OWN_CONTEXT_YAMLS_DIR, fname) if not os.path.exists(own_yaml_path): raise FileNotFoundError( - f"Context {name} not found in {USER_CONTEXT_YAMLS_DIR} or in {SERENAS_OWN_CONTEXT_YAMLS_DIR}." + f"Context {name} not found in {SerenaPaths().user_contexts_dir} or in {SERENAS_OWN_CONTEXT_YAMLS_DIR}." f"Available contexts:\n{cls.list_registered_context_names()}" ) return own_yaml_path @@ -166,6 +179,16 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): @classmethod def from_name(cls, name: str) -> Self: """Load a registered Serena context.""" + legacy_name_mapping = { + "ide-assistant": "claude-code", + } + if name in legacy_name_mapping: + log.warning( + f"Context name '{name}' is deprecated and has been renamed to '{legacy_name_mapping[name]}'. " + f"Please update your configuration; refer to the configuration guide for more details: " + "https://oraios.github.io/serena/02-usage/050_configuration.html#contexts" + ) + name = legacy_name_mapping[name] context_path = cls.get_path(name) return cls.from_yaml(context_path) @@ -186,7 +209,7 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): @classmethod def list_custom_context_names(cls) -> list[str]: """Names of all custom contexts defined by the user.""" - return [f.stem for f in Path(USER_CONTEXT_YAMLS_DIR).glob("*.yml")] + return [f.stem for f in Path(SerenaPaths().user_contexts_dir).glob("*.yml")] @classmethod def load_default(cls) -> Self: @@ -198,35 +221,3 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): print(f"{self.name}:\n {self.description}") if self.excluded_tools: print(" excluded tools:\n " + ", ".join(sorted(self.excluded_tools))) - - -class RegisteredContext(Enum): - """A registered context.""" - - IDE_ASSISTANT = "ide-assistant" - """For Serena running within an assistant that already has basic tools, like Claude Code, Cline, Cursor, etc.""" - DESKTOP_APP = "desktop-app" - """For Serena running within Claude Desktop or a similar app which does not have built-in tools for code editing.""" - AGENT = "agent" - """For Serena running as a standalone agent, e.g. through agno.""" - - def load(self) -> SerenaAgentContext: - """Load the context.""" - return SerenaAgentContext.from_name(self.value) - - -class RegisteredMode(Enum): - """A registered mode.""" - - INTERACTIVE = "interactive" - """Interactive mode, for multi-turn interactions.""" - EDITING = "editing" - """Editing tools are activated.""" - PLANNING = "planning" - """Editing tools are deactivated.""" - ONE_SHOT = "one-shot" - """Non-interactive mode, where the goal is to finish a task autonomously.""" - - def load(self) -> SerenaAgentMode: - """Load the mode.""" - return SerenaAgentMode.from_name(self.value) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 00bbc901..c3fb2a4a 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -2,6 +2,7 @@ The Serena Model Context Protocol (MCP) Server """ +import dataclasses import os import shutil from collections.abc import Iterable @@ -19,11 +20,11 @@ from sensai.util.logging import LogTime, datetime_tag from sensai.util.string import ToStringMixin from serena.constants import ( - DEFAULT_ENCODING, + DEFAULT_SOURCE_FILE_ENCODING, PROJECT_TEMPLATE_FILE, REPO_ROOT, SERENA_CONFIG_TEMPLATE_FILE, - SERENA_MANAGED_DIR_IN_HOME, + SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME, ) from serena.util.general import load_yaml, save_yaml @@ -32,6 +33,7 @@ from solidlsp.ls_config import Language from ..analytics import RegisteredTokenCountEstimator from ..util.class_decorators import singleton +from ..util.cli_util import ask_yes_no if TYPE_CHECKING: from ..project import Project @@ -39,6 +41,8 @@ if TYPE_CHECKING: log = logging.getLogger(__name__) T = TypeVar("T") DEFAULT_TOOL_TIMEOUT: float = 240 +DictType = dict | CommentedMap +TDict = TypeVar("TDict", bound=DictType) @singleton @@ -48,9 +52,32 @@ class SerenaPaths: """ def __init__(self) -> None: - self.user_config_dir: str = SERENA_MANAGED_DIR_IN_HOME + home_dir = os.getenv("SERENA_HOME") + if home_dir is None or home_dir.strip() == "": + home_dir = str(Path.home() / SERENA_MANAGED_DIR_NAME) + else: + home_dir = home_dir.strip() + self.serena_user_home_dir: str = home_dir """ - the path to the user's Serena configuration directory, which is typically ~/.serena + the path to the Serena home directory, where the user's configuration/data is stored. + This is ~/.serena by default, but it can be overridden via the SERENA_HOME environment variable. + """ + self.user_prompt_templates_dir: str = os.path.join(self.serena_user_home_dir, "prompt_templates") + """ + directory containing prompt templates defined by the user. + Prompts defined by the user take precedence over Serena's built-in prompt templates. + """ + self.user_contexts_dir: str = os.path.join(self.serena_user_home_dir, "contexts") + """ + directory containing contexts defined by the user. + If a name of a context matches a name of a context in SERENAS_OWN_CONTEXT_YAMLS_DIR, + the user context will override the default context definition. + """ + self.user_modes_dir: str = os.path.join(self.serena_user_home_dir, "modes") + """ + directory containing modes defined by the user. + If a name of a mode matches a name of a mode in SERENAS_OWN_MODES_YAML_DIR, + the user mode will override the default mode definition. """ def get_next_log_file_path(self, prefix: str) -> str: @@ -58,79 +85,13 @@ class SerenaPaths: :param prefix: the filename prefix indicating the type of the log file :return: the full path to the log file to use """ - log_dir = os.path.join(self.user_config_dir, "logs", datetime.now().strftime("%Y-%m-%d")) + log_dir = os.path.join(self.serena_user_home_dir, "logs", datetime.now().strftime("%Y-%m-%d")) os.makedirs(log_dir, exist_ok=True) return os.path.join(log_dir, prefix + "_" + datetime_tag() + ".txt") # TODO: Paths from constants.py should be moved here -class ToolSet: - def __init__(self, tool_names: set[str]) -> None: - self._tool_names = tool_names - - @classmethod - def default(cls) -> "ToolSet": - """ - :return: the default tool set, which contains all tools that are enabled by default - """ - from serena.tools import ToolRegistry - - return cls(set(ToolRegistry().get_tool_names_default_enabled())) - - def apply(self, *tool_inclusion_definitions: "ToolInclusionDefinition") -> "ToolSet": - """ - :param tool_inclusion_definitions: the definitions to apply - :return: a new tool set with the definitions applied - """ - from serena.tools import ToolRegistry - - registry = ToolRegistry() - tool_names = set(self._tool_names) - for definition in tool_inclusion_definitions: - included_tools = [] - excluded_tools = [] - for included_tool in definition.included_optional_tools: - if not registry.is_valid_tool_name(included_tool): - raise ValueError(f"Invalid tool name '{included_tool}' provided for inclusion") - if included_tool not in tool_names: - tool_names.add(included_tool) - included_tools.append(included_tool) - for excluded_tool in definition.excluded_tools: - if not registry.is_valid_tool_name(excluded_tool): - raise ValueError(f"Invalid tool name '{excluded_tool}' provided for exclusion") - if excluded_tool in tool_names: - tool_names.remove(excluded_tool) - excluded_tools.append(excluded_tool) - if included_tools: - log.info(f"{definition} included {len(included_tools)} tools: {', '.join(included_tools)}") - if excluded_tools: - log.info(f"{definition} excluded {len(excluded_tools)} tools: {', '.join(excluded_tools)}") - return ToolSet(tool_names) - - def without_editing_tools(self) -> "ToolSet": - """ - :return: a new tool set that excludes all tools that can edit - """ - from serena.tools import ToolRegistry - - registry = ToolRegistry() - tool_names = set(self._tool_names) - for tool_name in self._tool_names: - if registry.get_tool_class_by_name(tool_name).can_edit(): - tool_names.remove(tool_name) - return ToolSet(tool_names) - - def get_tool_names(self) -> set[str]: - """ - Returns the names of the tools that are currently included in the tool set. - """ - return self._tool_names - - def includes_name(self, tool_name: str) -> bool: - return tool_name in self._tool_names - - @dataclass class ToolInclusionDefinition: excluded_tools: Iterable[str] = () @@ -161,12 +122,12 @@ def is_running_in_docker() -> bool: @dataclass(kw_only=True) class ProjectConfig(ToolInclusionDefinition, ToStringMixin): project_name: str - language: Language + languages: list[Language] ignored_paths: list[str] = field(default_factory=list) read_only: bool = False ignore_all_files_in_gitignore: bool = True initial_prompt: str = "" - encoding: str = DEFAULT_ENCODING + encoding: str = DEFAULT_SOURCE_FILE_ENCODING SERENA_DEFAULT_PROJECT_FILE = "project.yml" @@ -175,7 +136,12 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): @classmethod def autogenerate( - cls, project_root: str | Path, project_name: str | None = None, project_language: Language | None = None, save_to_disk: bool = True + cls, + project_root: str | Path, + project_name: str | None = None, + languages: list[Language] | None = None, + save_to_disk: bool = True, + interactive: bool = False, ) -> Self: """ Autogenerate a project configuration for a given project root. @@ -183,8 +149,9 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): :param project_root: the path to the project root :param project_name: the name of the project; if None, the name of the project will be the name of the directory containing the project - :param project_language: the programming language of the project; if None, it will be determined automatically + :param languages: the languages of the project; if None, they will be determined automatically :param save_to_disk: whether to save the project configuration to disk + :param interactive: whether to run in interactive CLI mode, asking the user for input where appropriate :return: the project configuration """ project_root = Path(project_root).resolve() @@ -192,61 +159,131 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): raise FileNotFoundError(f"Project root not found: {project_root}") with LogTime("Project configuration auto-generation", logger=log): project_name = project_name or project_root.name - if project_language is None: + if languages is None: + # determine languages automatically language_composition = determine_programming_language_composition(str(project_root)) if len(language_composition) == 0: + language_values = ", ".join([lang.value for lang in Language]) raise ValueError( f"No source files found in {project_root}\n\n" - f"To use Serena with this project, you need to either:\n" - f"1. Add source files in one of the supported languages (Python, JavaScript/TypeScript, Java, C#, Rust, Go, Ruby, C++, PHP, Swift, Elixir, Terraform, Bash)\n" - f"2. Create a project configuration file manually at:\n" - f" {os.path.join(project_root, cls.rel_path_to_project_yml())}\n\n" - f"Example project.yml:\n" - f" project_name: {project_name}\n" - f" language: python # or typescript, java, csharp, rust, go, ruby, cpp, php, swift, elixir, terraform, bash\n" + f"To use Serena with this project, you need to either\n" + f" 1. specify a programming language by adding parameters --language \n" + f" when creating the project via the Serena CLI command OR\n" + f" 2. add source files in one of the supported languages first.\n\n" + f"Supported languages are: {language_values}\n" + f"Read the documentation for more information." ) - # find the language with the highest percentage - dominant_language = max(language_composition.keys(), key=lambda lang: language_composition[lang]) + # sort languages by number of files found + languages_and_percentages = sorted(language_composition.items(), key=lambda item: item[1], reverse=True) + # find the language with the highest percentage and enable it + top_language_pair = languages_and_percentages[0] + other_language_pairs = languages_and_percentages[1:] + languages_to_use = [top_language_pair[0]] + # if in interactive mode, ask the user which other languages to enable + if len(other_language_pairs) > 0 and interactive: + print( + "Detected and enabled main language '%s' (%.2f%% of source files)." % (top_language_pair[0], top_language_pair[1]) + ) + print(f"Additionally detected {len(other_language_pairs)} other language(s).\n") + print("Note: Enable only languages you need symbolic retrieval/editing capabilities for.") + print(" Additional language servers use resources and some languages may require additional") + print(" system-level installations/configuration (see Serena documentation).") + print("\nWhich additional languages do you want to enable?") + for lang, perc in other_language_pairs: + enable = ask_yes_no("Enable %s (%.2f%% of source files)?" % (lang, perc), default=False) + if enable: + languages_to_use.append(lang) + print() else: - dominant_language = project_language.value - config_with_comments = load_yaml(PROJECT_TEMPLATE_FILE, preserve_comments=True) + languages_to_use = [lang.value for lang in languages] + config_with_comments = cls.load_commented_map(PROJECT_TEMPLATE_FILE) config_with_comments["project_name"] = project_name - config_with_comments["language"] = dominant_language + config_with_comments["languages"] = languages_to_use if save_to_disk: - save_yaml(str(project_root / cls.rel_path_to_project_yml()), config_with_comments, preserve_comments=True) + save_yaml(cls.path_to_project_yml(project_root), config_with_comments, preserve_comments=True) return cls._from_dict(config_with_comments) + @classmethod + def path_to_project_yml(cls, project_root: str | Path) -> str: + return os.path.join(project_root, cls.rel_path_to_project_yml()) + @classmethod def rel_path_to_project_yml(cls) -> str: return os.path.join(SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE) + @classmethod + def _apply_defaults_to_dict(cls, data: TDict) -> TDict: + # apply defaults for new fields + data["languages"] = data.get("languages", []) + data["ignored_paths"] = data.get("ignored_paths", []) + data["excluded_tools"] = data.get("excluded_tools", []) + data["included_optional_tools"] = data.get("included_optional_tools", []) + data["read_only"] = data.get("read_only", False) + data["ignore_all_files_in_gitignore"] = data.get("ignore_all_files_in_gitignore", True) + data["initial_prompt"] = data.get("initial_prompt", "") + data["encoding"] = data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING) + + # backward compatibility: handle single "language" field + if len(data["languages"]) == 0 and "language" in data: + data["languages"] = [data["language"]] + if "language" in data: + del data["language"] + + return data + + @classmethod + def load_commented_map(cls, yml_path: str) -> CommentedMap: + """ + Load the project configuration as a CommentedMap, preserving comments and ensuring + completeness of the configuration by applying default values for missing fields + and backward compatibility adjustments. + + :param yml_path: the path to the project.yml file + :return: a CommentedMap representing a full project configuration + """ + data = load_yaml(yml_path, preserve_comments=True) + return cls._apply_defaults_to_dict(data) + @classmethod def _from_dict(cls, data: dict[str, Any]) -> Self: """ - Create a ProjectConfig instance from a configuration dictionary + Create a ProjectConfig instance from a (full) configuration dictionary """ - language_str = data["language"].lower() - project_name = data["project_name"] - # backwards compatibility - if language_str == "javascript": - log.warning(f"Found deprecated project language `javascript` in project {project_name}, please change to `typescript`") - language_str = "typescript" - try: - language = Language(language_str) - except ValueError as e: - raise ValueError(f"Invalid language: {data['language']}.\nValid languages are: {[l.value for l in Language]}") from e + lang_name_mapping = {"javascript": "typescript"} + languages: list[Language] = [] + for language_str in data["languages"]: + orig_language_str = language_str + try: + language_str = language_str.lower() + if language_str in lang_name_mapping: + language_str = lang_name_mapping[language_str] + language = Language(language_str) + languages.append(language) + except ValueError as e: + raise ValueError( + f"Invalid language: {orig_language_str}.\nValid language_strings are: {[l.value for l in Language]}" + ) from e + return cls( - project_name=project_name, - language=language, - ignored_paths=data.get("ignored_paths", []), - excluded_tools=data.get("excluded_tools", []), - included_optional_tools=data.get("included_optional_tools", []), - read_only=data.get("read_only", False), - ignore_all_files_in_gitignore=data.get("ignore_all_files_in_gitignore", True), - initial_prompt=data.get("initial_prompt", ""), - encoding=data.get("encoding", DEFAULT_ENCODING), + project_name=data["project_name"], + languages=languages, + ignored_paths=data["ignored_paths"], + excluded_tools=data["excluded_tools"], + included_optional_tools=data["included_optional_tools"], + read_only=data["read_only"], + ignore_all_files_in_gitignore=data["ignore_all_files_in_gitignore"], + initial_prompt=data["initial_prompt"], + encoding=data["encoding"], ) + def to_yaml_dict(self) -> dict: + """ + :return: a yaml-serializable dictionary representation of this configuration + """ + d = dataclasses.asdict(self) + d["languages"] = [lang.value for lang in self.languages] + return d + @classmethod def load(cls, project_root: Path | str, autogenerate: bool = False) -> Self: """ @@ -259,8 +296,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): return cls.autogenerate(project_root) else: raise FileNotFoundError(f"Project configuration file not found: {yaml_path}") - with open(yaml_path, encoding="utf-8") as f: - yaml_data = yaml.safe_load(f) + yaml_data = cls.load_commented_map(str(yaml_path)) if "project_name" not in yaml_data: yaml_data["project_name"] = project_root.name return cls._from_dict(yaml_data) @@ -339,10 +375,8 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): """ whether to apply JetBrains mode """ - record_tool_usage_stats: bool = False - """Whether to record tool usage statistics, they will be shown in the web dashboard if recording is active. - """ - token_count_estimator: str = RegisteredTokenCountEstimator.TIKTOKEN_GPT4O.name + + token_count_estimator: str = RegisteredTokenCountEstimator.CHAR_COUNT.name """Only relevant if `record_tool_usage` is True; the name of the token count estimator to use for tool usage statistics. See the `RegisteredTokenCountEstimator` enum for available options. @@ -365,7 +399,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): return ["config_file_path"] @classmethod - def generate_config_file(cls, config_file_path: str) -> None: + def _generate_config_file(cls, config_file_path: str) -> None: """ Generates a Serena configuration file at the specified path from the template file. @@ -383,7 +417,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): if is_running_in_docker(): return os.path.join(REPO_ROOT, cls.CONFIG_FILE_DOCKER) else: - config_path = os.path.join(SERENA_MANAGED_DIR_IN_HOME, cls.CONFIG_FILE) + config_path = os.path.join(SerenaPaths().serena_user_home_dir, cls.CONFIG_FILE) # if the config file does not exist, check if we can migrate it from the old location if not os.path.exists(config_path): @@ -407,7 +441,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): if not generate_if_missing: raise FileNotFoundError(f"Serena configuration file not found: {config_file_path}") log.info(f"Serena configuration file not found at {config_file_path}, autogenerating...") - cls.generate_config_file(config_file_path) + cls._generate_config_file(config_file_path) # load the configuration log.info(f"Loading Serena configuration from {config_file_path}") @@ -456,7 +490,6 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): instance.excluded_tools = loaded_commented_yaml.get("excluded_tools", []) instance.included_optional_tools = loaded_commented_yaml.get("included_optional_tools", []) instance.jetbrains = loaded_commented_yaml.get("jetbrains", False) - instance.record_tool_usage_stats = loaded_commented_yaml.get("record_tool_usage_stats", False) instance.token_count_estimator = loaded_commented_yaml.get( "token_count_estimator", RegisteredTokenCountEstimator.TIKTOKEN_GPT4O.name ) @@ -483,11 +516,11 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): """ log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.") try: - with open(path, encoding="utf-8") as f: + with open(path, encoding=SERENA_FILE_ENCODING) as f: project_config_data = yaml.safe_load(f) if "project_name" not in project_config_data: project_name = path.stem - with open(path, "a", encoding="utf-8") as f: + with open(path, "a", encoding=SERENA_FILE_ENCODING) as f: f.write(f"\nproject_name: {project_name}") project_root = project_config_data["project_root"] shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml())) diff --git a/src/serena/constants.py b/src/serena/constants.py index 1e5d98a1..f5ccd660 100644 --- a/src/serena/constants.py +++ b/src/serena/constants.py @@ -4,31 +4,27 @@ _repo_root_path = Path(__file__).parent.parent.parent.resolve() _serena_pkg_path = Path(__file__).parent.resolve() SERENA_MANAGED_DIR_NAME = ".serena" -_serena_in_home_managed_dir = Path.home() / ".serena" - -SERENA_MANAGED_DIR_IN_HOME = str(_serena_in_home_managed_dir) # TODO: Path-related constants should be moved to SerenaPaths; don't add further constants here. REPO_ROOT = str(_repo_root_path) PROMPT_TEMPLATES_DIR_INTERNAL = str(_serena_pkg_path / "resources" / "config" / "prompt_templates") -PROMPT_TEMPLATES_DIR_IN_USER_HOME = str(_serena_in_home_managed_dir / "prompt_templates") SERENAS_OWN_CONTEXT_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "contexts") """The contexts that are shipped with the Serena package, i.e. the default contexts.""" -USER_CONTEXT_YAMLS_DIR = str(_serena_in_home_managed_dir / "contexts") -"""Contexts defined by the user. If a name of a context matches a name of a context in SERENAS_OWN_CONTEXT_YAMLS_DIR, the user context will override the default one.""" SERENAS_OWN_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "modes") """The modes that are shipped with the Serena package, i.e. the default modes.""" -USER_MODE_YAMLS_DIR = str(_serena_in_home_managed_dir / "modes") -"""Modes defined by the user. If a name of a mode matches a name of a mode in SERENAS_OWN_MODE_YAMLS_DIR, the user mode will override the default one.""" INTERNAL_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "internal_modes") """Internal modes, never overridden by user modes.""" SERENA_DASHBOARD_DIR = str(_serena_pkg_path / "resources" / "dashboard") SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons") -DEFAULT_ENCODING = "utf-8" +DEFAULT_SOURCE_FILE_ENCODING = "utf-8" +"""The default encoding assumed for project source files.""" DEFAULT_CONTEXT = "desktop-app" DEFAULT_MODES = ("interactive", "editing") +SERENA_FILE_ENCODING = "utf-8" +"""The encoding used for Serena's own files, such as configuration files and memories.""" + PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.yml") SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml") diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index c4a5f91d..6005bcad 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -2,7 +2,7 @@ import os import socket import threading from collections.abc import Callable -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Self from flask import Flask, Response, request, send_from_directory from pydantic import BaseModel @@ -10,6 +10,7 @@ from sensai.util import logging from serena.analytics import ToolUsageStats from serena.constants import SERENA_DASHBOARD_DIR +from serena.task_executor import TaskExecutor from serena.util.logging import MemoryLogHandler if TYPE_CHECKING: @@ -39,6 +40,82 @@ class ResponseToolStats(BaseModel): stats: dict[str, dict[str, int]] +class ResponseConfigOverview(BaseModel): + active_project: dict[str, str | None] + context: dict[str, str] + modes: list[dict[str, str]] + active_tools: list[str] + tool_stats_summary: dict[str, dict[str, int]] + registered_projects: list[dict[str, str | bool]] + available_tools: list[dict[str, str | bool]] + available_modes: list[dict[str, str | bool]] + available_contexts: list[dict[str, str | bool]] + available_memories: list[str] | None + jetbrains_mode: bool + languages: list[str] + encoding: str | None + + +class ResponseAvailableLanguages(BaseModel): + languages: list[str] + + +class RequestAddLanguage(BaseModel): + language: str + + +class RequestRemoveLanguage(BaseModel): + language: str + + +class RequestGetMemory(BaseModel): + memory_name: str + + +class ResponseGetMemory(BaseModel): + content: str + memory_name: str + + +class RequestSaveMemory(BaseModel): + memory_name: str + content: str + + +class RequestDeleteMemory(BaseModel): + memory_name: str + + +class ResponseGetSerenaConfig(BaseModel): + content: str + + +class RequestSaveSerenaConfig(BaseModel): + content: str + + +class RequestCancelTaskExecution(BaseModel): + task_id: int + + +class QueuedExecution(BaseModel): + task_id: int + is_running: bool + name: str + finished_successfully: bool + logged: bool + + @classmethod + def from_task_info(cls, task_info: TaskExecutor.TaskInfo) -> Self: + return cls( + task_id=task_info.task_id, + is_running=task_info.is_running, + name=task_info.name, + finished_successfully=task_info.finished_successfully(), + logged=task_info.logged, + ) + + class SerenaDashboardAPI: log = logging.getLogger(__qualname__) @@ -73,6 +150,11 @@ class SerenaDashboardAPI: return send_from_directory(SERENA_DASHBOARD_DIR, "index.html") # API routes + + @self._app.route("/heartbeat", methods=["GET"]) + def get_heartbeat() -> dict[str, Any]: + return {"status": "alive"} + @self._app.route("/get_log_messages", methods=["POST"]) def get_log_messages() -> dict[str, Any]: request_data = request.get_json() @@ -104,11 +186,136 @@ class SerenaDashboardAPI: estimator_name = self._tool_usage_stats.token_estimator_name if self._tool_usage_stats else "unknown" return {"token_count_estimator_name": estimator_name} + @self._app.route("/get_config_overview", methods=["GET"]) + def get_config_overview() -> dict[str, Any]: + result = self._agent.execute_task(self._get_config_overview, logged=False) + return result.model_dump() + @self._app.route("/shutdown", methods=["PUT"]) def shutdown() -> dict[str, str]: self._shutdown() return {"status": "shutting down"} + @self._app.route("/get_available_languages", methods=["GET"]) + def get_available_languages() -> dict[str, Any]: + result = self._get_available_languages() + return result.model_dump() + + @self._app.route("/add_language", methods=["POST"]) + def add_language() -> dict[str, str]: + request_data = request.get_json() + if not request_data: + return {"status": "error", "message": "No data provided"} + request_add_language = RequestAddLanguage.model_validate(request_data) + try: + self._add_language(request_add_language) + return {"status": "success", "message": f"Language {request_add_language.language} added successfully"} + except Exception as e: + return {"status": "error", "message": str(e)} + + @self._app.route("/remove_language", methods=["POST"]) + def remove_language() -> dict[str, str]: + request_data = request.get_json() + if not request_data: + return {"status": "error", "message": "No data provided"} + request_remove_language = RequestRemoveLanguage.model_validate(request_data) + try: + self._remove_language(request_remove_language) + return {"status": "success", "message": f"Language {request_remove_language.language} removed successfully"} + except Exception as e: + return {"status": "error", "message": str(e)} + + @self._app.route("/get_memory", methods=["POST"]) + def get_memory() -> dict[str, Any]: + request_data = request.get_json() + if not request_data: + return {"status": "error", "message": "No data provided"} + request_get_memory = RequestGetMemory.model_validate(request_data) + try: + result = self._get_memory(request_get_memory) + return result.model_dump() + except Exception as e: + return {"status": "error", "message": str(e)} + + @self._app.route("/save_memory", methods=["POST"]) + def save_memory() -> dict[str, str]: + request_data = request.get_json() + if not request_data: + return {"status": "error", "message": "No data provided"} + request_save_memory = RequestSaveMemory.model_validate(request_data) + try: + self._save_memory(request_save_memory) + return {"status": "success", "message": f"Memory {request_save_memory.memory_name} saved successfully"} + except Exception as e: + return {"status": "error", "message": str(e)} + + @self._app.route("/delete_memory", methods=["POST"]) + def delete_memory() -> dict[str, str]: + request_data = request.get_json() + if not request_data: + return {"status": "error", "message": "No data provided"} + request_delete_memory = RequestDeleteMemory.model_validate(request_data) + try: + self._delete_memory(request_delete_memory) + return {"status": "success", "message": f"Memory {request_delete_memory.memory_name} deleted successfully"} + except Exception as e: + return {"status": "error", "message": str(e)} + + @self._app.route("/get_serena_config", methods=["GET"]) + def get_serena_config() -> dict[str, Any]: + try: + result = self._get_serena_config() + return result.model_dump() + except Exception as e: + return {"status": "error", "message": str(e)} + + @self._app.route("/save_serena_config", methods=["POST"]) + def save_serena_config() -> dict[str, str]: + request_data = request.get_json() + if not request_data: + return {"status": "error", "message": "No data provided"} + request_save_config = RequestSaveSerenaConfig.model_validate(request_data) + try: + self._save_serena_config(request_save_config) + return {"status": "success", "message": "Serena config saved successfully"} + except Exception as e: + return {"status": "error", "message": str(e)} + + @self._app.route("/queued_task_executions", methods=["GET"]) + def get_queued_executions() -> dict[str, Any]: + try: + current_executions = self._agent.get_current_tasks() + response = [QueuedExecution.from_task_info(task_info).model_dump() for task_info in current_executions] + return {"queued_executions": response, "status": "success"} + except Exception as e: + return {"status": "error", "message": str(e)} + + @self._app.route("/cancel_task_execution", methods=["POST"]) + def cancel_task_execution() -> dict[str, Any]: + request_data = request.get_json() + try: + request_cancel_task = RequestCancelTaskExecution.model_validate(request_data) + for task in self._agent.get_current_tasks(): + if task.task_id == request_cancel_task.task_id: + task.cancel() + return {"status": "success", "was_cancelled": True} + return { + "status": "success", + "was_cancelled": False, + "message": f"Task with id {request_data.get('task_id')} not found, maybe execution was already finished", + } + except Exception as e: + return {"status": "error", "message": str(e), "was_cancelled": False} + + @self._app.route("/last_execution", methods=["GET"]) + def get_last_execution() -> dict[str, Any]: + try: + last_execution_info = self._agent.get_last_executed_task() + response = QueuedExecution.from_task_info(last_execution_info).model_dump() if last_execution_info is not None else None + return {"last_execution": response, "status": "success"} + except Exception as e: + return {"status": "error", "message": str(e)} + def _get_log_messages(self, request_log: RequestLog) -> ResponseLog: all_messages = self._memory_log_handler.get_log_messages() requested_messages = all_messages[request_log.start_idx :] if request_log.start_idx <= len(all_messages) else [] @@ -129,6 +336,118 @@ class SerenaDashboardAPI: if self._tool_usage_stats is not None: self._tool_usage_stats.clear() + def _get_config_overview(self) -> ResponseConfigOverview: + from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode + + # Get active project info + project = self._agent.get_active_project() + active_project_name = project.project_name if project else None + project_info = { + "name": active_project_name, + "language": ", ".join([l.value for l in project.project_config.languages]) if project else None, + "path": str(project.project_root) if project else None, + } + + # Get context info + context = self._agent.get_context() + context_info = { + "name": context.name, + "description": context.description, + "path": SerenaAgentContext.get_path(context.name), + } + + # Get active modes + modes = self._agent.get_active_modes() + modes_info = [{"name": mode.name, "description": mode.description, "path": SerenaAgentMode.get_path(mode.name)} for mode in modes] + active_mode_names = [mode.name for mode in modes] + + # Get active tools + active_tools = self._agent.get_active_tool_names() + + # Get registered projects + registered_projects: list[dict[str, str | bool]] = [] + for proj in self._agent.serena_config.projects: + registered_projects.append( + { + "name": proj.project_name, + "path": str(proj.project_root), + "is_active": proj.project_name == active_project_name, + } + ) + + # Get all available tools (excluding active ones) + all_tool_names = sorted([tool.get_name_from_cls() for tool in self._agent._all_tools.values()]) + available_tools: list[dict[str, str | bool]] = [] + for tool_name in all_tool_names: + if tool_name not in active_tools: + available_tools.append( + { + "name": tool_name, + "is_active": False, + } + ) + + # Get all available modes + all_mode_names = SerenaAgentMode.list_registered_mode_names() + available_modes: list[dict[str, str | bool]] = [] + for mode_name in all_mode_names: + available_modes.append( + { + "name": mode_name, + "is_active": mode_name in active_mode_names, + "path": SerenaAgentMode.get_path(mode_name), + } + ) + + # Get all available contexts + all_context_names = SerenaAgentContext.list_registered_context_names() + available_contexts: list[dict[str, str | bool]] = [] + for context_name in all_context_names: + available_contexts.append( + { + "name": context_name, + "is_active": context_name == context.name, + "path": SerenaAgentContext.get_path(context_name), + } + ) + + # Get basic tool stats (just num_calls for overview) + tool_stats_summary = {} + if self._tool_usage_stats is not None: + full_stats = self._tool_usage_stats.get_tool_stats_dict() + tool_stats_summary = {name: {"num_calls": stats["num_times_called"]} for name, stats in full_stats.items()} + + # Get available memories if ReadMemoryTool is active + available_memories = None + if self._agent.tool_is_active("read_memory") and project is not None: + available_memories = project.memories_manager.list_memories() + + # Get list of languages for the active project + languages = [] + if project is not None: + languages = [lang.value for lang in project.project_config.languages] + + # Get file encoding for the active project + encoding = None + if project is not None: + encoding = project.project_config.encoding + + return ResponseConfigOverview( + active_project=project_info, + context=context_info, + modes=modes_info, + active_tools=active_tools, + tool_stats_summary=tool_stats_summary, + registered_projects=registered_projects, + available_tools=available_tools, + available_modes=available_modes, + available_contexts=available_contexts, + available_memories=available_memories, + jetbrains_mode=self._agent.serena_config.jetbrains, + languages=languages, + encoding=encoding, + ) + def _shutdown(self) -> None: log.info("Shutting down Serena") if self._shutdown_callback: @@ -138,6 +457,96 @@ class SerenaDashboardAPI: # noinspection PyUnresolvedReferences os._exit(0) + def _get_available_languages(self) -> ResponseAvailableLanguages: + from solidlsp.ls_config import Language + + def run() -> ResponseAvailableLanguages: + all_languages = [lang.value for lang in Language.iter_all(include_experimental=False)] + + # Filter out already added languages for the active project + project = self._agent.get_active_project() + if project: + current_languages = [lang.value for lang in project.project_config.languages] + available_languages = [lang for lang in all_languages if lang not in current_languages] + else: + available_languages = all_languages + + return ResponseAvailableLanguages(languages=sorted(available_languages)) + + return self._agent.execute_task(run, logged=False) + + def _get_memory(self, request_get_memory: RequestGetMemory) -> ResponseGetMemory: + def run() -> ResponseGetMemory: + project = self._agent.get_active_project() + if project is None: + raise ValueError("No active project") + + content = project.memories_manager.load_memory(request_get_memory.memory_name) + return ResponseGetMemory(content=content, memory_name=request_get_memory.memory_name) + + return self._agent.execute_task(run, logged=False) + + def _save_memory(self, request_save_memory: RequestSaveMemory) -> None: + def run() -> None: + project = self._agent.get_active_project() + if project is None: + raise ValueError("No active project") + + project.memories_manager.save_memory(request_save_memory.memory_name, request_save_memory.content) + + self._agent.execute_task(run, logged=True, name="SaveMemory") + + def _delete_memory(self, request_delete_memory: RequestDeleteMemory) -> None: + def run() -> None: + project = self._agent.get_active_project() + if project is None: + raise ValueError("No active project") + + project.memories_manager.delete_memory(request_delete_memory.memory_name) + + self._agent.execute_task(run, logged=True, name="DeleteMemory") + + def _get_serena_config(self) -> ResponseGetSerenaConfig: + config_path = self._agent.serena_config.config_file_path + if config_path is None or not os.path.exists(config_path): + raise ValueError("Serena config file not found") + + with open(config_path, encoding="utf-8") as f: + content = f.read() + + return ResponseGetSerenaConfig(content=content) + + def _save_serena_config(self, request_save_config: RequestSaveSerenaConfig) -> None: + def run() -> None: + config_path = self._agent.serena_config.config_file_path + if config_path is None: + raise ValueError("Serena config file path not set") + + with open(config_path, "w", encoding="utf-8") as f: + f.write(request_save_config.content) + + self._agent.execute_task(run, logged=True, name="SaveSerenaConfig") + + def _add_language(self, request_add_language: RequestAddLanguage) -> None: + from solidlsp.ls_config import Language + + try: + language = Language(request_add_language.language) + except ValueError: + raise ValueError(f"Invalid language: {request_add_language.language}") + # add_language is already thread-safe + self._agent.add_language(language) + + def _remove_language(self, request_remove_language: RequestRemoveLanguage) -> None: + from solidlsp.ls_config import Language + + try: + language = Language(request_remove_language.language) + except ValueError: + raise ValueError(f"Invalid language: {request_remove_language.language}") + # remove_language is already thread-safe + self._agent.remove_language(language) + @staticmethod def _find_first_free_port(start_port: int) -> int: port = start_port diff --git a/src/serena/gui_log_viewer.py b/src/serena/gui_log_viewer.py index f46d8c30..c9a43e01 100644 --- a/src/serena/gui_log_viewer.py +++ b/src/serena/gui_log_viewer.py @@ -55,6 +55,7 @@ class GuiLogViewer: self.message_queue = queue.Queue() self.running = False self.log_thread = None + self.menubar: tk.Menu | None = None self.tool_names = [] # List to store tool names for highlighting # Define colors for different log levels @@ -98,6 +99,17 @@ class GuiLogViewer: """ self.tool_names = tool_names + def set_dashboard_url(self, url: str) -> None: + def copy_url(): + self.root.clipboard_clear() + self.root.clipboard_append(url) + log.info(f"Copied dashboard URL to clipboard: {url}") + + if self.menubar is not None: + dashboard_menu = tk.Menu(self.menubar, tearoff=0) + dashboard_menu.add_command(label="Copy URL", command=copy_url) # type: ignore + self.menubar.add_cascade(label="Dashboard", menu=dashboard_menu) + def add_log(self, message): """ Add a log message to the viewer. @@ -287,11 +299,11 @@ class GuiLogViewer: # Create menu bar if self.mode == "dashboard": - menubar = tk.Menu(self.root) - server_menu = tk.Menu(menubar, tearoff=0) + self.menubar = tk.Menu(self.root) + server_menu = tk.Menu(self.menubar, tearoff=0) server_menu.add_command(label="Shutdown", command=self._shutdown_server) # type: ignore - menubar.add_cascade(label="Server", menu=server_menu) - self.root.config(menu=menubar) + self.menubar.add_cascade(label="Server", menu=server_menu) + self.root.config(menu=self.menubar) # Configure icons icon_16 = tk.PhotoImage(file=dashboard_path / "serena-icon-16.png") diff --git a/src/serena/ls_manager.py b/src/serena/ls_manager.py new file mode 100644 index 00000000..8f3681bf --- /dev/null +++ b/src/serena/ls_manager.py @@ -0,0 +1,213 @@ +import logging +import threading +from collections.abc import Iterator + +from sensai.util.logging import LogTime + +from serena.config.serena_config import SerenaPaths +from serena.constants import SERENA_MANAGED_DIR_NAME +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language, LanguageServerConfig +from solidlsp.settings import SolidLSPSettings + +log = logging.getLogger(__name__) + + +class LanguageServerFactory: + def __init__( + self, + project_root: str, + encoding: str, + ignored_patterns: list[str], + ls_timeout: float | None = None, + ls_specific_settings: dict | None = None, + trace_lsp_communication: bool = False, + ): + self.project_root = project_root + self.encoding = encoding + self.ignored_patterns = ignored_patterns + self.ls_timeout = ls_timeout + self.ls_specific_settings = ls_specific_settings + self.trace_lsp_communication = trace_lsp_communication + + def create_language_server(self, language: Language) -> SolidLanguageServer: + ls_config = LanguageServerConfig( + code_language=language, + ignored_paths=self.ignored_patterns, + trace_lsp_communication=self.trace_lsp_communication, + encoding=self.encoding, + ) + + log.info(f"Creating language server instance for {self.project_root}, language={language}.") + return SolidLanguageServer.create( + ls_config, + self.project_root, + timeout=self.ls_timeout, + solidlsp_settings=SolidLSPSettings( + solidlsp_dir=SerenaPaths().serena_user_home_dir, + project_data_relative_path=SERENA_MANAGED_DIR_NAME, + ls_specific_settings=self.ls_specific_settings or {}, + ), + ) + + +class LanguageServerManager: + """ + Manages one or more language servers for a project. + """ + + def __init__( + self, + language_servers: dict[Language, SolidLanguageServer], + language_server_factory: LanguageServerFactory | None = None, + ) -> None: + """ + :param language_servers: a mapping from language to language server; the servers are assumed to be already started. + The first server in the iteration order is used as the default server. + All servers are assumed to serve the same project root. + :param language_server_factory: factory for language server creation; if None, dynamic (re)creation of language servers + is not supported + """ + self._language_servers = language_servers + self._language_server_factory = language_server_factory + self._default_language_server = next(iter(language_servers.values())) + self._root_path = self._default_language_server.repository_root_path + + @staticmethod + def from_languages(languages: list[Language], factory: LanguageServerFactory) -> "LanguageServerManager": + """ + Creates a manager with language servers for the given languages using the given factory. + The language servers are started in parallel threads. + + :param languages: the languages for which to spawn language servers + :param factory: the factory for language server creation + :return: the instance + """ + language_servers: dict[Language, SolidLanguageServer] = {} + threads = [] + exceptions = {} + lock = threading.Lock() + + def start_language_server(language: Language) -> None: + try: + with LogTime(f"Language server startup (language={language.value})"): + language_server = factory.create_language_server(language) + language_server.start() + if not language_server.is_running(): + raise RuntimeError(f"Failed to start the language server for language {language.value}") + with lock: + language_servers[language] = language_server + except Exception as e: + log.error(f"Error starting language server for language {language.value}: {e}", exc_info=e) + with lock: + exceptions[language] = e + + # start language servers in parallel threads + for language in languages: + thread = threading.Thread(target=start_language_server, args=(language,), name="StartLS:" + language.value) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + + # If any server failed to start up, raise an exception and stop all started language servers + if exceptions: + for ls in language_servers.values(): + ls.stop() + failure_messages = "\n".join([f"{lang.value}: {e}" for lang, e in exceptions.items()]) + raise Exception(f"Failed to start language servers:\n{failure_messages}") + + return LanguageServerManager(language_servers, factory) + + def get_root_path(self) -> str: + return self._root_path + + def _ensure_functional_ls(self, ls: SolidLanguageServer) -> SolidLanguageServer: + if not ls.is_running(): + log.warning(f"Language server for language {ls.language} is not running; restarting ...") + ls = self.restart_language_server(ls.language) + return ls + + def get_language_server(self, relative_path: str) -> SolidLanguageServer: + ls: SolidLanguageServer | None = None + if len(self._language_servers) > 1: + for candidate in self._language_servers.values(): + if not candidate.is_ignored_path(relative_path, ignore_unsupported_files=True): + ls = candidate + break + if ls is None: + ls = self._default_language_server + return self._ensure_functional_ls(ls) + + def _create_and_start_language_server(self, language: Language) -> SolidLanguageServer: + if self._language_server_factory is None: + raise ValueError(f"No language server factory available to create language server for {language}") + language_server = self._language_server_factory.create_language_server(language) + language_server.start() + self._language_servers[language] = language_server + return language_server + + def restart_language_server(self, language: Language) -> SolidLanguageServer: + """ + Forces recreation and restart of the language server for the given language. + It is assumed that the language server for the given language is no longer running. + + :param language: the language + :return: the newly created language server + """ + if language not in self._language_servers: + raise ValueError(f"No language server for language {language.value} present; cannot restart") + return self._create_and_start_language_server(language) + + def add_language_server(self, language: Language) -> SolidLanguageServer: + """ + Dynamically adds a new language server for the given language. + + :param language: the language + :param factory: the factory to create the language server + :return: the newly created language server + """ + if language in self._language_servers: + raise ValueError(f"Language server for language {language.value} already present") + return self._create_and_start_language_server(language) + + def remove_language_server(self, language: Language, save_cache: bool = False) -> None: + """ + Removes the language server for the given language, stopping it if it is running. + + :param language: the language + """ + if language not in self._language_servers: + raise ValueError(f"No language server for language {language.value} present; cannot remove") + ls = self._language_servers.pop(language) + self._stop_language_server(ls, save_cache=save_cache) + + @staticmethod + def _stop_language_server(ls: SolidLanguageServer, save_cache: bool = False, timeout: float = 2.0) -> None: + if ls.is_running(): + if save_cache: + ls.save_cache() + log.info(f"Stopping language server for language {ls.language} ...") + ls.stop(shutdown_timeout=timeout) + + def iter_language_servers(self) -> Iterator[SolidLanguageServer]: + for ls in self._language_servers.values(): + yield self._ensure_functional_ls(ls) + + def stop_all(self, save_cache: bool = False, timeout: float = 2.0) -> None: + """ + Stops all managed language servers. + + :param save_cache: whether to save the cache before stopping + :param timeout: timeout for shutdown of each language server + """ + for ls in self.iter_language_servers(): + self._stop_language_server(ls, save_cache=save_cache, timeout=timeout) + + def save_all_caches(self) -> None: + """ + Saves the caches of all managed language servers. + """ + for ls in self.iter_language_servers(): + if ls.is_running(): + ls.save_cache() diff --git a/src/serena/project.py b/src/serena/project.py index 966fc8c7..968ad545 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -1,27 +1,66 @@ +import json import logging import os from pathlib import Path from typing import Any import pathspec +from sensai.util.string import ToStringMixin -from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig -from serena.constants import SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME +from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig, get_serena_managed_in_project_dir +from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME +from serena.ls_manager import LanguageServerFactory, LanguageServerManager from serena.text_utils import MatchedConsecutiveLines, search_files from serena.util.file_system import GitignoreParser, match_path +from serena.util.general import save_yaml from solidlsp import SolidLanguageServer -from solidlsp.ls_config import Language, LanguageServerConfig -from solidlsp.ls_logger import LanguageServerLogger -from solidlsp.settings import SolidLSPSettings +from solidlsp.ls_config import Language +from solidlsp.ls_utils import FileUtils log = logging.getLogger(__name__) -class Project: +class MemoriesManager: + def __init__(self, project_root: str): + self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" + self._memory_dir.mkdir(parents=True, exist_ok=True) + self._encoding = SERENA_FILE_ENCODING + + def get_memory_file_path(self, name: str) -> Path: + # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not. + name = name.replace(".md", "") + filename = f"{name}.md" + return self._memory_dir / filename + + def load_memory(self, name: str) -> str: + memory_file_path = self.get_memory_file_path(name) + if not memory_file_path.exists(): + return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it." + with open(memory_file_path, encoding=self._encoding) as f: + return f.read() + + def save_memory(self, name: str, content: str) -> str: + memory_file_path = self.get_memory_file_path(name) + with open(memory_file_path, "w", encoding=self._encoding) as f: + f.write(content) + return f"Memory {name} written." + + def list_memories(self) -> list[str]: + return [f.name.replace(".md", "") for f in self._memory_dir.iterdir() if f.is_file()] + + def delete_memory(self, name: str) -> str: + memory_file_path = self.get_memory_file_path(name) + memory_file_path.unlink() + return f"Memory {name} deleted." + + +class Project(ToStringMixin): def __init__(self, project_root: str, project_config: ProjectConfig, is_newly_created: bool = False): self.project_root = project_root self.project_config = project_config - self.is_newly_created = is_newly_created + self.memories_manager = MemoriesManager(project_root) + self.language_server_manager: LanguageServerManager | None = None + self._is_newly_created = is_newly_created # create .gitignore file in the project's Serena data folder if not yet present serena_data_gitignore_path = os.path.join(self.path_to_serena_data_folder(), ".gitignore") @@ -32,7 +71,7 @@ class Project: f.write(f"/{SolidLanguageServer.CACHE_FOLDER_NAME}\n") # gather ignored paths from the project configuration and gitignore files - ignored_patterns = project_config.ignored_paths + ignored_patterns = list(project_config.ignored_paths) if len(ignored_patterns) > 0: log.info(f"Using {len(ignored_patterns)} ignored paths from the explicit project configuration.") log.debug(f"Ignored paths: {ignored_patterns}") @@ -53,14 +92,16 @@ class Project: log.debug(f"Processing {len(processed_patterns)} ignored paths") self._ignore_spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, processed_patterns) + def _tostring_includes(self) -> list[str]: + return [] + + def _tostring_additional_entries(self) -> dict[str, Any]: + return {"root": self.project_root, "name": self.project_name} + @property def project_name(self) -> str: return self.project_config.project_name - @property - def language(self) -> Language: - return self.project_config.language - @classmethod def load(cls, project_root: str | Path, autogenerate: bool = True) -> "Project": project_root = Path(project_root).resolve() @@ -69,12 +110,42 @@ class Project: project_config = ProjectConfig.load(project_root, autogenerate=autogenerate) return Project(project_root=str(project_root), project_config=project_config) + def save_config(self) -> None: + """ + Saves the current project configuration to disk. + """ + config_path = os.path.join(self.project_root, self.project_config.rel_path_to_project_yml()) + log.info("Saving updated project configuration to %s", config_path) + config_with_comments = ProjectConfig.load_commented_map(config_path) + config_with_comments.update(self.project_config.to_yaml_dict()) + save_yaml(config_path, config_with_comments, preserve_comments=True) + def path_to_serena_data_folder(self) -> str: return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME) def path_to_project_yml(self) -> str: return os.path.join(self.project_root, self.project_config.rel_path_to_project_yml()) + def get_activation_message(self) -> str: + """ + :return: a message providing information about the project upon activation (e.g. programming language, memories, initial prompt) + """ + if self._is_newly_created: + msg = f"Created and activated a new project with name '{self.project_name}' at {self.project_root}. " + else: + msg = f"The project with name '{self.project_name}' at {self.project_root} is activated." + languages_str = ", ".join([lang.value for lang in self.project_config.languages]) + msg += f"\nProgramming languages: {languages_str}; file encoding: {self.project_config.encoding}" + memories = self.memories_manager.list_memories() + if memories: + msg += ( + f"\nAvailable project memories: {json.dumps(memories)}\n" + + "Use the `read_memory` tool to read these memories later if they are relevant to the task." + ) + if self.project_config.initial_prompt: + msg += f"\nAdditional project-specific instructions:\n {self.project_config.initial_prompt}" + return msg + def read_file(self, relative_path: str) -> str: """ Reads a file relative to the project root. @@ -83,9 +154,7 @@ class Project: :return: the content of the file """ abs_path = Path(self.project_root) / relative_path - if not abs_path.exists(): - raise FileNotFoundError(f"File not found: {abs_path}") - return abs_path.read_text(encoding=self.project_config.encoding) + return FileUtils.read_file(str(abs_path), self.project_config.encoding) def get_ignore_spec(self) -> pathspec.PathSpec: """ @@ -105,6 +174,12 @@ class Project: :return: whether the path should be ignored """ + # special case, never ignore the project root itself + # If the user ignores hidden files, "." might match against the corresponding PathSpec pattern. + # The empty string also points to the project root and should never be ignored. + if str(relative_path) in [".", ""]: + return False + abs_path = os.path.join(self.project_root, relative_path) if not os.path.exists(abs_path): raise FileNotFoundError(f"File {abs_path} not found, the ignore check cannot be performed") @@ -112,8 +187,13 @@ class Project: # Check file extension if it's a file is_file = os.path.isfile(abs_path) if is_file and ignore_non_source_files: - fn_matcher = self.language.get_source_fn_matcher() - if not fn_matcher.is_relevant_filename(abs_path): + is_file_in_supported_language = False + for language in self.project_config.languages: + fn_matcher = language.get_source_fn_matcher() + if fn_matcher.is_relevant_filename(abs_path): + is_file_in_supported_language = True + break + if not is_file_in_supported_language: return True # Create normalized path for consistent handling @@ -249,6 +329,7 @@ class Project: relative_file_paths, pattern, root_path=self.project_root, + file_reader=self.read_file, context_lines_before=context_lines_before, context_lines_after=context_lines_after, paths_include_glob=paths_include_glob, @@ -277,42 +358,87 @@ class Project: source_file_path=relative_file_path, ) - def create_language_server( + def create_language_server_manager( self, log_level: int = logging.INFO, ls_timeout: float | None = DEFAULT_TOOL_TIMEOUT - 5, trace_lsp_communication: bool = False, ls_specific_settings: dict[Language, Any] | None = None, - ) -> SolidLanguageServer: + ) -> LanguageServerManager: """ - Create a language server for a project. Note that you will have to start it - before performing any LS operations. + Creates the language server manager for the project, starting one language server per configured programming language. - :param project: either a path to the project root or a ProjectConfig instance. - If no project.yml is found, the default project configuration will be used. :param log_level: the log level for the language server :param ls_timeout: the timeout for the language server :param trace_lsp_communication: whether to trace LSP communication :param ls_specific_settings: optional LS specific configuration of the language server, see docstrings in the inits of subclasses of SolidLanguageServer to see what values may be passed. - :return: the language server + :return: the language server manager, which is also stored in the project instance """ - ls_config = LanguageServerConfig( - code_language=self.language, - ignored_paths=self._ignored_patterns, + # if there is an existing instance, stop its language servers first + if self.language_server_manager is not None: + log.info("Stopping existing language server manager ...") + self.language_server_manager.stop_all() + self.language_server_manager = None + + log.info(f"Creating language server manager for {self.project_root}") + factory = LanguageServerFactory( + project_root=self.project_root, + encoding=self.project_config.encoding, + ignored_patterns=self._ignored_patterns, + ls_timeout=ls_timeout, + ls_specific_settings=ls_specific_settings, trace_lsp_communication=trace_lsp_communication, ) - ls_logger = LanguageServerLogger(log_level=log_level) + self.language_server_manager = LanguageServerManager.from_languages(self.project_config.languages, factory) + return self.language_server_manager - log.info(f"Creating language server instance for {self.project_root}.") - return SolidLanguageServer.create( - ls_config, - ls_logger, - self.project_root, - timeout=ls_timeout, - solidlsp_settings=SolidLSPSettings( - solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME, - project_data_relative_path=SERENA_MANAGED_DIR_NAME, - ls_specific_settings=ls_specific_settings or {}, - ), - ) + def add_language(self, language: Language) -> None: + """ + Adds a new programming language to the project configuration, starting the corresponding + language server instance if the LS manager is active. + The project configuration is saved to disk after adding the language. + + :param language: the programming language to add + """ + if language in self.project_config.languages: + log.info(f"Language {language.value} is already present in the project configuration.") + return + + # start the language server (if the LS manager is active) + if self.language_server_manager is None: + log.info("Language server manager is not active; skipping language server startup for the new language.") + else: + log.info("Adding and starting the language server for new language %s ...", language.value) + self.language_server_manager.add_language_server(language) + + # update the project configuration + self.project_config.languages.append(language) + self.save_config() + + def remove_language(self, language: Language) -> None: + """ + Removes a programming language from the project configuration, stopping the corresponding + language server instance if the LS manager is active. + The project configuration is saved to disk after removing the language. + + :param language: the programming language to remove + """ + if language not in self.project_config.languages: + log.info(f"Language {language.value} is not present in the project configuration.") + return + # update the project configuration + self.project_config.languages.remove(language) + self.save_config() + + # stop the language server (if the LS manager is active) + if self.language_server_manager is None: + log.info("Language server manager is not active; skipping language server shutdown for the removed language.") + else: + log.info("Removing and stopping the language server for language %s ...", language.value) + self.language_server_manager.remove_language_server(language) + + def shutdown(self, timeout: float = 2.0) -> None: + if self.language_server_manager is not None: + self.language_server_manager.stop_all(save_cache=True, timeout=timeout) + self.language_server_manager = None diff --git a/src/serena/prompt_factory.py b/src/serena/prompt_factory.py index 40f51e7a..31be6f49 100644 --- a/src/serena/prompt_factory.py +++ b/src/serena/prompt_factory.py @@ -1,6 +1,7 @@ import os -from serena.constants import PROMPT_TEMPLATES_DIR_IN_USER_HOME, PROMPT_TEMPLATES_DIR_INTERNAL +from serena.config.serena_config import SerenaPaths +from serena.constants import PROMPT_TEMPLATES_DIR_INTERNAL from serena.generated.generated_prompt_factory import PromptFactory @@ -10,5 +11,6 @@ class SerenaPromptFactory(PromptFactory): """ def __init__(self) -> None: - os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True) - super().__init__(prompts_dir=[PROMPT_TEMPLATES_DIR_IN_USER_HOME, PROMPT_TEMPLATES_DIR_INTERNAL]) + user_templates_dir = SerenaPaths().user_prompt_templates_dir + os.makedirs(user_templates_dir, exist_ok=True) + super().__init__(prompts_dir=[user_templates_dir, PROMPT_TEMPLATES_DIR_INTERNAL]) diff --git a/src/serena/resources/config/contexts/agent.yml b/src/serena/resources/config/contexts/agent.yml index e2984283..a3b324f0 100644 --- a/src/serena/resources/config/contexts/agent.yml +++ b/src/serena/resources/config/contexts/agent.yml @@ -1,7 +1,6 @@ -description: All tools except InitialInstructionsTool for agent context +description: Agent context where the system prompt (initial instructions) are provided at startup prompt: | - You are running in agent context where the system prompt is provided externally. You should use symbolic - tools when possible for code understanding and modification. + You are running in an agent context. excluded_tools: - initial_instructions diff --git a/src/serena/resources/config/contexts/chatgpt.yml b/src/serena/resources/config/contexts/chatgpt.yml index c758b2c3..ae1c3db6 100644 --- a/src/serena/resources/config/contexts/chatgpt.yml +++ b/src/serena/resources/config/contexts/chatgpt.yml @@ -1,4 +1,4 @@ -description: A configuration specific for chatgpt, which has a limit of 30 tools and requires short descriptions. +description: A configuration specific for ChatGPT, which has a limit of 30 tools and requires short descriptions. prompt: | You are running in desktop app context where the tools give you access to the code base as well as some access to the file system, if configured. You interact with the user through a chat interface that is separated @@ -16,11 +16,11 @@ included_optional_tools: tool_description_overrides: find_symbol: | - Retrieves symbols matching `name_path` in a file. - Use `depth > 0` to include children. `name_path` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar" - replace_regex: | - Replaces text using regular expressions. Preferred for smaller edits where symbol-level tools aren't appropriate. - Use wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. + Retrieves symbols matching `name_path_pattern` in a file. + Use `depth > 0` to include children. `name_path_pattern` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar" + replace_content: | + Replaces content in files. Preferred for smaller edits where symbol-level tools aren't appropriate. + Use mode "regex" with wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. Essential for multi-line replacements. search_for_pattern: | Flexible pattern search across codebase. Prefer symbolic operations when possible. diff --git a/src/serena/resources/config/contexts/claude-code.yml b/src/serena/resources/config/contexts/claude-code.yml new file mode 100644 index 00000000..118ed8b4 --- /dev/null +++ b/src/serena/resources/config/contexts/claude-code.yml @@ -0,0 +1,26 @@ +description: Claude Code (CLI agent where file operations, basic edits, etc. are already covered; single project mode) +prompt: | + You are running in a CLI coding agent context where file operations, basic (line-based) edits and reads + as well as shell commands are handled by your own, internal tools. + + If Serena's tools can be used to achieve your task, you should prioritize them. + In particular, it is important that you avoid reading entire source code files unless it is strictly necessary! + Instead, for exploring and reading code in a token-efficient manner, use Serena's overview and symbolic search tools. + For non-code files or for reads where you don't know the symbol's name path, you can use the pattern search tool. + +excluded_tools: + - create_text_file + - read_file + - execute_shell_command + - prepare_for_new_conversation + - replace_content + +tool_description_overrides: {} + +# whether to assume that Serena shall only work on a single project in this context (provided that a project is given +# when Serena is started). +# If set to true and a project is provided at startup, the set of tools is limited to those required by the project's +# concrete configuration, and other tools are excluded completely, allowing the set of tools to be minimal. +# Tools explicitly disabled by the project will not be available at all. +# The `activate_project` tool is always disabled in this case, as project switching cannot be allowed. +single_project: true diff --git a/src/serena/resources/config/contexts/codex.yml b/src/serena/resources/config/contexts/codex.yml index 8ca5433f..a492e8f9 100644 --- a/src/serena/resources/config/contexts/codex.yml +++ b/src/serena/resources/config/contexts/codex.yml @@ -1,27 +1,20 @@ -description: Non-symbolic editing tools and general shell tool are excluded +description: Codex Non-symbolic editing tools and general shell tool are excluded prompt: | - You are running in IDE assistant context where file operations, basic (line-based) edits and reads, - and shell commands are handled by your own, internal tools. - The initial instructions and the current config inform you on which tools are available to you, - and how to use them. - Don't attempt to use any excluded tools, instead rely on your own internal tools - for achieving the basic file or shell operations. + You are running in the Codex IDE assistant mode, where file operations, basic (line-based) edits and reads + as well as shell commands are handled by your own, internal tools. + Don't attempt to use any excluded tools; instead, rely on your own internal tools for basic file or shell operations. - If serena's tools can be used for achieving your task, - you should prioritize them. In particular, it is important that you avoid reading entire source code files, - unless it is strictly necessary! Instead, for exploring and reading code in a token-efficient manner, - you should use serena's overview and symbolic search tools. The call of the read_file tool on an entire source code - file should only happen in exceptional cases, usually you should first explore the file (by itself or as part of exploring - the directory containing it) using the symbol_overview tool, and then make targeted reads using find_symbol and other symbolic tools. - For non-code files or for reads where you don't know the symbol's name path you can use the patterns searching tool, - using the read_file as a last resort. + If Serena's tools can be used to achieve your task, you should prioritize them. + In particular, it is important that you avoid reading entire source code files unless it is strictly necessary! + Instead, for exploring and reading code in a token-efficient manner, use Serena's overview and symbolic search tools. + For non-code files or for reads where you don't know the symbol's name path, you can use the pattern search tool. excluded_tools: - create_text_file - read_file - execute_shell_command - prepare_for_new_conversation - - replace_regex + - replace_content tool_description_overrides: {} diff --git a/src/serena/resources/config/contexts/context.template.yml b/src/serena/resources/config/contexts/context.template.yml index c01a1177..f9873413 100644 --- a/src/serena/resources/config/contexts/context.template.yml +++ b/src/serena/resources/config/contexts/context.template.yml @@ -8,4 +8,11 @@ included_optional_tools: [] # mapping of tool names to an override of their descriptions (the default description is the docstring of the Tool's apply method). # Sometimes, tool descriptions are too long (e.g., for ChatGPT), or users may want to override them for another reason. -tool_description_overrides: {} \ No newline at end of file +tool_description_overrides: {} + +# whether to assume that Serena shall only work on a single project in this context (provided that a project is given +# when Serena is started). +# If set to true and a project is provided at startup, the set of tools is limited to those required by the project's +# concrete configuration, and other tools are excluded completely, allowing the set of tools to be minimal. +# The `activate_project` tool will, therefore, be disabled in this case, as project switching is not allowed. +single_project: false \ No newline at end of file diff --git a/src/serena/resources/config/contexts/desktop-app.yml b/src/serena/resources/config/contexts/desktop-app.yml index c3adcfb4..8d49cec5 100644 --- a/src/serena/resources/config/contexts/desktop-app.yml +++ b/src/serena/resources/config/contexts/desktop-app.yml @@ -1,15 +1,13 @@ -description: All tools included for desktop app context +description: Desktop application context (chat application detached from code) where Serena's full toolset is provided prompt: | - You are running in desktop app context where the tools give you access to the code base as well as some - access to the file system, if configured. You interact with the user through a chat interface that is separated - from the code base. As a consequence, if you are in interactive mode, your communication with the user should + You are running in a desktop application context. + Serena's tools give you access to the code base as well as some access to the file system (if enabled). + You interact with the user through a chat interface that is separated from the code base. + As a consequence, if you are in interactive mode, your communication with the user should involve high-level thinking and planning as well as some summarization of any code edits that you make. - For viewing the code edits the user will view them in a separate code editor window, and the back-and-forth - between the chat and the code editor should be minimized as well as facilitated by you. - If complex changes have been made, advise the user on how to review them in the code editor. - If complex relationships that the user asked for should be visualized or explained, consider creating - a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering - options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. + To view the code edits you make, the user will have switch to a separate application. + To illustrate complex relationships, consider creating diagrams in addition to your text-based communication + (depending on the options for text, html, mermaid diagrams, etc. that you are provided with in your initial instructions). excluded_tools: [] included_optional_tools: - switch_modes diff --git a/src/serena/resources/config/contexts/ide-assistant.yml b/src/serena/resources/config/contexts/ide-assistant.yml deleted file mode 100644 index e9332c3e..00000000 --- a/src/serena/resources/config/contexts/ide-assistant.yml +++ /dev/null @@ -1,27 +0,0 @@ -description: Non-symbolic editing tools and general shell tool are excluded -prompt: | - You are running in IDE assistant context where file operations, basic (line-based) edits and reads, - and shell commands are handled by your own, internal tools. - The initial instructions and the current config inform you on which tools are available to you, - and how to use them. - Don't attempt to use any excluded tools, instead rely on your own internal tools - for achieving the basic file or shell operations. - - If serena's tools can be used for achieving your task, - you should prioritize them. In particular, it is important that you avoid reading entire source code files, - unless it is strictly necessary! Instead, for exploring and reading code in a token-efficient manner, - you should use serena's overview and symbolic search tools. - Before reading a full file, it is usually best to first explore the file using the symbol_overview tool, - and then make targeted reads using find_symbol and other symbolic tools. - For non-code files or for reads where you don't know the symbol's name path, you can use the pattern searching tool, - and read full files only as a last resort. - -excluded_tools: - - create_text_file - - read_file - - execute_shell_command - - prepare_for_new_conversation - - replace_regex - -tool_description_overrides: {} - diff --git a/src/serena/resources/config/contexts/ide.yml b/src/serena/resources/config/contexts/ide.yml new file mode 100644 index 00000000..4a917a56 --- /dev/null +++ b/src/serena/resources/config/contexts/ide.yml @@ -0,0 +1,25 @@ +description: Generic IDE coding agent context (basic file operations and shell operations assumed to be covered; single project mode) +prompt: | + You are running in an IDE assistant context where file operations, basic (line-based) edits and reads, + and shell commands are handled by your own, internal tools. + + If Serena's tools can be used to achieve your task, you should prioritize them. + In particular, it is important that you avoid reading entire source code files unless it is strictly necessary! + Instead, for exploring and reading code in a token-efficient manner, use Serena's overview and symbolic search tools. + For non-code files or for reads where you don't know the symbol's name path, you can use the pattern search tool. + +excluded_tools: + - create_text_file + - read_file + - execute_shell_command + - prepare_for_new_conversation + +tool_description_overrides: {} + +# whether to assume that Serena shall only work on a single project in this context (provided that a project is given +# when Serena is started). +# If set to true and a project is provided at startup, the set of tools is limited to those required by the project's +# concrete configuration, and other tools are excluded completely, allowing the set of tools to be minimal. +# Tools explicitly disabled by the project will not be available at all. +# The `activate_project` tool is always disabled in this case, as project switching cannot be allowed. +single_project: true diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index b82603b1..4de188b2 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -1,110 +1,35 @@ description: All tools, with detailed instructions for code editing prompt: | - You are operating in editing mode. You can edit files with the provided tools - to implement the requested changes to the code base while adhering to the project's code style and patterns. + You are operating in editing mode. You can edit files with the provided tools. + You adhere to the project's code style and patterns. + Use symbolic editing tools whenever possible for precise code modifications. - If no editing task has yet been provided, wait for the user to provide one. + If no explicit editing task has yet been provided, wait for the user to provide one. Do not be overly eager. When writing new code, think about where it belongs best. Don't generate new files if you don't plan on actually - integrating them into the codebase, instead use the editing tools to insert the code directly into the existing files in that case. + properly integrating them into the codebase. - You have two main approaches for editing code - editing by regex and editing by symbol. + You have two main approaches for editing code: (a) editing at the symbol level and (b) file-based editing. The symbol-based approach is appropriate if you need to adjust an entire symbol, e.g. a method, a class, a function, etc. - But it is not appropriate if you need to adjust just a few lines of code within a symbol, for that you should - use the regex-based approach that is described below. + It is not appropriate if you need to adjust just a few lines of code within a larger symbol. - Let us first discuss the symbol-based approach. - Symbols are identified by their name path and relative file path, see the description of the `find_symbol` tool for more details - on how the `name_path` matches symbols. - You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file, - or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible - while still solving your task, meaning you only read the bodies when you need to, and after you have found the symbol you want to edit. - Before calling symbolic reading tools, you should have a basic understanding of the repository structure that you can get from memories - or by using the `list_dir` and `find_file` tools (or similar). - For example, if you are working with python code and already know that you need to read the body of the constructor of the class Foo, you can directly - use `find_symbol` with the name path `Foo/__init__` and `include_body=True`. If you don't know yet which methods in `Foo` you need to read or edit, - you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding - to read the desired methods with `include_body=True`. - In particular, keep in mind the description of the `replace_symbol_body` tool. If you want to add some new code at the end of the file, you should - use the `insert_after_symbol` tool with the last top-level symbol in the file. If you want to add an import, often a good strategy is to use - `insert_before_symbol` with the first top-level symbol in the file. - You can understand relationships between symbols by using the `find_referencing_symbols` tool. If not explicitly requested otherwise by a user, - you make sure that when you edit a symbol, it is either done in a backward-compatible way, or you find and adjust the references as needed. - The `find_referencing_symbols` tool will give you code snippets around the references, as well as symbolic information. - You will generally be able to use the info from the snippets and the regex-based approach to adjust the references as well. - You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error. + **Symbolic editing** + Use symbolic retrieval tools to identify the symbols you need to edit. + If you need to replace the definition of a symbol, use the `replace_symbol_body` tool. + If you want to add some new code at the end of the file, use the `insert_after_symbol` tool with the last top-level symbol in the file. + Similarly, you can use `insert_before_symbol` with the first top-level symbol in the file to insert code at the beginning of a file. + You can understand relationships between symbols by using the `find_referencing_symbols` tool. If not explicitly requested otherwise by the user, + you make sure that when you edit a symbol, the change is either backward-compatible or you find and update all references as needed. + The `find_referencing_symbols` tool will give you code snippets around the references as well as symbolic information. + You can assume that all symbol editing tools are reliable, so you never need to verify the results if the tools return without error. - {% if 'replace_regex' in available_tools %} - Let us discuss the regex-based approach. - The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation. - This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol. - You use other tools to find the relevant content and - then use your knowledge of the codebase to write the regex, if you haven't collected enough information of this content yet. + {% if 'replace_content' in available_tools %} + **File-based editing** + The `replace_content` tool allows you to perform regex-based replacements within files (as well as simple string replacements). + This is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation, + e.g. if you need to adjust just a few lines of code within a method. You are extremely good at regex, so you never need to check whether the replacement produced the correct result. - In particular, you know what to escape and what not to escape, and you know how to use wildcards. - Also, the regex tool never adds any indentation (contrary to the symbolic editing tools), so you have to take care to add the correct indentation - when using it to insert code. - Moreover, the replacement tool will fail if it can't perform the desired replacement, and this is all the feedback you need. - Your overall goal for replacement operations is to use relatively short regexes, since I want you to minimize the number - of output tokens. For replacements of larger chunks of code, this means you intelligently make use of wildcards for the middle part - and of characteristic snippets for the before/after parts that uniquely identify the chunk. - - For small replacements, up to a single line, you follow the following rules: - - 1. If the snippet to be replaced is likely to be unique within the file, you perform the replacement by directly using the escaped version of the - original. - 2. If the snippet is probably not unique, and you want to replace all occurrences, you use the `allow_multiple_occurrences` flag. - 3. If the snippet is not unique, and you want to replace a specific occurrence, you make use of the code surrounding the snippet - to extend the regex with content before/after such that the regex will have exactly one match. - 4. You generally assume that a snippet is unique, knowing that the tool will return an error on multiple matches. You only read more file content - (for crafvarting a more specific regex) if such a failure unexpectedly occurs. - - Examples: - - 1 Small replacement - You have read code like - - ```python - ... - x = linear(x) - x = relu(x) - return x - ... - ``` - - and you want to replace `x = relu(x)` with `x = gelu(x)`. - You first try `replace_regex()` with the regex `x = relu\(x\)` and the replacement `x = gelu(x)`. - If this fails due to multiple matches, you will try `(linear\(x\)\s*)x = relu\(x\)(\s*return)` with the replacement `\1x = gelu(x)\2`. - - 2 Larger replacement - - You have read code like - - ```python - def my_func(): - ... - # a comment before the snippet - x = add_fifteen(x) - # beginning of long section within my_func - .... - # end of long section - call_subroutine(z) - call_second_subroutine(z) - ``` - and you want to replace the code starting with `x = add_fifteen(x)` until (including) `call_subroutine(z)`, but not `call_second_subroutine(z)`. - Initially, you assume that the the beginning and end of the chunk uniquely determine it within the file. - Therefore, you perform the replacement by using the regex `x = add_fifteen\(x\)\s*.*?call_subroutine\(z\)` - and the replacement being the new code you want to insert. - - If this fails due to multiple matches, you will try to extend the regex with the content before/after the snippet and match groups. - The matching regex becomes: - `(before the snippet\s*)x = add_fifteen\(x\)\s*.*?call_subroutine\(z\)` - and the replacement includes the group as (schematically): - `\1` - - Generally, I remind you that you rely on the regex tool with providing you the correct feedback, no need for more verification! - - IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE LONG REGEXES WITHOUT USING WILDCARDS INSTEAD! + In particular, you know how to use wildcards effectively in order to avoid specifying the full original text to be replaced! {% endif %} excluded_tools: - replace_lines diff --git a/src/serena/resources/config/modes/interactive.yml b/src/serena/resources/config/modes/interactive.yml index 4da123d7..cdae4e99 100644 --- a/src/serena/resources/config/modes/interactive.yml +++ b/src/serena/resources/config/modes/interactive.yml @@ -6,6 +6,6 @@ prompt: | Break down complex tasks into smaller steps and explain your thinking at each stage. When you're uncertain about a decision, present options to the user and ask for guidance rather than making assumptions. - Focus on providing informative results for intermediate steps so the user can follow along with your progress and + Focus on providing informative results for intermediate steps, such that the user can follow along with your progress and provide feedback as needed. excluded_tools: [] \ No newline at end of file diff --git a/src/serena/resources/config/modes/mode.template.yml b/src/serena/resources/config/modes/mode.template.yml index 6d836ca3..d7efce0d 100644 --- a/src/serena/resources/config/modes/mode.template.yml +++ b/src/serena/resources/config/modes/mode.template.yml @@ -1,7 +1,8 @@ # See Serena's documentation for more details on concept of modes. -description: Description of the mode, not used in the code. -prompt: Prompt that will form part of the message sent to the model when activating this mode. +description: Description of the mode (meta-information only) +prompt: | + Provide a prompt that will form part of the instructions sent to the model when this mode is activated. +# tools that are to be excluded by this mode excluded_tools: [] - # several tools are excluded by default and have to be explicitly included by the user included_optional_tools: [] \ No newline at end of file diff --git a/src/serena/resources/config/modes/no-memories.yml b/src/serena/resources/config/modes/no-memories.yml new file mode 100644 index 00000000..7a522875 --- /dev/null +++ b/src/serena/resources/config/modes/no-memories.yml @@ -0,0 +1,11 @@ +description: Excludes Serena's memory tools (and onboarding tools, which rely on memory) +prompt: | + Serena's memory tools are not available and the onboarding workflow is not being applied. +excluded_tools: + - write_memory + - read_memory + - delete_memory + - edit_memory + - list_memories + - onboarding + - check_onboarding_performed diff --git a/src/serena/resources/config/modes/no-onboarding.yml b/src/serena/resources/config/modes/no-onboarding.yml index e4d15cc7..4b893eff 100644 --- a/src/serena/resources/config/modes/no-onboarding.yml +++ b/src/serena/resources/config/modes/no-onboarding.yml @@ -1,8 +1,6 @@ -description: Onboarding was already performed, exclude all onboarding tools +description: The onboarding process is not used (memories may have been created externally) prompt: | - You have already performed onboarding, meaning that memories have already been created. - Read a list of available memories using the `list_memories` tool before proceeding with the task. - You don't need to read the actual memories, just remember that they exist and that you can read them later if they are relevant for your task. + The onboarding process is not applied. excluded_tools: - onboarding - check_onboarding_performed diff --git a/src/serena/resources/config/modes/planning.yml b/src/serena/resources/config/modes/planning.yml index a3f76d79..a24d0dfb 100644 --- a/src/serena/resources/config/modes/planning.yml +++ b/src/serena/resources/config/modes/planning.yml @@ -1,8 +1,7 @@ description: Only read-only tools, focused on analysis and planning prompt: | You are operating in planning mode. Your task is to analyze code but not write any code. - The user may ask you to assist in creating a comprehensive plan, or to learn something about the codebase - - either a small aspect of it or about the whole project. + The user may ask you to assist in creating a comprehensive plan, or to learn something about the codebase. excluded_tools: - create_text_file - replace_symbol_body @@ -12,4 +11,4 @@ excluded_tools: - replace_lines - insert_at_line - execute_shell_command - - replace_regex + - replace_content diff --git a/src/serena/resources/config/prompt_templates/system_prompt.yml b/src/serena/resources/config/prompt_templates/system_prompt.yml index 9b4f7a12..13064c14 100644 --- a/src/serena/resources/config/prompt_templates/system_prompt.yml +++ b/src/serena/resources/config/prompt_templates/system_prompt.yml @@ -3,64 +3,52 @@ # is encouraged (via the tool description) to call at the beginning of the conversation. prompts: system_prompt: | - You are a professional coding agent concerned with one particular codebase. You have - access to semantic coding tools on which you rely heavily for all your work, as well as collection of memory - files containing general information about the codebase. You operate in a resource-efficient and intelligent manner, always - keeping in mind to not read or generate content that is not needed for the task at hand. + You are a professional coding agent. + You have access to semantic coding tools upon which you rely heavily for all your work. + You operate in a resource-efficient and intelligent manner, always keeping in mind to not read or generate + content that is not needed for the task at hand. - When reading code in order to answer a user question or task, you should try reading only the necessary code. Some tasks may require you to understand the architecture of large parts of the codebase, while for others, it may be enough to read a small set of symbols or a single file. - Generally, you should avoid reading entire files unless it is absolutely necessary, instead relying on - intelligent step-by-step acquisition of information. {% if 'ToolMarkerSymbolicRead' in available_markers %}However, if you already read a file, it does not make - sense to further analyse it with the symbolic tools (except for the `find_referencing_symbols` tool), - as you already have the information.{% endif %} + You avoid reading entire files unless it is absolutely necessary, instead relying on intelligent step-by-step + acquisition of information. {% if 'ToolMarkerSymbolicRead' in available_markers %}Once you have read a full file, it does not make + sense to analyse it with the symbolic read tools; you already have the information.{% endif %} - I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! - {% if 'ToolMarkerSymbolicRead' in available_markers %} - CONSIDER INSTEAD USING THE OVERVIEW TOOL AND SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! - I WILL BE EVEN MORE UPSET IF AFTER HAVING READ AN ENTIRE FILE YOU KEEP READING THE SAME CONTENT WITH THE SYMBOLIC TOOLS! - THE PURPOSE OF THE SYMBOLIC TOOLS IS TO HAVE TO READ LESS CODE, NOT READ THE SAME CONTENT MULTIPLE TIMES! - {% endif %} - - You can achieve the intelligent reading of code by using the symbolic tools for getting an overview of symbols and - the relations between them, and then only reading the bodies of symbols that are necessary to answer the question - or complete the task. + You can achieve intelligent reading of code by using the symbolic tools for getting an overview of symbols and + the relations between them, and then only reading the bodies of symbols that are necessary to complete the task at hand. You can use the standard tools like list_dir, find_file and search_for_pattern if you need to. - When tools allow it, you pass the `relative_path` parameter to restrict the search to a specific file or directory. - For some tools, `relative_path` can only be a file path, so make sure to properly read the tool descriptions. + Where appropriate, you pass the `relative_path` parameter to restrict the search to a specific file or directory. {% if 'search_for_pattern' in available_tools %} If you are unsure about a symbol's name or location{% if 'find_symbol' in available_tools %} (to the extent that substring_matching for the symbol name is not enough){% endif %}, you can use the `search_for_pattern` tool, which allows fast - and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %}This way you can first find candidates for symbols or files, + and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %} In this way, you can first find candidates for symbols or files, and then proceed with the symbolic tools.{% endif %} {% endif %} {% if 'ToolMarkerSymbolicRead' in available_markers %} - Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbol` tool for more details - on how the `name_path` matches symbols. - You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file, - or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible - while still solving your task, meaning you only read the bodies when you need to, and after you have found the symbol you want to edit. - For example, if you are working with python code and already know that you need to read the body of the constructor of the class Foo, you can directly - use `find_symbol` with the name path `Foo/__init__` and `include_body=True`. If you don't know yet which methods in `Foo` you need to read or edit, - you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding - to read the desired methods with `include_body=True` + Symbols are identified by their `name_path` and `relative_path` (see the description of the `find_symbol` tool). + You can get information about the symbols in a file by using the `get_symbols_overview` tool or use the `find_symbol` to search. + You only read the bodies of symbols when you need to (e.g. if you want to fully understand or edit it). + For example, if you are working with Python code and already know that you need to read the body of the constructor of the class Foo, you can directly + use `find_symbol` with name path pattern `Foo/__init__` and `include_body=True`. If you don't know yet which methods in `Foo` you need to read or edit, + you can use `find_symbol` with name path pattern `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding + to read the desired methods with `include_body=True`. You can understand relationships between symbols by using the `find_referencing_symbols` tool. {% endif %} {% if 'read_memory' in available_tools %} - You generally have access to memories and it may be useful for you to read them, but also only if they help you - to answer the question or complete the task. You can infer which memories are relevant to the current task by reading - the memory names and descriptions. + You generally have access to memories and it may be useful for you to read them. + You infer whether memories are relevant based on their names. {% endif %} - The context and modes of operation are described below. From them you can infer how to interact with your user - and which tasks and kinds of interactions are expected of you. + The context and modes of operation are described below. These determine how to interact with your user + and which kinds of interactions are expected of you. Context description: {{ context_system_prompt }} Modes descriptions: {% for prompt in mode_system_prompts %} - - {{ prompt }} + {{ prompt }} {% endfor %} + + You have hereby read the 'Serena Instructions Manual' and do not need to read it again. diff --git a/src/serena/resources/dashboard/dashboard.css b/src/serena/resources/dashboard/dashboard.css new file mode 100644 index 00000000..8795b895 --- /dev/null +++ b/src/serena/resources/dashboard/dashboard.css @@ -0,0 +1,1176 @@ +html { + scrollbar-gutter: stable; /* Prevent layout shift when scrollbar appears */ +} + +:root { + /* Light theme variables */ + --bg-primary: #f5f5f5; + --bg-secondary: #ffffff; + --text-primary: #000000; + --text-secondary: #333333; + --text-muted: #666666; + --border-color: #ddd; + --btn-primary: #eaa45d; + --btn-hover: #dca662; + --btn-disabled: #6c757d; + --shadow: 0 2px 4px rgba(0, 0, 0, 0.1); + --tool-highlight: #ffff00; + --tool-highlight-text: #000000; + --log-debug: #808080; + --log-info: #000000; + --log-warning: #FF8C00; + --log-error: #FF0000; + --stats-header: #f8f9fa; + --header-height: 150px; + --header-padding: 20px; + --header-gap-main: 25px; + --frame-padding: 25px; + --border-radius: 5px; +} + +[data-theme="dark"] { + /* Dark theme variables */ + --bg-primary: #1a1a1a; + --bg-secondary: #2d2d2d; + --text-primary: #ffffff; + --text-secondary: #e0e0e0; + --text-muted: #b0b0b0; + --border-color: #444; + --btn-primary: #eaa45d; + --btn-hover: #dca662; + --btn-disabled: #6c757d; + --shadow: 0 2px 4px rgba(0, 0, 0, 0.3); + --tool-highlight: #ffd700; + --tool-highlight-text: #000000; + --log-debug: #808080; + --log-info: #ffffff; + --log-warning: #FF8C00; + --log-error: #FF0000; + --stats-header: #3a3a3a; +} + +body { + font-family: 'Consolas', 'Monaco', 'Courier New', monospace; + margin: 0; + background-color: var(--bg-primary); + color: var(--text-primary); + transition: background-color 0.3s ease, color 0.3s ease; +} + +#frame { + max-width: 1600px; + margin: 0 auto; + padding: var(--frame-padding); + padding-top: 0; + min-width: 1280px; +} + +.main { + padding-top: var(--header-gap-main); +} + +.header { + top: 0; + left: 0; + right: 0; + height: var(--header-height); + background-color: var(--bg-secondary); + border-bottom: 1px solid var(--border-color); + border-bottom-left-radius: var(--border-radius); + border-bottom-right-radius: var(--border-radius); + padding: var(--header-padding); + display: flex; + justify-content: space-between; + align-items: center; + gap: 20px; + z-index: 1000; + transition: background-color 0.3s ease, border-color 0.3s ease; + min-height: 90px; + box-shadow: var(--shadow); + max-width: 1600px; + margin: 0 auto; +} + +.header-left { + display: flex; + gap: 30px; +} + +.logo-container { + order: 1; + flex-shrink: 0; +} + +.header-sponsor { + position: relative; + top: 0; + left: 0; + order: 2; + max-height: var(--header-height); +} + +.logo-container img { + height: var(--header-height); + display: block; +} + +.header-nav { + position: relative; + display: flex; + align-items: flex-end; + width: 200px; + height: 100%; + gap: 10px; + flex-shrink: 0; +} + +.menu-button { + background-color: var(--bg-secondary); + color: var(--text-primary); + border: 1px solid var(--border-color); + padding: 8px 16px; + border-radius: 4px; + cursor: pointer; + font-size: 16px; + transition: background-color 0.3s ease, border-color 0.3s ease; + display: flex; + align-items: center; + gap: 8px; +} + +.menu-button:hover { + background-color: var(--border-color); +} + +.menu-dropdown { + position: absolute; + top: calc(var(--header-height) + 10px); + right: 0; + background-color: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 4px; + box-shadow: var(--shadow); + min-width: 200px; + z-index: 999; + transition: background-color 0.3s ease, border-color 0.3s ease; +} + +.menu-dropdown a { + display: block; + padding: 12px 20px; + color: var(--text-primary); + text-decoration: none; + transition: background-color 0.3s ease; +} + +.menu-dropdown a:hover { + background-color: var(--border-color); +} + +.menu-dropdown a.active { + background-color: var(--btn-primary); + color: white; +} + +.menu-dropdown hr { + border: none; + border-top: 1px solid var(--border-color); + margin: 5px 0; +} + +.platinum-sponsor-slide { + position: absolute; + top: 0; + left: 0; + pointer-events: none; + height: 100%; + opacity: 0; + transition: opacity 0.5s ease-in-out; +} + +.platinum-sponsor-slide.active { + opacity: 1; + pointer-events: auto; +} + +.sponsor-image { + object-fit: contain; + border-radius: var(--border-radius); +} + +.sponsor-border { + border: 1px solid var(--border-color); +} + +.platinum-sponsor-slide .sponsor-image { + max-height: 100%; + object-fit: contain; +} + +.gold-sponsors-section { + margin: 0 auto; + width: 100%; + position: relative; + align-items: center; + justify-content: center; + padding: 0; +} + +.gold-banner { + position: relative; + width: 100%; + height: 60px; + overflow: hidden; +} + +.gold-sponsor-slide { + position: absolute; + opacity: 0; + transition: opacity 0.5s ease-in-out; +} + +.gold-sponsor-slide.active { + opacity: 1; + pointer-events: auto; +} + +.gold-sponsor-slide .sponsor-image { + max-width: 100%; + object-fit: contain; +} + +.page-view { + /*max-width: 1600px;*/ + margin: 0 auto; +} + +/* Overview Page Layout */ +.overview-container { + display: grid; + grid-template-columns: 1fr 400px; + gap: 20px; +} + +.overview-left { + min-width: 0; +} + +.overview-right { + min-width: 0; +} + +/* Overview Page Styles */ +.config-section, +.basic-stats-section, +.projects-section { + background-color: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: var(--border-radius); + padding: 20px; + margin-bottom: 20px; + transition: background-color 0.3s ease, border-color 0.3s ease; +} + +.config-section h2, +.basic-stats-section h2 { + margin-top: 0; + color: var(--text-secondary); +} + +/* Collapsible Headers */ +.collapsible-header { + margin-top: 0; + margin-bottom: 0; + font-size: 18px; + color: var(--text-secondary); + cursor: pointer; + user-select: none; + display: flex; + justify-content: space-between; + align-items: center; +} + +.collapsible-header:hover { + color: var(--text-primary); +} + +.toggle-icon { + transition: transform 0.3s ease; + font-size: 14px; +} + +.toggle-icon.expanded { + transform: rotate(-180deg); +} + +.collapsible-content { + margin-top: 15px; + max-height: 400px; + overflow-y: auto; +} + +.config-grid { + display: grid; + grid-template-columns: 180px 1fr; + gap: 12px; + margin-bottom: 20px; +} + +.config-label { + font-weight: bold; + color: var(--text-secondary); +} + +.config-value { + color: var(--text-primary); +} + +.config-list { + list-style: none; + padding: 0; + margin: 0; +} + +.config-list li { + padding: 4px 0; +} + +.tools-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 8px; + margin-top: 10px; +} + +.tool-item { + background-color: var(--bg-primary); + padding: 6px 10px; + border-radius: 3px; + font-size: 13px; + color: var(--text-primary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + cursor: default; +} + +/* Projects List */ +.project-item { + padding: 10px 12px; + margin: 5px 0; + border-radius: 4px; + background-color: var(--bg-primary); + border: 1px solid var(--border-color); + transition: background-color 0.2s ease; +} + +.project-item:hover { + background-color: var(--border-color); +} + +.project-item.active { + background-color: var(--btn-primary); + color: white; + border-color: var(--btn-primary); +} + +.project-name { + font-weight: bold; + margin-bottom: 4px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.project-path { + font-size: 11px; + color: var(--text-muted); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.project-item.active .project-path { + color: rgba(255, 255, 255, 0.8); +} + +/* Generic Item Styles for Tools/Modes/Contexts */ +.info-item { + padding: 8px 12px; + margin: 5px 0; + border-radius: 4px; + background-color: var(--bg-primary); + border: 1px solid var(--border-color); + transition: background-color 0.2s ease; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + cursor: default; +} + +.info-item:hover { + background-color: var(--border-color); +} + +.info-item.active { + background-color: var(--btn-primary); + color: white; + border-color: var(--btn-primary); + font-weight: bold; +} + +/* Basic Stats Styles */ +.basic-stats-section { + background-color: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 5px; + padding: 20px; + margin-bottom: 20px; + transition: background-color 0.3s ease, border-color 0.3s ease; +} + +.basic-stats-section h2 { + margin-top: 0; + color: var(--text-secondary); +} + +/* Executions Styles */ +.executions-section { + background-color: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 5px; + padding: 20px; + margin-bottom: 20px; + transition: background-color 0.3s ease, border-color 0.3s ease; +} + +.executions-section h2 { + margin-top: 0; + color: var(--text-secondary); +} + +.execution-list { + display: flex; + flex-direction: column; + gap: 8px; +} + +.execution-item { + display: flex; + align-items: center; + gap: 10px; + background-color: var(--bg-primary); + border: 1px solid var(--border-color); + border-radius: 20px; + padding: 8px 12px; + min-height: 40px; + transition: background-color 0.2s ease, border-color 0.2s ease; +} + +.execution-item.running { + border-color: var(--btn-primary); + background: linear-gradient(to right, rgba(234, 164, 93, 0.1), var(--bg-primary)); +} + +.execution-item.cancelled { + border-color: var(--text-muted); + background-color: var(--bg-primary); + opacity: 0.7; +} + +.execution-item.abandoned { + border-color: var(--log-error); + background: linear-gradient(to right, rgba(255, 0, 0, 0.1), var(--bg-primary)); +} + +.execution-spinner { + width: 16px; + height: 16px; + border: 2px solid var(--border-color); + border-top-color: var(--btn-primary); + border-radius: 50%; + animation: spin 0.7s linear infinite; + flex-shrink: 0; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} + +.execution-name { + flex: 1; + font-size: 13px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + color: var(--text-primary); +} + +.execution-meta { + font-size: 11px; + color: var(--text-muted); + flex-shrink: 0; +} + +.execution-cancel-btn { + background: none; + border: none; + color: var(--text-muted); + font-size: 16px; + cursor: pointer; + width: 24px; + height: 24px; + display: flex; + align-items: center; + justify-content: center; + border-radius: 50%; + transition: background-color 0.15s ease, color 0.15s ease; + flex-shrink: 0; +} + +.execution-cancel-btn:hover { + background-color: rgba(255, 0, 0, 0.1); + color: var(--log-error); +} + +.execution-icon { + width: 16px; + height: 16px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + font-size: 10px; + flex-shrink: 0; +} + +.execution-icon.success { + background-color: rgba(34, 197, 94, 0.2); + border: 1px solid rgba(34, 197, 94, 0.6); + color: #22c55e; +} + +.execution-icon.cancelled { + background-color: var(--border-color); + border: 1px solid var(--text-muted); + color: var(--text-muted); +} + +.execution-icon.abandoned { + background-color: rgba(255, 0, 0, 0.2); + border: 1px solid var(--log-error); + color: var(--log-error); +} + +.execution-icon.error { + background-color: rgba(255, 0, 0, 0.2); + border: 1px solid var(--log-error); + color: var(--log-error); +} + +.last-execution-container { + display: flex; + align-items: center; + gap: 12px; + background: linear-gradient(to right, rgba(34, 197, 94, 0.08), transparent); + border: 1px solid rgba(34, 197, 94, 0.2); + border-radius: 8px; + padding: 12px; +} + +.last-execution-container.error { + background: linear-gradient(to right, rgba(255, 0, 0, 0.08), transparent); + border-color: rgba(255, 0, 0, 0.2); +} + +.last-execution-icon-container { + width: 28px; + height: 28px; + background-color: rgba(34, 197, 94, 0.2); + border: 1px solid rgba(34, 197, 94, 0.6); + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + font-size: 14px; + color: #22c55e; + flex-shrink: 0; +} + +.last-execution-container.error .last-execution-icon-container { + background-color: rgba(255, 0, 0, 0.2); + border-color: var(--log-error); + color: var(--log-error); +} + +.last-execution-body { + flex: 1; +} + +.last-execution-status { + font-size: 11px; + color: var(--text-muted); + margin-bottom: 2px; +} + +.last-execution-name { + font-size: 13px; + color: var(--text-primary); +} + +.stat-bar-container { + display: flex; + align-items: center; + margin: 8px 0; + gap: 12px; +} + +.stat-tool-name { + min-width: 200px; + max-width: 200px; + font-weight: bold; + color: var(--text-secondary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + cursor: default; +} + +.bar-wrapper { + flex: 1; + height: 24px; + background-color: var(--border-color); + border-radius: 3px; + overflow: hidden; + position: relative; +} + +.bar { + height: 100%; + background-color: var(--btn-primary); + transition: width 0.5s ease; + border-radius: 3px; +} + +.stat-count { + min-width: 60px; + text-align: right; + font-weight: bold; + color: var(--text-primary); +} + +.no-stats-message { + text-align: center; + color: var(--text-muted); + font-style: italic; + padding: 20px; +} + +/* Log Container Styles */ +.log-container { + background-color: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 5px; + height: calc(100vh - var(--header-height) - 2 * var(--header-padding) - 3 * var(--header-gap-main)); + max-height: 700px; + overflow-y: auto; + overflow-x: auto; + padding: 10px; + white-space: pre-wrap; + font-size: 12px; + line-height: 1.4; + color: var(--text-primary); + transition: background-color 0.3s ease, border-color 0.3s ease, color 0.3s ease; +} + +.controls { + position: sticky; + top: 90px; + z-index: 100; + background-color: var(--bg-primary); + padding: 10px 0; + margin-bottom: 10px; + text-align: center; + display: flex; + justify-content: center; + align-items: center; + gap: 10px; + flex-wrap: wrap; + transition: background-color 0.3s ease; +} + +.btn { + background-color: var(--btn-primary); + color: white; + border: none; + padding: 8px 16px; + border-radius: 4px; + cursor: pointer; + font-size: 14px; + transition: background-color 0.3s ease; +} + +.btn:hover { + background-color: var(--btn-hover); +} + +.btn:disabled { + background-color: var(--btn-disabled); + cursor: not-allowed; +} + +.theme-toggle { + display: flex; + align-items: center; + gap: 5px; + background-color: var(--bg-secondary); + color: var(--text-primary); + border: 1px solid var(--border-color); + border-radius: 4px; + padding: 8px 16px; + cursor: pointer; + font-size: 16px; + transition: background-color 0.3s ease, border-color 0.3s ease; +} + +.theme-toggle:hover { + background-color: var(--border-color); +} + +.theme-toggle span { + line-height: 1; +} + +.log-debug { + color: var(--log-debug); +} + +.log-info { + color: var(--log-info); +} + +.log-warning { + color: var(--log-warning); +} + +.log-error { + color: var(--log-error); +} + +.log-default { + color: var(--log-info); +} + +/* Tool name highlighting */ +.tool-name { + background-color: var(--tool-highlight); + color: var(--tool-highlight-text); + font-weight: bold; +} + +.loading { + text-align: center; + color: var(--text-muted); + font-style: italic; +} + +.error-message { + color: var(--log-error); + text-align: center; + margin: 10px 0; +} + +/* Advanced Stats Styles */ +.charts-container { + display: flex; + flex-wrap: wrap; + gap: 15px; + justify-content: space-between; + max-width: 1400px; + margin: 0 auto; +} + +.chart-group { + flex: 1; + min-width: 280px; + max-width: 320px; + text-align: center; +} + +.chart-wide { + flex: 0 0 100%; + min-width: 100%; + margin-top: 10px; +} + +.chart-group h3 { + margin: 0 0 10px 0; + color: var(--text-secondary); +} + +.stats-summary { + margin: 0 auto; + border-collapse: collapse; + background: var(--bg-secondary); + border-radius: 5px; + overflow: hidden; + box-shadow: var(--shadow); + transition: background-color 0.3s ease, box-shadow 0.3s ease; +} + +.stats-summary th, +.stats-summary td { + padding: 10px 20px; + text-align: left; + border-bottom: 1px solid var(--border-color); + color: var(--text-primary); + transition: border-color 0.3s ease, color 0.3s ease; +} + +.stats-summary th { + background-color: var(--stats-header); + font-weight: bold; + transition: background-color 0.3s ease; +} + +.stats-summary tr:last-child td { + border-bottom: none; +} + +@media (max-width: 1024px) { + .overview-container { + grid-template-columns: 1fr; + } + + .overview-right { + order: -1; + } +} + +@media (max-width: 768px) { + body { + padding-top: 140px; + } + + .header { + flex-direction: column; + gap: 10px; + padding: 10px 15px; + } + + .logo-container { + width: 100%; + text-align: center; + } + + .logo-container img { + max-width: 200px; + } + + .header-nav { + width: 100%; + justify-content: center; + } + + .charts-container { + flex-direction: column; + } + + .chart-group, + .chart-wide { + min-width: auto; + max-width: none; + } + + .controls { + flex-direction: column; + gap: 5px; + } + + .config-grid { + grid-template-columns: 1fr; + } + + .tools-grid { + grid-template-columns: 1fr; + } + + .stat-bar-container { + flex-wrap: wrap; + } + + .stat-tool-name { + min-width: 100%; + } +} + +/* Modal Styles */ +.modal { + position: fixed; + z-index: 1000; + left: 0; + top: 0; + width: 100%; + height: 100%; + overflow: auto; + background-color: rgba(0, 0, 0, 0.5); +} + +.modal-content { + background-color: var(--bg-primary); + margin: 10% auto; + padding: 25px; + border: 1px solid var(--border-color); + border-radius: 8px; + width: 90%; + max-width: 500px; + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.3); + position: relative; +} + +.modal-close { + color: var(--text-muted); + float: right; + font-size: 28px; + font-weight: bold; + line-height: 20px; + cursor: pointer; + transition: color 0.2s; +} + +.modal-close:hover, +.modal-close:focus { + color: var(--text-primary); +} + +.modal h3 { + margin-top: 0; + margin-bottom: 15px; + color: var(--text-primary); +} + +/* Language Badge Styles */ +.languages-container { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.language-badge { + position: relative; + display: inline-flex; + align-items: center; + padding: 6px 12px; + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 6px; + color: var(--text-primary); + font-size: 13px; + font-weight: 500; +} + +.language-badge.removable { + padding-right: 28px; +} + +.language-remove { + position: absolute; + top: 2px; + right: 2px; + width: 18px; + height: 18px; + display: flex; + align-items: center; + justify-content: center; + background: rgba(255, 68, 68, 0.1); + border-radius: 3px; + cursor: pointer; + color: #ff4444; + font-size: 14px; + font-weight: bold; + line-height: 1; + transition: all 0.2s; +} + +.language-remove:hover { + background: rgba(255, 68, 68, 0.2); + transform: scale(1.1); +} + +.language-add-btn { + padding: 6px 12px; + font-size: 13px; + font-weight: 500; + border-radius: 6px; + border: 1px dashed var(--border-color); + background: var(--bg-secondary); + color: var(--text-primary); + cursor: pointer; + transition: all 0.2s; +} + +.language-add-btn:hover { + background: var(--border-color); + border-color: var(--btn-primary); + color: var(--btn-primary); +} + +.memory-add-btn { + display: inline-flex; + align-items: center; + padding: 8px 12px; + margin: 5px; + border-radius: 4px; + border: 1px dashed var(--border-color); + background: var(--bg-secondary); + color: var(--text-primary); + cursor: pointer; + transition: all 0.2s; + font-family: inherit; + font-size: inherit; + font-weight: inherit; + line-height: inherit; +} + +.memory-add-btn:hover { + background: var(--border-color); + border-color: var(--btn-primary); + color: var(--btn-primary); +} + +.language-spinner { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 6px 12px; +} + +.spinner { + width: 16px; + height: 16px; + border: 2px solid var(--border-color); + border-top-color: var(--btn-primary); + border-radius: 50%; + animation: spin 0.8s linear infinite; +} + +@keyframes spin { + to { + transform: rotate(360deg); + } +} + +/* Memory Editor Styles */ +.modal-content-large { + max-width: 800px; + width: 90%; +} + +.memory-editor { + font-family: 'Courier New', monospace; + font-size: 13px; + line-height: 1.5; + tab-size: 4; + -moz-tab-size: 4; +} + +.memory-editor:focus { + outline: 2px solid var(--btn-primary); + outline-offset: -1px; +} + +/* Memory Item Styles */ +.memory-item { + position: relative; + display: inline-flex; + align-items: center; + padding: 8px 12px; + margin: 5px; + border-radius: 4px; + background-color: var(--bg-primary); + border: 1px solid var(--border-color); + transition: background-color 0.2s ease; + cursor: pointer; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.memory-item:hover { + background-color: var(--border-color); + text-decoration: underline; +} + +.memory-item.removable { + padding-right: 28px; +} + +.memory-remove { + position: absolute; + top: 2px; + right: 2px; + width: 18px; + height: 18px; + display: flex; + align-items: center; + justify-content: center; + background: rgba(255, 68, 68, 0.1); + border-radius: 3px; + cursor: pointer; + color: #ff4444; + font-size: 14px; + font-weight: bold; + line-height: 1; + transition: all 0.2s; +} + +.memory-remove:hover { + background: rgba(255, 68, 68, 0.2); + transform: scale(1.1); + text-decoration: none; +} + +.memories-container { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +/* Copy Logs Button */ +.copy-logs-btn { + position: absolute; + top: 15px; + right: 15px; + z-index: 10; + display: flex; + align-items: center; + gap: 6px; + padding: 8px 12px; + background-color: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 4px; + color: var(--text-primary); + cursor: pointer; + opacity: 0.8; + transition: opacity 0.2s ease, background-color 0.2s ease; + font-size: 13px; + font-weight: 500; +} + +.copy-logs-btn:hover { + opacity: 1; + background-color: var(--border-color); +} + +.copy-logs-btn svg { + flex-shrink: 0; +} + +.copy-logs-text { + display: none; + white-space: nowrap; +} + +.copy-logs-btn:hover .copy-logs-text { + display: inline; +} diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index 5ce12df4..ef8168fa 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -22,86 +22,1061 @@ class LogMessage { highlightToolNames(message, toolNames) { let highlightedMessage = message; - toolNames.forEach(function(toolName) { + toolNames.forEach(function (toolName) { const regex = new RegExp('\\b' + toolName + '\\b', 'gi'); highlightedMessage = highlightedMessage.replace(regex, '' + toolName + ''); }); return highlightedMessage; } - escapeHtml (convertString) { - if (typeof convertString !== 'string') return convertString; + escapeHtml(convertString) { + if (typeof convertString !== 'string') return convertString; const patterns = { - '<' : '<', - '>' : '>', - '&' : '&', - '"' : '"', - '\'' : ''', - '`' : '`' + '<': '<', '>': '>', '&': '&', '"': '"', '\'': ''', '`': '`' }; return convertString.replace(/[<>&"'`]/g, match => patterns[match]); - }; + }; +} + +class SponsorRotation { + constructor() { + this.platinumIndex = 0; + this.goldIndex = 0; + this.platinumTimer = null; + this.goldTimer = null; + this.platinumInterval = 5000; + this.goldInterval = 5000; + + this.init(); + } + + init() { + let self = this; + this.loadSponsors(function() { + self.startPlatinumRotation(); + self.startGoldRotation(); + }); + } + + loadSponsors(onSuccess) { + $.ajax({ + url: 'https://oraios-software.de/serena-sponsors/manifest.php', + type: 'GET', + success: function (response) { + console.log('Sponsors loaded:', response); + + function fillSponsors($container, sponsors, className) { + $.each(sponsors, function (index, sponsor) { + let $img = $(''); + let $anchor = $(''); + $anchor.append($img); + let $sponsor = $('
'); + $sponsor.append($anchor); + if (index === 0) { + $sponsor.addClass('active'); + } + if (sponsor.border) { + $img.addClass('sponsor-border'); + } + $container.append($sponsor); + }); + } + + fillSponsors($('#gold-sponsors'), response.gold, 'gold-sponsor'); + fillSponsors($('#platinum-sponsors'), response.platinum, 'platinum-sponsor'); + onSuccess(); + }, + error: function (xhr, status, error) { + console.error('Error loading sponsors:', error); + } + }); + } + + startPlatinumRotation() { + const self = this; + this.platinumTimer = setInterval(() => { + self.rotatePlatinum('next'); + }, this.platinumInterval); + } + + startGoldRotation() { + const self = this; + this.goldTimer = setInterval(() => { + self.rotateGold('next'); + }, this.goldInterval); + } + + rotatePlatinum(direction) { + const $slides = $('.platinum-sponsor-slide'); + const total = $slides.length; + + if (total === 0) return; + + // Remove active class from current slide + $slides.eq(this.platinumIndex).removeClass('active'); + + // Calculate next index + if (direction === 'next') { + this.platinumIndex = (this.platinumIndex + 1) % total; + } else { + this.platinumIndex = (this.platinumIndex - 1 + total) % total; + } + + // Add active class to new slide + $slides.eq(this.platinumIndex).addClass('active'); + + // Reset timer + clearInterval(this.platinumTimer); + this.startPlatinumRotation(); + } + + rotateGold(direction) { + const $groups = $('.gold-sponsor-slide'); + const total = $groups.length; + + if (total === 0) return; + + // Remove active class from current group + $groups.eq(this.goldIndex).removeClass('active'); + + // Calculate next index + if (direction === 'next') { + this.goldIndex = (this.goldIndex + 1) % total; + } else { + this.goldIndex = (this.goldIndex - 1 + total) % total; + } + + // Add active class to new group + $groups.eq(this.goldIndex).addClass('active'); + + // Reset timer + clearInterval(this.goldTimer); + this.startGoldRotation(); + } } class Dashboard { constructor() { let self = this; + // Page state + this.currentPage = 'overview'; + this.configData = null; + this.lastConfigDataJson = null; // Cache for comparison + this.jetbrainsMode = false; + this.activeProjectName = null; + this.languageToRemove = null; + this.currentMemoryName = null; + this.originalMemoryContent = null; + this.memoryContentDirty = false; + this.memoryToDelete = null; + this.isAddingLanguage = false; + this.waitingForConfigPollingResult = false; + this.waitingForExecutionsPollingResult = false; + this.originalSerenaConfigContent = null; + this.serenaConfigContentDirty = false; + + // Execution tracking + this.cancelledExecutions = []; + this.executionToCancel = null; + + // Tool names and stats this.toolNames = []; this.currentMaxIdx = -1; this.pollInterval = null; - this.failureCount = 0; + this.configPollInterval = null; + this.executionsPollInterval = null; + this.heartbeatFailureCount = 0; + + // jQuery elements this.$logContainer = $('#log-container'); this.$errorContainer = $('#error-container'); - this.$loadButton = $('#load-logs'); - this.$shutdownButton = $('#shutdown'); - this.$toggleStats = $('#toggle-stats'); - this.$statsSection = $('#stats-section'); - this.$refreshStats = $('#refresh-stats'); - this.$clearStats = $('#clear-stats'); + this.$copyLogsBtn = $('#copy-logs-btn'); + this.$menuToggle = $('#menu-toggle'); + this.$menuDropdown = $('#menu-dropdown'); + this.$menuShutdown = $('#menu-shutdown'); this.$themeToggle = $('#theme-toggle'); this.$themeIcon = $('#theme-icon'); this.$themeText = $('#theme-text'); + this.$configDisplay = $('#config-display'); + this.$basicStatsDisplay = $('#basic-stats-display'); + this.$statsSection = $('#stats-section'); + this.$refreshStats = $('#refresh-stats'); + this.$clearStats = $('#clear-stats'); + this.$projectsDisplay = $('#projects-display'); + this.$projectsHeader = $('#projects-header'); + this.$availableToolsDisplay = $('#available-tools-display'); + this.$availableModesDisplay = $('#available-modes-display'); + this.$availableContextsDisplay = $('#available-contexts-display'); + this.$addLanguageModal = $('#add-language-modal'); + this.$modalLanguageSelect = $('#modal-language-select'); + this.$modalProjectName = $('#modal-project-name'); + this.$modalAddBtn = $('#modal-add-btn'); + this.$modalCancelBtn = $('#modal-cancel-btn'); + this.$modalClose = $('.modal-close'); + this.$removeLanguageModal = $('#remove-language-modal'); + this.$removeLanguageName = $('#remove-language-name'); + this.$removeModalOkBtn = $('#remove-modal-ok-btn'); + this.$removeModalCancelBtn = $('#remove-modal-cancel-btn'); + this.$modalCloseRemove = $('.modal-close-remove'); + this.$editMemoryModal = $('#edit-memory-modal'); + this.$editMemoryName = $('#edit-memory-name'); + this.$editMemoryContent = $('#edit-memory-content'); + this.$editMemorySaveBtn = $('#edit-memory-save-btn'); + this.$editMemoryCancelBtn = $('#edit-memory-cancel-btn'); + this.$modalCloseEditMemory = $('.modal-close-edit-memory'); + this.$deleteMemoryModal = $('#delete-memory-modal'); + this.$deleteMemoryName = $('#delete-memory-name'); + this.$deleteMemoryOkBtn = $('#delete-memory-ok-btn'); + this.$deleteMemoryCancelBtn = $('#delete-memory-cancel-btn'); + this.$modalCloseDeleteMemory = $('.modal-close-delete-memory'); + this.$createMemoryModal = $('#create-memory-modal'); + this.$createMemoryProjectName = $('#create-memory-project-name'); + this.$createMemoryNameInput = $('#create-memory-name-input'); + this.$createMemoryCreateBtn = $('#create-memory-create-btn'); + this.$createMemoryCancelBtn = $('#create-memory-cancel-btn'); + this.$modalCloseCreateMemory = $('.modal-close-create-memory'); + this.$activeExecutionQueueDisplay = $('#active-executions-display'); + this.$lastExecutionDisplay = $('#last-execution-display'); + this.$cancelledExecutionsDisplay = $('#cancelled-executions-display'); + this.$cancelExecutionModal = $('#cancel-execution-modal'); + this.$cancelExecutionOkBtn = $('#cancel-execution-ok-btn'); + this.$cancelExecutionCancelBtn = $('#cancel-execution-cancel-btn'); + this.$modalCloseCancelExecution = $('.modal-close-cancel-execution'); + this.$editSerenaConfigModal = $('#edit-serena-config-modal'); + this.$editSerenaConfigContent = $('#edit-serena-config-content'); + this.$editSerenaConfigSaveBtn = $('#edit-serena-config-save-btn'); + this.$editSerenaConfigCancelBtn = $('#edit-serena-config-cancel-btn'); + this.$modalCloseEditSerenaConfig = $('.modal-close-edit-serena-config'); + // Chart references this.countChart = null; this.tokensChart = null; this.inputChart = null; this.outputChart = null; - // register event handlers - this.$loadButton.click(this.loadLogs.bind(this)); - this.$shutdownButton.click(this.shutdown.bind(this)); - this.$toggleStats.click(this.toggleStats.bind(this)); + // Register event handlers + this.$copyLogsBtn.click(this.copyLogs.bind(this)); + this.$menuShutdown.click(function (e) { + e.preventDefault(); + self.shutdown(); + }); + this.$menuToggle.click(this.toggleMenu.bind(this)); + this.$themeToggle.click(this.toggleTheme.bind(this)); this.$refreshStats.click(this.loadStats.bind(this)); this.$clearStats.click(this.clearStats.bind(this)); - this.$themeToggle.click(this.toggleTheme.bind(this)); + this.$modalAddBtn.click(this.addLanguageFromModal.bind(this)); + this.$modalCancelBtn.click(this.closeLanguageModal.bind(this)); + this.$modalClose.click(this.closeLanguageModal.bind(this)); + this.$removeModalOkBtn.click(this.confirmRemoveLanguageOk.bind(this)); + this.$removeModalCancelBtn.click(this.closeRemoveLanguageModal.bind(this)); + this.$modalCloseRemove.click(this.closeRemoveLanguageModal.bind(this)); + this.$editMemorySaveBtn.click(this.saveMemoryFromModal.bind(this)); + this.$editMemoryCancelBtn.click(this.closeEditMemoryModal.bind(this)); + this.$modalCloseEditMemory.click(this.closeEditMemoryModal.bind(this)); + this.$editMemoryContent.on('input', this.trackMemoryChanges.bind(this)); + this.$deleteMemoryOkBtn.click(this.confirmDeleteMemoryOk.bind(this)); + this.$deleteMemoryCancelBtn.click(this.closeDeleteMemoryModal.bind(this)); + this.$modalCloseDeleteMemory.click(this.closeDeleteMemoryModal.bind(this)); + this.$createMemoryCreateBtn.click(this.createMemoryFromModal.bind(this)); + this.$createMemoryCancelBtn.click(this.closeCreateMemoryModal.bind(this)); + this.$modalCloseCreateMemory.click(this.closeCreateMemoryModal.bind(this)); + this.$createMemoryNameInput.keypress(function (e) { + if (e.which === 13) { // Enter key + e.preventDefault(); + self.createMemoryFromModal(); + } + }); + this.$cancelExecutionOkBtn.click(this.confirmCancelExecutionOk.bind(this)); + this.$cancelExecutionCancelBtn.click(this.closeCancelExecutionModal.bind(this)); + this.$modalCloseCancelExecution.click(this.closeCancelExecutionModal.bind(this)); + this.$editSerenaConfigSaveBtn.click(this.saveSerenaConfigFromModal.bind(this)); + this.$editSerenaConfigCancelBtn.click(this.closeEditSerenaConfigModal.bind(this)); + this.$modalCloseEditSerenaConfig.click(this.closeEditSerenaConfigModal.bind(this)); - // initialize theme + // Page navigation + $('[data-page]').click(function (e) { + e.preventDefault(); + const page = $(this).data('page'); + self.navigateToPage(page); + }); + + // Close menu when clicking outside + $(document).click(function (e) { + if (!$(e.target).closest('.header-nav').length) { + self.$menuDropdown.hide(); + } + }); + + // Close modals when clicking outside + this.$addLanguageModal.click(function (e) { + if ($(e.target).hasClass('modal')) { + self.closeLanguageModal(); + } + }); + + this.$removeLanguageModal.click(function (e) { + if ($(e.target).hasClass('modal')) { + self.closeRemoveLanguageModal(); + } + }); + + this.$editMemoryModal.click(function (e) { + if ($(e.target).hasClass('modal')) { + self.closeEditMemoryModal(); + } + }); + + this.$deleteMemoryModal.click(function (e) { + if ($(e.target).hasClass('modal')) { + self.closeDeleteMemoryModal(); + } + }); + + this.$createMemoryModal.click(function (e) { + if ($(e.target).hasClass('modal')) { + self.closeCreateMemoryModal(); + } + }); + + this.$editSerenaConfigModal.click(function (e) { + if ($(e.target).hasClass('modal')) { + self.closeEditSerenaConfigModal(); + } + }); + + // Collapsible sections + $('.collapsible-header').click(function () { + const $header = $(this); + const $content = $header.next('.collapsible-content'); + const $icon = $header.find('.toggle-icon'); + + $content.slideToggle(300); + $icon.toggleClass('expanded'); + }); + + // Initialize theme this.initializeTheme(); - // initialize the application - this.loadToolNames().then(function() { - // Load logs on page load after tool names are loaded - self.loadLogs(); + // Initialize sponsor rotation + //this.sponsorRotation = new SponsorRotation(); + + // Add ESC key handler for closing modals + $(document).keydown(function (e) { + if (e.key === 'Escape' || e.keyCode === 27) { + if (self.$addLanguageModal.is(':visible')) { + self.closeLanguageModal(); + } else if (self.$removeLanguageModal.is(':visible')) { + self.closeRemoveLanguageModal(); + } else if (self.$editMemoryModal.is(':visible')) { + self.closeEditMemoryModal(); + } else if (self.$deleteMemoryModal.is(':visible')) { + self.closeDeleteMemoryModal(); + } else if (self.$createMemoryModal.is(':visible')) { + self.closeCreateMemoryModal(); + } + } + }); + + // Initialize the application + this.loadToolNames().then(function () { + // Start on overview page + self.loadConfigOverview(); + self.startConfigPolling(); + self.startExecutionsPolling(); + }); + // Initialize heartbeat interval + setInterval(this.heartbeat.bind(this), 250); + } + + heartbeat() { + let self = this; + $.ajax({ + url: '/heartbeat', + type: 'GET', + success: function (response) { + self.heartbeatFailureCount = 0; + }, + error: function (xhr, status, error) { + self.heartbeatFailureCount++; + console.error('Heartbeat failure; count = ', self.heartbeatFailureCount); + if (self.heartbeatFailureCount >= 1) { + console.log('Server appears to be down, closing tab'); + window.close(); + } + }, }); } + toggleMenu() { + this.$menuDropdown.toggle(); + } + + navigateToPage(page) { + // Hide menu + this.$menuDropdown.hide(); + + // Hide all pages + $('.page-view').hide(); + + // Show selected page + $('#page-' + page).show(); + + // Update menu active state + $('[data-page]').removeClass('active'); + $('[data-page="' + page + '"]').addClass('active'); + + // Update current page + this.currentPage = page; + + // Stop all polling + this.stopPolling(); + + // Start appropriate polling for the page + if (page === 'overview') { + this.loadConfigOverview(); + this.startConfigPolling(); + this.startExecutionsPolling(); + } else if (page === 'logs') { + this.loadLogs(); + } else if (page === 'stats') { + this.loadStats(); + } + } + + stopPolling() { + if (this.pollInterval) { + clearInterval(this.pollInterval); + this.pollInterval = null; + } + if (this.configPollInterval) { + clearInterval(this.configPollInterval); + this.configPollInterval = null; + } + if (this.executionsPollInterval) { + clearInterval(this.executionsPollInterval); + this.executionsPollInterval = null; + } + } + + // ===== Config Overview Methods ===== + + loadConfigOverview() { + if (this.waitingForConfigPollingResult) { + console.log('Still waiting for previous config poll result, skipping this poll'); + return; + } + this.waitingForConfigPollingResult = true; + console.log('Polling for config overview...'); + let self = this; + $.ajax({ + url: '/get_config_overview', + type: 'GET', + success: function (response) { + // Check if the config data has actually changed + const currentConfigJson = JSON.stringify(response); + const hasChanged = self.lastConfigDataJson !== currentConfigJson; + + if (hasChanged) { + console.log('Config has changed, updating display'); + self.lastConfigDataJson = currentConfigJson; + self.configData = response; + self.jetbrainsMode = response.jetbrains_mode; + self.activeProjectName = response.active_project.name; + self.displayConfig(response); + self.displayBasicStats(response.tool_stats_summary); + self.displayProjects(response.registered_projects); + self.displayAvailableTools(response.available_tools); + self.displayAvailableModes(response.available_modes); + self.displayAvailableContexts(response.available_contexts); + } else { + console.log('Config unchanged, skipping display update'); + } + }, error: function (xhr, status, error) { + console.error('Error loading config overview:', error); + self.$configDisplay.html('
Error loading configuration
'); + self.$basicStatsDisplay.html('
Error loading stats
'); + self.$projectsDisplay.html('
Error loading projects
'); + self.$availableToolsDisplay.html('
Error loading tools
'); + self.$availableModesDisplay.html('
Error loading modes
'); + self.$availableContextsDisplay.html('
Error loading contexts
'); + }, complete: function () { + self.waitingForConfigPollingResult = false; + } + }); + } + + startConfigPolling() { + this.configPollInterval = setInterval(this.loadConfigOverview.bind(this), 1000); + } + + startExecutionsPolling() { + // Poll every 1 second for executions (independent of config polling) + // This ensures stuck executions can still be cancelled even if config polling is blocked + this.loadExecutions() + this.executionsPollInterval = setInterval(() => { + this.loadQueuedExecutions(); + this.loadLastExecution(); + }, 1000); + } + + displayConfig(config) { + try { + // Check if tools and memories sections are currently expanded + const $existingToolsContent = $('#tools-content'); + const $existingMemoriesContent = $('#memories-content'); + const wasToolsExpanded = $existingToolsContent.is(':visible'); + const wasMemoriesExpanded = $existingMemoriesContent.is(':visible'); + + let html = '
'; + + // Project info + html += '
Active Project:
'; + if (config.active_project.name && config.active_project.path) { + const configPath = config.active_project.path + '/.serena/project.yml'; + html += '
' + config.active_project.name + '
'; + } else { + html += '
' + (config.active_project.name || 'None') + '
'; + } + + html += '
Languages:
'; + if (this.jetbrainsMode) { + html += '
Using JetBrains backend
'; + } else { + html += '
'; + if (config.languages && config.languages.length > 0) { + html += '
'; + config.languages.forEach(function (language, index) { + const isRemovable = config.languages.length > 1; + html += '
'; + html += language; + if (isRemovable) { + html += '×'; + } + html += '
'; + }); + // Add the "Add Language" button inline with language badges (only if active project exists) + if (config.active_project && config.active_project.name) { + // TODO: address after refactoring, it's not awesome to keep depending on state + if (this.isAddingLanguage) { + html += ''; + } else { + html += 'N/A'; + } + html += '
'; + } + + // Context info + html += '
Context:
'; + html += '
' + config.context.name + '
'; + + // Modes info + html += '
Active Modes:
'; + html += '
'; + if (config.modes.length > 0) { + const modeSpans = config.modes.map(function (mode) { + return '' + mode.name + ''; + }); + html += modeSpans.join(', '); + } else { + html += 'None'; + } + html += '
'; + + // File Encoding info + html += '
File Encoding:
'; + html += '
' + (config.encoding || 'N/A') + '
'; + + html += '
'; + + // Active tools - collapsible + html += '
'; + html += '

'; + html += 'Active Tools (' + config.active_tools.length + ')'; + html += '▼'; + html += '

'; + html += '
'; + config.active_tools.forEach(function (tool) { + html += '
' + tool + '
'; + }); + html += '
'; + html += '
'; + + // Available memories - collapsible (show if memories exist or if project exists) + if (config.active_project && config.active_project.name) { + html += '
'; + html += '

'; + const memoryCount = (config.available_memories && config.available_memories.length) || 0; + html += 'Available Memories (' + memoryCount + ')'; + html += '▼'; + html += '

'; + html += '
'; + if (config.available_memories && config.available_memories.length > 0) { + config.available_memories.forEach(function (memory) { + html += '
'; + html += memory; + html += '×'; + html += '
'; + }); + } + // Add Create Memory button + html += ''; + html += '
'; + html += '
'; + } + + // Configuration help link and edit config button + html += '
'; + html += '
'; + html += '📖 '; + html += 'View Configuration Guide'; + html += '
'; + html += ''; + html += '
'; + + this.$configDisplay.html(html); + + // Attach event handlers for the dynamically created add language button + $('#add-language-btn').click(this.openLanguageModal.bind(this)); + + // Attach event handler for edit serena config button + $('#edit-serena-config-btn').click(this.openEditSerenaConfigModal.bind(this)); + + // Attach event handlers for language remove buttons + const self = this; + $('.language-remove').click(function (e) { + e.preventDefault(); + e.stopPropagation(); + const language = $(this).data('language'); + self.confirmRemoveLanguage(language); + }); + + // Attach event handlers for memory items + $('.memory-item').click(function (e) { + e.preventDefault(); + const memoryName = $(this).data('memory'); + self.openEditMemoryModal(memoryName); + }); + + // Attach event handlers for memory remove buttons + $('.memory-remove').click(function (e) { + e.preventDefault(); + e.stopPropagation(); + const memoryName = $(this).data('memory'); + self.confirmDeleteMemory(memoryName); + }); + + // Attach event handler for create memory button + $('#create-memory-btn').click(this.openCreateMemoryModal.bind(this)); + + // Re-attach collapsible handler for the newly created tools header + $('#tools-header').click(function () { + const $header = $(this); + const $content = $('#tools-content'); + const $icon = $header.find('.toggle-icon'); + + $content.slideToggle(300); + $icon.toggleClass('expanded'); + }); + + // Re-attach collapsible handler for the newly created memories header + $('#memories-header').click(function () { + const $header = $(this); + const $content = $('#memories-content'); + const $icon = $header.find('.toggle-icon'); + + $content.slideToggle(300); + $icon.toggleClass('expanded'); + }); + } catch (error) { + console.error('Error in displayConfig:', error); + this.$configDisplay.html('
Error displaying configuration: ' + error.message + '
'); + } + } + + displayBasicStats(stats) { + if (Object.keys(stats).length === 0) { + this.$basicStatsDisplay.html('
No tool usage stats collected yet.
'); + return; + } + + // Sort tools by call count (descending) + const sortedTools = Object.keys(stats).sort((a, b) => { + return stats[b].num_calls - stats[a].num_calls; + }); + + const maxCalls = Math.max(...sortedTools.map(tool => stats[tool].num_calls)); + + let html = ''; + sortedTools.forEach(function (toolName) { + const count = stats[toolName].num_calls; + const percentage = maxCalls > 0 ? (count / maxCalls * 100) : 0; + + html += '
'; + html += '
' + toolName + '
'; + html += '
'; + html += '
'; + html += '
'; + html += '
' + count + '
'; + html += '
'; + }); + + this.$basicStatsDisplay.html(html); + } + + displayProjects(projects) { + if (!projects || projects.length === 0) { + this.$projectsDisplay.html('
No projects registered.
'); + return; + } + + let html = ''; + projects.forEach(function (project) { + const activeClass = project.is_active ? ' active' : ''; + html += '
'; + html += '
' + project.name + '
'; + html += '
' + project.path + '
'; + html += '
'; + }); + + this.$projectsDisplay.html(html); + } + + displayAvailableTools(tools) { + if (!tools || tools.length === 0) { + this.$availableToolsDisplay.html('
All tools are active.
'); + return; + } + + let html = ''; + tools.forEach(function (tool) { + html += '
' + tool.name + '
'; + }); + + this.$availableToolsDisplay.html(html); + } + + displayAvailableModes(modes) { + if (!modes || modes.length === 0) { + this.$availableModesDisplay.html('
No modes available.
'); + return; + } + + let html = ''; + modes.forEach(function (mode) { + const activeClass = mode.is_active ? ' active' : ''; + html += '
' + mode.name + '
'; + }); + + this.$availableModesDisplay.html(html); + } + + displayAvailableContexts(contexts) { + if (!contexts || contexts.length === 0) { + this.$availableContextsDisplay.html('
No contexts available.
'); + return; + } + + let html = ''; + contexts.forEach(function (context) { + const activeClass = context.is_active ? ' active' : ''; + html += '
' + context.name + '
'; + }); + + this.$availableContextsDisplay.html(html); + } + + // ===== Executions Methods ===== + + loadQueuedExecutions() { + let self = this; + $.ajax({ + url: '/queued_task_executions', type: 'GET', success: function (response) { + if (response.status === 'success') { + self.displayActiveExecutionsQueue(response.queued_executions || []); + } else { + console.error('Error loading executions:', response.message); + } + }, error: function (xhr, status, error) { + console.error('Error loading executions:', error); + self.$activeExecutionQueueDisplay.html('
Error loading executions
'); + } + }); + } + + loadLastExecution() { + let self = this; + $.ajax({ + url: '/last_execution', type: 'GET', success: function (response) { + if (response.status === 'success') { + if (response.last_execution !== null && response.last_execution.logged) { + self.displayLastExecution(response.last_execution); + } + } else { + console.error('Error loading last execution:', response.message); + } + }, error: function (xhr, status, error) { + console.error('Error loading last execution:', error); + self.$lastExecutionDisplay.html('
Error loading last execution
'); + } + }); + } + + loadExecutions() { + if (this.waitingForExecutionsPollingResult) { + console.log('Still waiting for previous executions poll result, skipping this poll'); + } else { + this.waitingForExecutionsPollingResult = true; + console.log('Polling for executions...'); + this.loadQueuedExecutions(); + this.loadLastExecution(); + } + } + + displayActiveExecutionsQueue(executions) { + if (!executions || executions.length === 0) { + return; + } + + let html = '
'; + let self = this; + + executions.forEach(function (execution) { + const isRunning = execution.is_running; + const logged = execution.logged; + + if (!logged) { + return; // Skip unlogged executions + } + + let itemClass = 'execution-item'; + if (isRunning) { + itemClass += ' running'; + } + + // Escape JSON for HTML attribute - replace single quotes and use HTML entities + const executionJson = JSON.stringify(execution).replace(/'/g, '''); + + html += '
'; + + if (isRunning) { + html += ''; + } + + html += '
' + self.escapeHtml(execution.name) + '
'; + + if (isRunning) { + html += '
#' + execution.task_id + '
'; + } else { + html += '
queued · #' + execution.task_id + '
'; + } + + html += ''; + html += '
'; + }); + + html += '
'; + this.$activeExecutionQueueDisplay.html(html); + + // Attach event handlers for cancel buttons + $('.execution-cancel-btn').click(function (e) { + e.preventDefault(); + console.log('Cancel button clicked'); + const $item = $(this).closest('.execution-item'); + console.log('Found item:', $item.length); + const executionDataStr = $item.attr('data-execution'); + console.log('Execution data string:', executionDataStr); + if (executionDataStr) { + // Unescape HTML entities + const unescapedStr = executionDataStr.replace(/'/g, "'"); + const executionData = JSON.parse(unescapedStr); + console.log('Parsed execution data:', executionData); + self.confirmCancelExecution(executionData); + } else { + console.error('No execution data found on element'); + } + }); + + // Update cancelled executions display + this.displayCancelledExecutions(executions); + } + + displayLastExecution(execution) { + if (!execution) { + this.$lastExecutionDisplay.html('
No executions yet.
'); + return; + } + + const isSuccess = execution.finished_successfully; + let html = '
'; + + html += '
'; + html += isSuccess ? '✓' : '✕'; + html += '
'; + + html += '
'; + html += '
' + (isSuccess ? 'Succeeded' : 'Failed') + '
'; + html += '
' + this.escapeHtml(execution.name) + '
'; + html += '
'; + + html += '
#' + execution.task_id + '
'; + html += '
'; + + this.$lastExecutionDisplay.html(html); + } + + displayCancelledExecutions() { + let self = this; + const cancelledExecs = self.cancelledExecutions + + if (cancelledExecs.length === 0) { + // Hide the cancelled executions section + $('.executions-section').eq(2).hide(); + return; + } + + // Show the cancelled executions section + $('.executions-section').eq(2).show(); + + let html = '
'; + + cancelledExecs.forEach(function (execution) { + const isAbandoned = execution.is_running; + + html += '
'; + html += '
'; + html += isAbandoned ? '!' : '✕'; + html += '
'; + html += '
' + self.escapeHtml(execution.name) + '
'; + html += '
' + (isAbandoned ? 'abandoned · ' : '') + '#' + execution.task_id + '
'; + html += '
'; + }); + + html += '
'; + this.$cancelledExecutionsDisplay.html(html); + } + + confirmCancelExecution(executionData) { + console.log('confirmCancelExecution called with:', executionData); + this.executionToCancel = executionData; + + if (executionData.is_running) { + // Show modal for running executions + console.log('Showing modal for running execution'); + this.$cancelExecutionModal.fadeIn(200); + } else { + // Directly cancel queued executions + console.log('Directly cancelling queued execution'); + this.cancelExecution(executionData); + } + } + + confirmCancelExecutionOk() { + if (this.executionToCancel) { + this.cancelExecution(this.executionToCancel); + } + this.closeCancelExecutionModal(); + } + + cancelExecution(executionData) { + const self = this; + + console.log('cancelExecution called with full execution data:', executionData); + console.log('Attempting to cancel task:', executionData.task_id); + + // Call backend API to cancel the task + $.ajax({ + url: '/cancel_task_execution', type: 'POST', contentType: 'application/json', data: JSON.stringify({ + task_id: executionData.task_id + }), success: function (response) { + console.log('Cancel task response:', response); + + if (response.status === 'error') { + console.error('Backend returned error status:', response.message); + alert('Error cancelling task: ' + response.message); + return; + } + + if (response.status === 'success') { + if (response.was_cancelled) { + console.log('Task ' + executionData.task_id + ' was successfully cancelled'); + // Add to cancelled list (only managed in JS, not persisted) + const alreadyCancelled = self.cancelledExecutions.some(function (exec) { + return exec.task_id === executionData.task_id; + }); + if (!alreadyCancelled) { + console.log('Adding execution to cancelled list:', executionData); + self.cancelledExecutions.push(executionData); + console.log('Cancelled executions array now contains:', self.cancelledExecutions); + } else { + console.log('Execution already in cancelled list'); + } + } else { + console.log('Task ' + executionData.task_id + ' could not be cancelled (may have already completed). ' + response.message); + } + // Refresh display regardless + self.loadQueuedExecutions(); + } else { + console.error('Unexpected response status:', response.status); + alert('Unexpected response from server'); + } + }, error: function (xhr, status, error) { + console.error('AJAX error cancelling task:'); + console.error(' Status:', status); + console.error(' Error:', error); + console.error(' XHR:', xhr); + console.error(' Response:', xhr.responseText); + + let errorMessage = error; + if (xhr.responseJSON && xhr.responseJSON.message) { + errorMessage = xhr.responseJSON.message; + } else if (xhr.responseText) { + errorMessage = xhr.responseText; + } + + alert('Error cancelling task: ' + errorMessage); + } + }); + } + + closeCancelExecutionModal() { + this.$cancelExecutionModal.fadeOut(200); + this.executionToCancel = null; + } + + escapeHtml(text) { + if (typeof text !== 'string') return text; + + const patterns = { + '<': '<', '>': '>', '&': '&', '"': '"', "'": ''', '`': '`' + }; + + return text.replace(/[<>&"'`]/g, match => patterns[match]); + } + + // ===== Logs Methods ===== + displayLogMessage(message) { - $('#log-container').append(new LogMessage(message, this.toolNames).$elem); + this.$logContainer.append(new LogMessage(message, this.toolNames).$elem); } loadToolNames() { let self = this; return $.ajax({ - url: '/get_tool_names', - type: 'GET', - success: function(response) { + url: '/get_tool_names', type: 'GET', success: function (response) { self.toolNames = response.tool_names || []; console.log('Loaded tool names:', self.toolNames); - }, - error: function(xhr, status, error) { + }, error: function (xhr, status, error) { console.error('Error loading tool names:', error); } }); @@ -111,23 +1086,41 @@ class Dashboard { document.title = activeProject ? `${activeProject} – Serena Dashboard` : 'Serena Dashboard'; } + copyLogs() { + const logText = this.$logContainer.text(); + + if (!logText) { + alert('No logs to copy'); + return; + } + + // Use the Clipboard API to copy text + navigator.clipboard.writeText(logText).then(() => { + // Visual feedback - temporarily change icon to grey checkmark + const originalHtml = this.$copyLogsBtn.html(); + const checkmarkSvg = 'copy logs'; + this.$copyLogsBtn.html(checkmarkSvg); + + setTimeout(() => { + this.$copyLogsBtn.html(originalHtml); + }, 1500); + }).catch(err => { + console.error('Failed to copy logs:', err); + alert('Failed to copy logs to clipboard'); + }); + } + loadLogs() { console.log("Loading logs"); let self = this; - // Disable button and show loading state - self.$loadButton.prop('disabled', true).text('Loading...'); self.$errorContainer.empty(); // Make API call $.ajax({ - url: '/get_log_messages', - type: 'POST', - contentType: 'application/json', - data: JSON.stringify({ + url: '/get_log_messages', type: 'POST', contentType: 'application/json', data: JSON.stringify({ start_idx: 0 - }), - success: function(response) { + }), success: function (response) { // Clear existing logs self.$logContainer.empty(); @@ -136,7 +1129,7 @@ class Dashboard { // Display each log message if (response.messages && response.messages.length > 0) { - response.messages.forEach(function(message) { + response.messages.forEach(function (message) { self.displayLogMessage(message); }); @@ -151,15 +1144,9 @@ class Dashboard { // Start periodic polling for new logs self.startPeriodicPolling(); - }, - error: function(xhr, status, error) { + }, error: function (xhr, status, error) { console.error('Error loading logs:', error); - self.$errorContainer.html('
Error loading logs: ' + - (xhr.responseJSON ? xhr.responseJSON.detail : error) + '
'); - }, - complete: function() { - // Re-enable button - self.$loadButton.prop('disabled', false).text('Reload Log'); + self.$errorContainer.html('
Error loading logs: ' + (xhr.responseJSON ? xhr.responseJSON.detail : error) + '
'); } }); } @@ -174,8 +1161,7 @@ class Dashboard { data: JSON.stringify({ start_idx: self.currentMaxIdx + 1 }), - success: function(response) { - self.failureCount = 0; + success: function (response) { // Only append new messages if we have any if (response.messages && response.messages.length > 0) { let wasAtBottom = false; @@ -187,7 +1173,7 @@ class Dashboard { } // Append new messages - response.messages.forEach(function(message) { + response.messages.forEach(function (message) { self.displayLogMessage(message); }); @@ -205,14 +1191,6 @@ class Dashboard { // Update window title with active project self.updateTitle(response.active_project); - }, - error: function(xhr, status, error) { - console.error('Error polling for new logs:', error); - self.failureCount++; - if (self.failureCount >= 3) { - console.log('Server appears to be down, closing tab'); - window.close(); - } } }); } @@ -227,41 +1205,28 @@ class Dashboard { this.pollInterval = setInterval(this.pollForNewLogs.bind(this), 1000); } - toggleStats() { - if (this.$statsSection.is(':visible')) { - this.$statsSection.hide(); - this.$toggleStats.text('Show Stats'); - } else { - this.$statsSection.show(); - this.$toggleStats.text('Hide Stats'); - this.loadStats(); - } - } + // ===== Stats Methods ===== loadStats() { let self = this; - $.when( - $.ajax({ url: '/get_tool_stats', type: 'GET' }), - $.ajax({ url: '/get_token_count_estimator_name', type: 'GET' }) - ).done(function(statsResp, estimatorResp) { + $.when($.ajax({url: '/get_tool_stats', type: 'GET'}), $.ajax({ + url: '/get_token_count_estimator_name', + type: 'GET' + })).done(function (statsResp, estimatorResp) { const stats = statsResp[0].stats; const tokenCountEstimatorName = estimatorResp[0].token_count_estimator_name; self.displayStats(stats, tokenCountEstimatorName); - }).fail(function() { + }).fail(function () { console.error('Error loading stats or estimator name'); }); } - clearStats() { let self = this; $.ajax({ - url: '/clear_tool_stats', - type: 'POST', - success: function() { + url: '/clear_tool_stats', type: 'POST', success: function () { self.loadStats(); - }, - error: function(xhr, status, error) { + }, error: function (xhr, status, error) { console.error('Error clearing stats:', error); } }); @@ -269,7 +1234,7 @@ class Dashboard { displayStats(stats, tokenCountEstimatorName) { const names = Object.keys(stats); - // If no stats collected + // If no stats collected if (names.length === 0) { // hide summary, charts, estimator name $('#stats-summary').hide(); @@ -292,12 +1257,12 @@ class Dashboard { const inputTokens = names.map(n => stats[n].input_tokens); const outputTokens = names.map(n => stats[n].output_tokens); const totalTokens = names.map(n => stats[n].input_tokens + stats[n].output_tokens); - + // Calculate totals for summary table const totalCalls = counts.reduce((sum, count) => sum + count, 0); const totalInputTokens = inputTokens.reduce((sum, tokens) => sum + tokens, 0); const totalOutputTokens = outputTokens.reduce((sum, tokens) => sum + tokens, 0); - + // Generate consistent colors for tools const colors = this.generateColors(names.length); @@ -324,27 +1289,18 @@ class Dashboard { // Tool calls pie chart this.countChart = new Chart(countCtx, { - type: 'pie', - data: { - labels: names, - datasets: [{ - data: counts, - backgroundColor: colors - }] - }, - options: { + type: 'pie', data: { + labels: names, datasets: [{ + data: counts, backgroundColor: colors + }] + }, options: { plugins: { - legend: { - display: true, - labels: { + legend: { + display: true, labels: { color: textColor } - }, - datalabels: { - display: true, - color: 'white', - font: { weight: 'bold' }, - formatter: (value) => value + }, datalabels: { + display: true, color: 'white', font: {weight: 'bold'}, formatter: (value) => value } } } @@ -352,27 +1308,18 @@ class Dashboard { // Input tokens pie chart this.inputChart = new Chart(inputCtx, { - type: 'pie', - data: { - labels: names, - datasets: [{ - data: inputTokens, - backgroundColor: colors - }] - }, - options: { + type: 'pie', data: { + labels: names, datasets: [{ + data: inputTokens, backgroundColor: colors + }] + }, options: { plugins: { - legend: { - display: true, - labels: { + legend: { + display: true, labels: { color: textColor } - }, - datalabels: { - display: true, - color: 'white', - font: { weight: 'bold' }, - formatter: (value) => value + }, datalabels: { + display: true, color: 'white', font: {weight: 'bold'}, formatter: (value) => value } } } @@ -380,27 +1327,18 @@ class Dashboard { // Output tokens pie chart this.outputChart = new Chart(outputCtx, { - type: 'pie', - data: { - labels: names, - datasets: [{ - data: outputTokens, - backgroundColor: colors - }] - }, - options: { + type: 'pie', data: { + labels: names, datasets: [{ + data: outputTokens, backgroundColor: colors + }] + }, options: { plugins: { - legend: { - display: true, - labels: { + legend: { + display: true, labels: { color: textColor } - }, - datalabels: { - display: true, - color: 'white', - font: { weight: 'bold' }, - formatter: (value) => value + }, datalabels: { + display: true, color: 'white', font: {weight: 'bold'}, formatter: (value) => value } } } @@ -408,78 +1346,42 @@ class Dashboard { // Combined input/output tokens bar chart this.tokensChart = new Chart(tokensCtx, { - type: 'bar', - data: { - labels: names, - datasets: [ - { - label: 'Input Tokens', - data: inputTokens, - backgroundColor: colors.map(color => color + '80'), // Semi-transparent - borderColor: colors, - borderWidth: 2, - borderSkipped: false, - yAxisID: 'y' - }, - { - label: 'Output Tokens', - data: outputTokens, - backgroundColor: colors, - yAxisID: 'y1' - } - ] - }, - options: { - responsive: true, - plugins: { + type: 'bar', data: { + labels: names, datasets: [{ + label: 'Input Tokens', data: inputTokens, backgroundColor: colors.map(color => color + '80'), // Semi-transparent + borderColor: colors, borderWidth: 2, borderSkipped: false, yAxisID: 'y' + }, { + label: 'Output Tokens', data: outputTokens, backgroundColor: colors, yAxisID: 'y1' + }] + }, options: { + responsive: true, plugins: { legend: { labels: { color: textColor } } - }, - scales: { + }, scales: { x: { ticks: { color: textColor - }, - grid: { + }, grid: { color: gridColor } - }, - y: { - type: 'linear', - display: true, - position: 'left', - beginAtZero: true, - title: { - display: true, - text: 'Input Tokens', + }, y: { + type: 'linear', display: true, position: 'left', beginAtZero: true, title: { + display: true, text: 'Input Tokens', color: textColor + }, ticks: { color: textColor - }, - ticks: { - color: textColor - }, - grid: { + }, grid: { color: gridColor } - }, - y1: { - type: 'linear', - display: true, - position: 'right', - beginAtZero: true, - title: { - display: true, - text: 'Output Tokens', + }, y1: { + type: 'linear', display: true, position: 'right', beginAtZero: true, title: { + display: true, text: 'Output Tokens', color: textColor + }, ticks: { color: textColor - }, - ticks: { - color: textColor - }, - grid: { - drawOnChartArea: false, - color: gridColor + }, grid: { + drawOnChartArea: false, color: gridColor } } } @@ -488,10 +1390,7 @@ class Dashboard { } generateColors(count) { - const colors = [ - '#FF6384', '#36A2EB', '#FFCE56', '#4BC0C0', '#9966FF', - '#FF9F40', '#FF6384', '#C9CBCF', '#4BC0C0', '#FF6384' - ]; + const colors = ['#FF6384', '#36A2EB', '#FFCE56', '#4BC0C0', '#9966FF', '#FF9F40', '#FF6384', '#C9CBCF', '#4BC0C0', '#FF6384']; return Array.from({length: count}, (_, i) => colors[i % colors.length]); } @@ -508,10 +1407,12 @@ class Dashboard { $('#stats-summary').html(tableHtml); } + // ===== Theme Methods ===== + initializeTheme() { // Check if user has manually set a theme preference const savedTheme = localStorage.getItem('serena-theme'); - + if (savedTheme) { // User has manually set a preference, use it this.setTheme(savedTheme); @@ -519,7 +1420,7 @@ class Dashboard { // No manual preference, detect system color scheme this.detectSystemTheme(); } - + // Listen for system theme changes this.setupSystemThemeListener(); } @@ -534,7 +1435,7 @@ class Dashboard { setupSystemThemeListener() { // Listen for changes in system color scheme const mediaQuery = window.matchMedia('(prefers-color-scheme: dark)'); - + const handleSystemThemeChange = (e) => { // Only auto-switch if user hasn't manually set a preference const savedTheme = localStorage.getItem('serena-theme'); @@ -543,7 +1444,7 @@ class Dashboard { this.setTheme(newTheme); } }; - + // Add listener for system theme changes if (mediaQuery.addEventListener) { mediaQuery.addEventListener('change', handleSystemThemeChange); @@ -556,7 +1457,7 @@ class Dashboard { toggleTheme() { const currentTheme = document.documentElement.getAttribute('data-theme') || 'light'; const newTheme = currentTheme === 'light' ? 'dark' : 'light'; - + // When user manually toggles, save their preference localStorage.setItem('serena-theme', newTheme); this.setTheme(newTheme); @@ -565,8 +1466,8 @@ class Dashboard { setTheme(theme) { // Set the theme on the document element document.documentElement.setAttribute('data-theme', theme); - - // Update the toggle button + + // Update the theme toggle button if (theme === 'dark') { this.$themeIcon.text('☀️'); this.$themeText.text('Light'); @@ -574,13 +1475,13 @@ class Dashboard { this.$themeIcon.text('🌙'); this.$themeText.text('Dark'); } - + // Update the logo based on theme this.updateLogo(theme); - + // Save to localStorage localStorage.setItem('serena-theme', theme); - + // Update charts if they exist this.updateChartsTheme(); } @@ -589,9 +1490,9 @@ class Dashboard { const logoElement = document.getElementById('serena-logo'); if (logoElement) { if (theme === 'dark') { - logoElement.src = 'serena-logs-dark-mode.png'; + logoElement.src = 'serena-logo-dark-mode.svg'; } else { - logoElement.src = 'serena-logs.png'; + logoElement.src = 'serena-logo.svg'; } } } @@ -600,56 +1501,456 @@ class Dashboard { const isDark = document.documentElement.getAttribute('data-theme') === 'dark'; const textColor = isDark ? '#ffffff' : '#000000'; const gridColor = isDark ? '#444' : '#ddd'; - - // Update existing charts - if (this.countChart) { - this.countChart.options.scales.x.ticks.color = textColor; - this.countChart.options.scales.y.ticks.color = textColor; - this.countChart.options.scales.x.grid.color = gridColor; - this.countChart.options.scales.y.grid.color = gridColor; + + // Update existing charts if they exist and have the scales property + if (this.countChart && this.countChart.options.plugins) { + if (this.countChart.options.plugins.legend) { + this.countChart.options.plugins.legend.labels.color = textColor; + } this.countChart.update(); } - - if (this.inputChart) { - this.inputChart.options.scales.x.ticks.color = textColor; - this.inputChart.options.scales.y.ticks.color = textColor; - this.inputChart.options.scales.x.grid.color = gridColor; - this.inputChart.options.scales.y.grid.color = gridColor; + + if (this.inputChart && this.inputChart.options.plugins) { + if (this.inputChart.options.plugins.legend) { + this.inputChart.options.plugins.legend.labels.color = textColor; + } this.inputChart.update(); } - - if (this.outputChart) { - this.outputChart.options.scales.x.ticks.color = textColor; - this.outputChart.options.scales.y.ticks.color = textColor; - this.outputChart.options.scales.x.grid.color = gridColor; - this.outputChart.options.scales.y.grid.color = gridColor; + + if (this.outputChart && this.outputChart.options.plugins) { + if (this.outputChart.options.plugins.legend) { + this.outputChart.options.plugins.legend.labels.color = textColor; + } this.outputChart.update(); } - - if (this.tokensChart) { + + if (this.tokensChart && this.tokensChart.options.scales) { this.tokensChart.options.scales.x.ticks.color = textColor; this.tokensChart.options.scales.y.ticks.color = textColor; this.tokensChart.options.scales.y1.ticks.color = textColor; this.tokensChart.options.scales.x.grid.color = gridColor; this.tokensChart.options.scales.y.grid.color = gridColor; this.tokensChart.options.scales.y1.grid.color = gridColor; + this.tokensChart.options.scales.y.title.color = textColor; + this.tokensChart.options.scales.y1.title.color = textColor; + if (this.tokensChart.options.plugins && this.tokensChart.options.plugins.legend) { + this.tokensChart.options.plugins.legend.labels.color = textColor; + } this.tokensChart.update(); } } + // ===== Language Management Methods ===== + + confirmRemoveLanguage(language) { + // Store the language to remove + this.languageToRemove = language; + + // Set language name in modal + this.$removeLanguageName.text(language); + + // Show modal + this.$removeLanguageModal.fadeIn(200); + } + + closeRemoveLanguageModal() { + this.$removeLanguageModal.fadeOut(200); + this.languageToRemove = null; + } + + confirmRemoveLanguageOk() { + if (this.languageToRemove) { + this.removeLanguage(this.languageToRemove); + this.closeRemoveLanguageModal(); + } + } + + removeLanguage(language) { + const self = this; + + $.ajax({ + url: '/remove_language', type: 'POST', contentType: 'application/json', data: JSON.stringify({ + language: language + }), success: function (response) { + if (response.status === 'success') { + // Reload config to show updated language list + self.loadConfigOverview(); + } else { + alert('Error removing language ' + language + ": " + response.message); + } + }, error: function (xhr, status, error) { + console.error('Error removing language:', error); + alert('Error removing language: ' + (xhr.responseJSON ? xhr.responseJSON.message : error)); + } + }); + } + + openLanguageModal() { + // Set project name in modal + this.$modalProjectName.text(this.activeProjectName || 'Unknown'); + + // Load available languages into modal dropdown + this.loadAvailableLanguages(); + + // Show modal + this.$addLanguageModal.fadeIn(200); + } + + closeLanguageModal() { + this.$addLanguageModal.fadeOut(200); + this.$modalLanguageSelect.empty(); + this.$modalAddBtn.prop('disabled', false).text('Add Language'); + } + + loadAvailableLanguages() { + let self = this; + $.ajax({ + url: '/get_available_languages', type: 'GET', success: function (response) { + const languages = response.languages || []; + // Clear all existing options + self.$modalLanguageSelect.empty(); + + if (languages.length === 0) { + // Show message if no languages available + self.$modalLanguageSelect.append($('