diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 058ecbea..e757f030 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -139,7 +139,10 @@ jobs: - name: Setup .NET SDK (for F# and C# languages) uses: actions/setup-dotnet@v4 with: - dotnet-version: '8.0.x' + dotnet-version: '10.0.x' + - name: List .NET runtimes + shell: bash + run: dotnet --list-runtimes - name: Install Terraform uses: hashicorp/setup-terraform@v3 with: @@ -196,6 +199,23 @@ jobs: - name: Install Ruby language server shell: bash run: gem install ruby-lsp + - name: Install OCaml and opam + uses: ocaml/setup-ocaml@v3 + with: + ocaml-compiler: ${{ runner.os == 'Windows' && '4.14' || '5.3.x' }} + dune-cache: true + opam-repositories: | + ${{ runner.os == 'Windows' && 'opam-repository-mingw: https://github.com/ocaml-opam/opam-repository-mingw.git#sunset' || '' }} + default: https://github.com/ocaml/opam-repository.git + - name: Install OCaml packages + shell: bash + run: | + if [ "$RUNNER_OS" = "Windows" ]; then + opam install -y dune ocaml-lsp-server + else + # Require ocaml-lsp-server >= 1.23.0 for cross-file reference support + opam install -y dune 'ocaml-lsp-server>=1.23.0' + fi - name: Install R uses: r-lib/actions/setup-r@v2 with: @@ -363,6 +383,9 @@ jobs: echo "PERL_MB_OPT=--install_base \"$HOME/perl5\"" >> $GITHUB_ENV echo "PERL_MM_OPT=INSTALL_BASE=$HOME/perl5" >> $GITHUB_ENV echo "$HOME/perl5/bin" >> $GITHUB_PATH + - name: Install ansible-core and ansible-lint (for Ansible language server tests) + shell: bash + run: uv run pip install ansible-core ansible-lint - name: Install Elm shell: bash run: npm install -g elm@0.19.1-6 @@ -394,7 +417,7 @@ jobs: - name: Install Regal (Rego Language Server) shell: bash run: | - REGAL_VERSION="0.36.1" + REGAL_VERSION="0.39.0" if [[ "${{ runner.os }}" == "Linux" ]]; then if [[ "$(uname -m)" == "x86_64" ]]; then @@ -524,6 +547,10 @@ jobs: echo "ERROR: FPCDIR does not exist: $FPCDIR" exit 1 fi + - name: Build Lean 4 test project + uses: leanprover/lean-action@v1 + with: + lake-package-directory: test/resources/repos/lean4/test_repo - name: Cache language servers id: cache-language-servers uses: actions/cache@v3 diff --git a/.gitignore b/.gitignore index ec2ca442..2dad922f 100644 --- a/.gitignore +++ b/.gitignore @@ -226,13 +226,22 @@ tmp/ # Exception: Don't ignore Nix test repository lib directory (contains source code) !/test/resources/repos/nix/test_repo/lib +# Exception: Don't ignore OCaml test repository lib directory (contains source code) +!/test/resources/repos/ocaml/test_repo/lib + # Exception: Don't ignore Julia test repository lib directory (contains source code) !/test/resources/repos/julia/test_repo/lib +# Exception: Don't ignore Solidity test repository lib directory (contains source code) +!/test/resources/repos/solidity/test_repo/contracts/lib + # Swift /test/resources/repos/swift/test_repo/.build /test/resources/repos/swift/test_repo/.swiftpm +# OCaml +/test/resources/repos/ocaml/test_repo/_build + # Elm /test/resources/repos/elm/test_repo/.elm/ /test/resources/repos/elm/test_repo/elm-stuff/ @@ -253,5 +262,8 @@ dist-newstyle/ cabal.project.local* .ghc.environment.* +# Lean 4 +.lake/ + zz-misc/ -vue-implementation/ \ No newline at end of file +vue-implementation/ diff --git a/.serena/.gitignore b/.serena/.gitignore index 14d86ad6..2e510aff 100644 --- a/.serena/.gitignore +++ b/.serena/.gitignore @@ -1 +1,2 @@ /cache +/project.local.yml diff --git a/.serena/project.yml b/.serena/project.yml index f416c01c..004546bd 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -90,8 +90,22 @@ included_optional_tools: [] # This cannot be combined with non-empty excluded_tools or included_optional_tools. # initial prompt for the project. It will always be given to the LLM upon activating the project # (contrary to the memories, which are loaded on demand). -initial_prompt: "" +initial_prompt: | + IMPORTANT: You use an idiomatic, object-oriented style. + In particular, this implies that, for any non-trivial interfaces, you use interfaces that expect explicitly typed abstractions + rather than mere functions (i.e. use the strategy pattern, for example). + You avoid the use of low-level data structures in all cases where an object-oriented abstraction would be more appropriate. + For simple data storage, you use dataclasses instead of dictionaries or tuples. + You structure function implementations into functional blocks that are separated by blank lines. + Atop each functional block, you write an elliptical phrase (starting with lower-case letter) that describes the purpose of the + block in a concise manner. + + Docstrings: You consistently use reStructuredText. + + Comments: + When describing parameters, methods/functions and classes, you use a precise style, where the initial (elliptical) phrase + clearly defines *what* it is. Any details then follow in subsequent sentences. # the encoding used by text files in the project # For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings encoding: utf-8 @@ -128,3 +142,11 @@ symbol_info_budget: # Note: the backend is fixed at startup. If a project with a different backend # is activated post-init, an error will be returned. language_backend: + +# list of regex patterns which, when matched, mark a memory entry as read‑only. +# Extends the list from the global configuration, merging the two lists. +read_only_memory_patterns: [] + +# line ending convention for file writes: unset (use global setting, "lf", "crlf", or "native" (platform default) +# If not undefined, overrides the global setting in serena_config.yml +line_ending: diff --git a/CHANGELOG.md b/CHANGELOG.md index e0d94210..701cfce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,30 @@ -# latest +# Latest + Status of the `main` branch. Changes prior to the next official version change will appear here. +* Memories: + * Add `ignored_memory_patterns` configuration option (regex-based) to completely exclude + matching memories from listing, reading, and writing. Configurable at both global + (`serena_config.yml`) and project (`project.yml`) level, merged additively. + Useful for projects with large numbers of archived memory files that inflate + `activate_project` output. + +* New language support: + * Add Solidity language server support (`Language.SOLIDITY`) using the + Nomic Foundation `@nomicfoundation/solidity-language-server`. Automatically + installed via npm. Supports `.sol` files with go-to-definition, find references, + document symbols, hover, and diagnostics. Works best with a `foundry.toml` or + `hardhat.config.js` in the project root. * General: * Add monorepo/multi-language support * Project configuration files (`project.yml`) can now define multiple languages. Auto-detection adds only the most prominent language by default. * Additional languages can be conveniently added via the Dashboard while a project is already activated. - * The current project can be activated automatically even when the MCP configuration is global (through the --project-from-cwd flag) + * Add support for querying projects other than the currently active one via new tools `QueryProjectTool` and `ListQueryableProjectsTool`. + The `QueryProjectTool` allows Serena tools to be called on other projects. + * For the LSP backend, calling symbolic tools require a project server to be spawned that will launch the respective language servers + * For the JetBrains backend, all projects for which IDE instances are open can directly be queried * Support overloaded symbols in `FindSymbolTool` and related tools * Name paths of overloaded symbols now include an index (e.g., `myOverloadedFunction[2]`) * Responses of the Java language server, which handled this in its own way, are now adapted accordingly, @@ -19,26 +36,35 @@ Status of the `main` branch. Changes prior to the next official version change w * View tool usage statistics * View and create memories and edit the serena configuration file * Log page now has save (downloads a snapshot) and clear (resets log view) buttons alongside the existing copy button - * New two-tier caching of language server document symbols and considerable performance improvements surrounding symbol retrieval/indexing + * Language server backend: + * New two-tier caching of language server document symbols and considerable performance improvements surrounding symbol retrieval/indexing + * Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`) + * Add the JetBrains language backend as an alternative to language servers + * Improve management of the Serena projects + * Facilitate project activation based on the current directory (through the `--project-from-cwd` parameter) + * Add notion of a "single-project context" (flag `single_project`), allowing user-defined contexts to behave + like the built-in `ide-assistant` context (where the available tools are restricted to ones required by the active + project and project changes are disabled) + * The location of Serena's project-specific data folder can now be flexibly configured, allowing, in particular, + locations outside of the project folder, thus improving support for read-only projects. + * Add support for `project.local.yml` for local overrides that should not be versioned * Various fixes related to indexing, special paths and determination of ignored paths - * Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations - * Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`) - * Add notion of a "single-project context" (flag `single_project`), allowing user-defined contexts to behave - like the built-in `ide-assistant` context (where the available tools are restricted to ones required by the active - project and project changes are disabled) * Client support: * New mode `oaicompat-agent` and extensions enhancing OpenAI tool compatibility, permitting Serena to work with llama.cpp * Tools: - * *New tool*: `jet_brains_type_hierarchy` * Symbol information (hover, docstring, quick-info) is now provided as part of `find_symbol` and related tool responses. + * Added `QueryProjectTool` and `ListQueryableProjectTool` (see above) * Added `RenameSymbolTool` for renaming symbols across the codebase (if LS supports this operation). * Replaced `ReplaceRegexTool` with `ReplaceContentTool`, which supports both plain text and regex-based replacements (and which requires no escaping in the replacement text, making it more robust) + * Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations * Language support: + * **Add support for Lean 4** via built-in `lean --server` with cross-file reference support (requires `lean` and `lake` via [elan](https://github.com/leanprover/elan)) + * **Add support for OCaml** via ocaml-lsp-server with cross-file reference support on OCaml 5.2+ (requires opam; see [setup guide](docs/03-special-guides/ocaml_setup_guide_for_serena.md)) * **Add Phpactor as alternative PHP language server** (specify `php_phpactor` as language; requires PHP 8.1+) * **Add support for Fortran** via fortls language server (requires `pip install fortls`) * **Add partial support for Groovy** requires user-provided Groovy language server JAR (see [setup guide](docs/03-special-guides/groovy_setup_guide_for_serena.md)) diff --git a/CLAUDE.md b/CLAUDE.md index 8f1ef25c..53ddc373 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Development Commands **Essential Commands (use these exact commands):** -- `uv run poe format` - Format code (BLACK + RUFF) - ONLY allowed formatting command +- `uv run poe format` - Format code (RUFF) - ONLY allowed formatting command - `uv run poe type-check` - Run mypy type checking - ONLY allowed type checking command - `uv run poe test` - Run tests with default markers (excludes java/rust by default) - `uv run poe test -m "python or go"` - Run specific language tests @@ -108,7 +108,7 @@ Configuration is loaded from (in order of precedence): ## Working with the Codebase - Project uses Python 3.11 with `uv` for dependency management -- Strict typing with mypy, formatted with black + ruff +- Strict typing with mypy, formatted with ruff - Language servers run as separate processes with LSP communication - Memory system enables persistent project knowledge - Context/mode system allows workflow customization \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 177762c9..a59b63be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,9 @@ The following types of contributions can be submitted directly via pull requests For other changes, please open an issue first to discuss your ideas with the maintainers. +When submitting a PR, ensure a well-defined scope. +Every PR should cover a single logical change or a set of closely related changes. + ### Adding Support for a New Language Server See the corresponding [memory](.serena/memories/adding_new_language_support_guide.md). diff --git a/README.md b/README.md index b03b07c1..d4fa11ed 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,6 @@ Instead, it can use code-centric tools like `find_symbol`, `find_referencing_sym

Changelog - Roadmap Lessons Learned

@@ -81,8 +80,8 @@ Serena incorporates a powerful abstraction layer for the integration of language that implement the language server protocol (LSP). The underlying language servers are typically open-source projects (like Serena) or at least freely available for use. -With Serena's LSP library, we provide **support for over 30 programming languages**, including -AL, Bash, C#, C/C++, Clojure, Dart, Elixir, Elm, Erlang, Fortran, Go, Groovy (partial support), Haskell, Java, Javascript, Julia, Kotlin, Lua, Markdown, MATLAB, Nix, Perl, PHP, PowerShell, Python, R, Ruby, Rust, Scala, Swift, TOML, TypeScript, YAML, and Zig. +With Serena's LSP library, we provide **support for over 40 programming languages**, including +AL, Ansible, Bash, C#, C/C++, Clojure, Dart, Elixir, Elm, Erlang, Fortran, F# (currently with some bugs), GLSL, Go, Groovy (partial support), Haskell, HLSL, Java, Javascript, Julia, Kotlin, Lean 4, Lua, Luau, Markdown, MATLAB, Nix, OCaml, Perl, PHP, PowerShell, Python, R, Ruby, Rust, Scala, Solidity, Swift, TOML, TypeScript, WGSL, YAML, and Zig. > [!IMPORTANT] > Some language servers require additional dependencies to be installed; see the [Language Support](https://oraios.github.io/serena/01-about/020_programming-languages.html) page for details. @@ -90,14 +89,13 @@ AL, Bash, C#, C/C++, Clojure, Dart, Elixir, Elm, Erlang, Fortran, Go, Groovy (pa ### The Serena JetBrains Plugin As an alternative to language servers, the [Serena JetBrains Plugin](https://plugins.jetbrains.com/plugin/28946-serena/) -leverages the powerful code analysis capabilities of your JetBrains IDE. -The plugin naturally supports all programming languages and frameworks that are supported by JetBrains IDEs, -including IntelliJ IDEA, PyCharm, Android Studio, WebStorm, PhpStorm, RubyMine, GoLand, CLion, and others. -Only Rider is not supported. +leverages the powerful code analysis capabilities of your JetBrains IDE. +The plugin naturally supports all programming languages and frameworks that are supported by JetBrains IDEs, +including IntelliJ IDEA, PyCharm, Android Studio, WebStorm, PhpStorm, RubyMine, GoLand, and potentially others (Rider and CLion are unsupported though). -The plugin offers the most robust and most powerful Serena experience. +The plugin offers the most robust and most powerful Serena experience. See our [documentation page](https://oraios.github.io/serena/02-usage/025_jetbrains_plugin.html) for further details and instructions. ## Quick Start diff --git a/docs/01-about/.gitignore b/docs/01-about/.gitignore new file mode 100644 index 00000000..e1fbb867 --- /dev/null +++ b/docs/01-about/.gitignore @@ -0,0 +1 @@ +/035_tools.md \ No newline at end of file diff --git a/docs/01-about/020_programming-languages.md b/docs/01-about/020_programming-languages.md index 97c7fc40..241e0b62 100644 --- a/docs/01-about/020_programming-languages.md +++ b/docs/01-about/020_programming-languages.md @@ -28,9 +28,15 @@ We currently provide direct, out-of-the-box support for the programming language Some languages require additional installations or setup steps, as noted. * **AL** +* **Ansible** + (experimental; requires Node.js and npm; automatically installs `@ansible/ansible-language-server`; + must be explicitly specified in the `languages` entry in the `project.yml`; requires `ansible` in PATH for full functionality) + the upstream `@ansible/ansible-language-server@1.2.3` supports hover, completion, definition, + semantic tokens, and validation; document symbols, workspace symbols, references, and rename + are not supported by this version) * **Bash** * **C#** - (by default, uses the Roslyn language server (language `csharp`), requires `pwsh` ([PowerShell 7+](https://learn.microsoft.com/en-us/powershell/scripting/install/install-powershell-on-windows?view=powershell-7.5)) on Windows; + (by default, uses the Roslyn language server (language `csharp`), requiring [.NET v10+](https://dotnet.microsoft.com/en-us/download/dotnet) and, on Windows, `pwsh` ([PowerShell 7+](https://learn.microsoft.com/en-us/powershell/scripting/install/install-powershell-on-windows?view=powershell-7.5)); set language to `csharp_omnisharp` to use OmiSharp instead) * **C/C++** (by default, uses the clangd language server (language `cpp`) but we also support ccls (language `cpp_ccls`); @@ -45,25 +51,36 @@ Some languages require additional installations or setup steps, as noted. * **Erlang** (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls); experimental, might be slow or hang) * **F#** - (requires .NET SDK 8.0+; uses FsAutoComplete/Ionide, which is auto-installed; for Homebrew .NET on macOS, set DOTNET_ROOT in your environment) + (requires [.NET v8.0+](https://dotnet.microsoft.com/en-us/download/dotnet); uses FsAutoComplete/Ionide, which is auto-installed; for Homebrew .NET on macOS, set DOTNET_ROOT in your environment) * **Fortran** (requires installation of fortls: `pip install fortls`) * **Go** (requires installation of `gopls`) * **Groovy** (requires local groovy-language-server.jar setup via `GROOVY_LS_JAR_PATH` or configuration) -* **Haskell** +* **Haskell** (automatically locates HLS via ghcup, stack, or system PATH; supports Stack and Cabal projects) +* **HLSL / GLSL / WGSL** + (uses [shader-language-server](https://github.com/antaalt/shader-sense) (language `hlsl`); automatically downloaded; + on macOS, requires Rust toolchain for building from source; + note: reference search is not supported by this language server) * **Java** -* **JavaScript** +* **JavaScript** + (supported via the TypeScript language server, i.e. use language `typescript` for both JavaScript and TypeScript) * **Julia** * **Kotlin** (uses the pre-alpha [official kotlin LS](https://github.com/Kotlin/kotlin-lsp), some issues may appear) +* **Lean 4** + (requires `lean` and `lake` installed via [elan](https://github.com/leanprover/elan); uses the built-in Lean 4 LSP; + the project must be a Lake project with `lake build` run before use) * **Lua** +* **Luau** * **Markdown** (must explicitly enable language `markdown`, primarily useful for documentation-heavy projects) * **Nix** (requires nixd installation) +* **OCaml** + (requires opam and ocaml-lsp-server to be installed manually; see the [OCaml Setup Guide](../03-special-guides/ocaml_setup_guide_for_serena.md)) * **Pascal** (uses Pascal/Lazarus, which is automatically downloaded; set `PP` and `FPCDIR` environment variables for source navigation) * **Perl** @@ -80,6 +97,9 @@ Some languages require additional installations or setup steps, as noted. (requires [rustup](https://rustup.rs/) - uses rust-analyzer from your toolchain) * **Scala** (requires some [manual setup](../03-special-guides/scala_setup_guide_for_serena); uses Metals LSP) +* **Solidity** + (experimental; requires Node.js and npm; automatically installs `@nomicfoundation/solidity-language-server`; + works best with a `foundry.toml` or `hardhat.config.js` in the project root) * **Swift** * **TypeScript** * **Vue** @@ -96,7 +116,8 @@ see Serena's [memory on that](https://github.com/oraios/serena/blob/main/.serena As an alternative to language servers, the [Serena JetBrains Plugin](https://plugins.jetbrains.com/plugin/28946-serena/) leverages the powerful code analysis capabilities of JetBrains IDEs. The plugin naturally supports all programming languages and frameworks that are supported by JetBrains IDEs, -including IntelliJ IDEA, PyCharm, Android Studio, AppCode, WebStorm, PhpStorm, RubyMine, GoLand, AppCode, CLion, and others. +including IntelliJ IDEA, PyCharm, Android Studio, WebStorm, PhpStorm, RubyMine, GoLand, and potentially others +(Rider and CLion are unsupported though). When using the plugin, Serena connects to an instance of your JetBrains IDE via the plugin. For users who already work in a JetBrains IDE, this means Serena seamlessly integrates with the IDE instance you typically have open anyway, @@ -120,4 +141,4 @@ of choice.

``` -See the [JetBrains Plugin documentation](../02-usage/025_jetbrains_plugin) for usage details. \ No newline at end of file +See the [JetBrains Plugin documentation](../02-usage/025_jetbrains_plugin) for usage details. diff --git a/docs/01-about/035_tools.md b/docs/01-about/035_tools.md deleted file mode 100644 index 293651e6..00000000 --- a/docs/01-about/035_tools.md +++ /dev/null @@ -1,49 +0,0 @@ -# List of Tools - -Find the full list of Serena's tools below (output of ` tools list --all`). - -Note that in most configurations, only a subset of these tools will be enabled simultaneously (see the section on [configuration](../02-usage/050_configuration) for details). -* `activate_project`: Activates a project based on the project name or path. -* `check_onboarding_performed`: Checks whether project onboarding was already performed. -* `create_text_file`: Creates/overwrites a file in the project directory. -* `delete_lines`: Deletes a range of lines within a file. -* `delete_memory`: Deletes a memory from Serena's project-specific memory store. -* `edit_memory`: -* `execute_shell_command`: Executes a shell command. -* `find_file`: Finds files in the given relative paths -* `find_referencing_symbols`: Finds symbols that reference the given symbol using the language server backend -* `find_symbol`: Performs a global (or local) search using the language server backend. -* `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. -* `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. -* `initial_instructions`: Provides instructions on how to use the Serena toolbox. - Should only be used in settings where the system prompt is not read automatically by the client. - - NOTE: Some MCP clients (including Claude Desktop) do not read the system prompt automatically! -* `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. -* `insert_at_line`: Inserts content at a given line in a file. -* `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. -* `jet_brains_find_referencing_symbols`: Finds symbols that reference the given symbol using the JetBrains backend -* `jet_brains_find_symbol`: Performs a global (or local) search for symbols using the JetBrains backend -* `jet_brains_get_symbols_overview`: Retrieves an overview of the top-level symbols within a specified file using the JetBrains backend -* `jet_brains_type_hierarchy`: Retrieves the type hierarchy (supertypes and/or subtypes) of a symbol using the JetBrains backend -* `list_dir`: Lists files and directories in the given directory (optionally with recursion). -* `list_memories`: Lists memories in Serena's project-specific memory store. -* `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). -* `open_dashboard`: Opens the Serena web dashboard in the default web browser. - The dashboard provides logs, session information, and tool usage statistics. -* `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context). -* `read_file`: Reads a file within the project directory. -* `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. -* `remove_project`: Removes a project from the Serena configuration. -* `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities. -* `replace_content`: Replaces content in a file (optionally using regular expressions). -* `replace_lines`: Replaces a range of lines within a file with new content. -* `replace_symbol_body`: Replaces the full definition of a symbol using the language server backend. -* `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. -* `search_for_pattern`: Performs a search for a pattern in the project. -* `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. -* `switch_modes`: Activates modes by providing a list of their names -* `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. -* `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. -* `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed. -* `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. \ No newline at end of file diff --git a/docs/02-usage/020_running.md b/docs/02-usage/020_running.md index 6a126368..3b82a73b 100644 --- a/docs/02-usage/020_running.md +++ b/docs/02-usage/020_running.md @@ -132,7 +132,7 @@ to use such a launch command. (streamable-http)= ### Streamable HTTP Mode -When using instead the *Streamable HTTP* mode, you control the server lifecycle yourself, +When using *Streamable HTTP* mode, you control the server lifecycle yourself, i.e. you start the server and provide the client with the URL to connect to it. Simply provide `start-mcp-server` with the `--transport streamable-http` option and optionally provide the desired port @@ -147,7 +147,14 @@ you would run and then configure your client to connect to `http://localhost:9121/mcp`. -Note that while the legacy SSE transport is also supported (via `--transport sse` with corresponding /sse endpoint), its use is discouraged. +**When to use.** Note that Serena is a stateful MCP server, and only one coding project can be active at a time. +Therefore, starting a single Serena instance and connecting it to multiple clients is only +appropriate if all clients will be working on the same project. +If you want several agents to work on different projects, making each client/agent start its own server +in stdio mode is likely the best option. +See section [The Project Workflow](040_workflow) for more information on how to manage projects in Serena. + +The legacy SSE transport is also supported (via `--transport sse` with corresponding /sse endpoint), its use is discouraged. (mcp-args)= ### MCP Server Command-Line Arguments diff --git a/docs/02-usage/040_workflow.md b/docs/02-usage/040_workflow.md index c5153401..f8b3603f 100644 --- a/docs/02-usage/040_workflow.md +++ b/docs/02-usage/040_workflow.md @@ -36,7 +36,7 @@ For instance, when using `uvx`, run * For an existing project, the main programming language will be detected automatically, but you can choose to explicitly specify multiple languages by passing the `--language` parameter multiple times (e.g. `--language python --language typescript`). - * You can optionally specify a custom project name with `--name "My Project"`. + * You can optionally specify a custom project name with `--name my-name`. * You can immediately index the project after creation with `--index`. (project-config)= @@ -57,7 +57,17 @@ The file allows you to configure ... * the set of tools and modes to use by default For detailed information on the parameters and possible settings, see the -[template file](https://github.com/oraios/serena/blob/main/src/serena/resources/project.template.yml). +[template file](https://github.com/oraios/serena/blob/main/src/serena/resources/project.template.yml). + +:::{note} +Many settings in project.yml *extend* or *override* settings in the global configuration file `serena_config.yml`. +So use the project configuration specifically for aspects that apply only to the particular project. +::: + +**Local Overrides**. The project.yml file is intended to be versioned together with the project. +You can specify local overrides for the settings in a `project.local.yml` file in the same directory +(which, by default, is ignored by git). +Any keys defined therein will override the respective key in `project.yml`. (indexing)= ### Indexing @@ -103,22 +113,12 @@ By default, Serena will perform an **onboarding process** when it is started for the first time for a project. The goal of the onboarding is for Serena to get familiar with the project and to store memories, which it can then draw upon in future interactions. -If an LLM should fail to complete the onboarding and does not actually write the -respective memories to disk, you may need to ask it to do so explicitly. -The onboarding will usually read a lot of content from the project, thus filling -up the context. It can therefore be advisable to switch to another conversation -once the onboarding is complete. -After the onboarding, we recommend that you have a quick look at the memories and, -if necessary, edit them or add additional ones. +In general, **memories** provide a way for Serena to store and retrieve +information about the project, relevant conventions, and other relevant aspects. -**Memories** are files stored in `.serena/memories/` in the project directory, -which the agent can choose to read in subsequent interactions. -Feel free to read and adjust them as needed; you can also add new ones manually. -Every file in the `.serena/memories/` directory is a memory file. -Whenever Serena starts working on a project, the list of memories is -provided, and the agent can decide to read them. -We found that memories can significantly improve the user experience with Serena. +For more information on this, including how to manage +or disable these features, see [Memories & Onboarding](045_memories). ## Preparing Your Project @@ -163,4 +163,55 @@ linting results, and test results to assess the correctness of its actions. Therefore, software that is designed to meaningful interpretable outputs (e.g. log messages) and that has a good test coverage is much easier to work with for Serena. -We generally recommend to start an editing task from a state where all linting checks and tests pass. \ No newline at end of file +We generally recommend to start an editing task from a state where all linting checks and tests pass. + +## Multiple Projects, Multiple Agents + +There are several ways in which you might want to work with multiple projects simultaneously. + +### A Single Agent Editing Multiple Projects Simultaneously + +If fulfilling a task requires a single agent to edit code in multiple projects, the recommended approach is to create a **monorepo folder**, +i.e. a folder that contains all the projects as sub-folders, and open that monorepo folder as a project in Serena. +You may also use symbolic links to create a monorepo folder if the projects are located in different places on your filesystem. + +If several languages are used across the projects, specify all of them as needed when using the LSP backend; +For JetBrains mode, make sure that your IDE is configured to work with all the languages used across the projects (e.g. by installing the respective language plugins). + +(query-projects)= +### Reading from External Projects + +If, while working on a project, you want Serena to be able to read code or other information from another project (e.g. a library or otherwise related project), +this can be enabled via the `query_project` tool. +Provided that the project you want to query is known to Serena (i.e. you have created it as described above), +the `query_project` tool allows the agent to query files and symbolic information from that project. + +To enable this tool, [activate the mode](modes) `query-projects`. +This also enables a second tool for listing projects that can be queried. + +Depending on the language backend being used, the management of resources for the external projects varies: + +* When using the JetBrains backend, make sure that every project for which you want symbolic queries to work is open in an IDE instance. +* When using the LSP backend, executing symbolic tools via the query tool requires that Serena's **Project Server** be started, + which will automatically spawn the necessary language servers for the projects that are queried. + + To start the server, run + + start-project-server + + where `` is your way of running Serena. For example, when using `uvx`, run + + uvx --from git+https://github.com/oraios/serena serena start-project-server + +### Multiple Agents Accessing a Single Serena Instance + +If you want multiple agents to access the same project via a single Serena instance, +i.e. you do not want several instances of Serena (including its language servers) to be running, +you can achieve this by [starting the Serena MCP server in HTTP mode](streamable-http) +and connecting all client agents to the same HTTP endpoint. +The agents will then share the resources of the single Serena instance. + +### Multiple Agents Working on Different Projects + +For this use case, simply run a separate instance of Serena for each project, which naturally +occurs when Serena is started by the MCP client in stdio mode. \ No newline at end of file diff --git a/docs/02-usage/045_memories.md b/docs/02-usage/045_memories.md new file mode 100644 index 00000000..7b364aa9 --- /dev/null +++ b/docs/02-usage/045_memories.md @@ -0,0 +1,101 @@ +# Memories & Onboarding + +Serena provides the functionality of a fully featured agent, and a useful aspect of this is Serena's memory system. +Despite its simplicity, we received positive feedback from many users who tend to combine it with their +agent's internal memory management (e.g., `AGENTS.md` files). + +## Memories + +Memories are simple, human-readable Markdown files that both you and +your agent can create, read, and edit. + +Serena differentiates between + * **project-specific memories**, which are stored in the `.serena/memories/` directory within your project folder, and + * **global memories**, which are shared across all projects and, by default, are stored in `~/.serena/memories/global/` + +The LLM is informed about the existence of memories and instructed to read them when appropriate, +inferring appropriateness from the file name. +When the agent starts working on a project, it receives the list of available memories. +The agent should be instructed to update memories by the user when appropriate. + +### Organizing Memories + +Memories can be organized into **topics** by using `/` in the memory name (e.g. `modules/frontend`). +The structure is mapped to the file system, where topics correspond to subdirectories. +The `list_memories` tool can filter by topic, allowing the agent to explore even large numbers of memories in a structured way. + +(global-memories)= +### Global Memories + +Global memories use the top-level topic `global`, i.e. whenever a memory name starts with `global/`, +it is stored in the global memories directory and is shared across all projects. + +By default, deletion and editing of global memories is allowed. +If you want to protect them from accidental modification by the agent, +you can add regex patterns to `read_only_memory_patterns` in your global or +project-level [configuration](050_configuration). For example, setting "global/.*" will mark all global memories as read-only. The agent will be informed which memories are read-only. + +Since global memories are not versioned alongside your project files, +it can be helpful to track global memories with git (i.e. to make `~/.serena/memories/` a git repository) +in order to have a history of changes and the possibility to revert them if needed. + +### Ignoring Memories + +Projects that accumulate large numbers of archived memory files can use `ignored_memory_patterns` +to exclude them from `list_memories` and `activate_project` output. Add regex patterns to the +global or project-level [configuration](050_configuration): + +```yaml +ignored_memory_patterns: ["_archive/.*", "_episodes/.*"] +``` + +Ignored memories are completely excluded — they cannot be accessed via `read_memory`, `write_memory`, +or any other memory tool. To read an ignored memory file, use the `read_file` tool on the raw file path +(e.g., `.serena/memories/_archive/2026-03/some-topic.md`). + +Like `read_only_memory_patterns`, patterns from the global and project-level configurations are merged additively. + +### Manually Editing Memories + +You may edit memories directly in the file system, using your preferred text editor or IDE. +Alternatively, access them via the [Serena Dashboard](060_dashboard), which provides a graphical interface for +viewing, creating, editing, and deleting memories while Serena is running. + +(onboarding)= +## Onboarding + +By default, Serena performs an **onboarding process** when it encounters a project +for the first time (i.e., when no project memories exist yet). +The goal of the onboarding is for Serena to get familiar with the project — +its structure, build system, testing setup, and other essential aspects — +and to store this knowledge as memories for future interactions. + +In further project activations, Serena will check whether onboarding was already +performed by looking for existing project memories and will skip the onboarding +process if memories are found. + +### How Onboarding Works + +1. When a project is activated, Serena checks whether onboarding was already + performed (by checking if any memories exist). +2. If no memories are found, Serena triggers the onboarding process, which + reads key files and directories to understand the project. +3. The gathered information is written into project-specific memory files (see above). + +### Tips for Onboarding + +- **Context usage**: The onboarding process will read a lot of content from the project, + filling up the context window. It is therefore advisable to **switch to a new conversation** + once the onboarding is complete. +- **LLM failures**: If an LLM fails to complete the onboarding and does not actually + write the respective memories to disk, you may need to ask it to do so explicitly. +- **Review the results**: After onboarding, we recommend having a quick look at the + generated memories and editing them or adding new ones as needed. + +## Disabling Memories and Onboarding + +If you do not require the functionality described in this section, you can selectively disable it. + + * To disable all memory related tools (including onboarding), adding `no-memories` to the `base_modes` + in Serena's [global configuration](050_configuration). + * Similarly, to disable only onboarding, add `no-onboarding` to the `base_modes`. diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md index aaf256af..4f52f1ec 100644 --- a/docs/02-usage/050_configuration.md +++ b/docs/02-usage/050_configuration.md @@ -33,6 +33,9 @@ Some of the configurable settings include: * logging settings * advanced settings specific to individual language servers (see [below](ls-specific-settings)) +The global configuration settings apply to all projects. +Some of the settings it contains can, however, be *extended* or *overridden* in project-specific settings, contexts and modes. + For detailed information on the parameters and possible settings, see the [template file](https://github.com/oraios/serena/blob/main/src/serena/resources/serena_config.template.yml). @@ -113,7 +116,8 @@ Examples of built-in modes include: * `one-shot`: Configures Serena for tasks that should be completed in a single response, often used with `planning` for generating reports or initial plans. * `no-onboarding`: Skips the initial onboarding process if it's not needed for a particular session but retains the memory tools (assuming initial memories were created externally). * `onboarding`: Focuses on the project onboarding process. -* `no-memories`: Disables all memory tools (and tools building on memories such as onboarding tools) +* `no-memories`: Disables all memory tools (and tools building on memories such as onboarding tools) +* `query-projects`: Enables tools for querying other Serena projects (without activating them); see section [Reading from External Projects](query-projects) Find the concrete definitions of these modes [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/modes). @@ -172,6 +176,35 @@ For advanced users, Serena's configuration can be further customized. The Serena user data directory (where configuration, language server files, logs, etc. are stored) defaults to `~/.serena`. You can change this location by setting the `SERENA_HOME` environment variable to your desired path. +### Per-Project Serena Folder Location + +By default, each project stores its Serena data (memories, caches, etc.) in a `.serena` folder inside the project root. +You can customize this location globally via the `project_serena_folder_location` setting in `serena_config.yml`. + +The setting supports two placeholders: + +| Placeholder | Description | +|----------------------|-------------------------------------------------| +| `$projectDir` | The absolute path to the project root directory | +| `$projectFolderName` | The name of the project folder | + +**Examples:** + +```yaml +# Default: data stored inside the project directory +project_serena_folder_location: "$projectDir/.serena" + +# Central location: all project data under a shared directory +project_serena_folder_location: "/projects-metadata/$projectFolderName/.serena" +``` + +When a project is loaded, Serena uses the following fallback logic: +1. Check if a `.serena` folder exists at the configured path. +2. If not, check if one exists in the project root (default/legacy location). +3. If neither exists, create the folder at the configured path. + +This ensures backward compatibility: existing projects that already have a `.serena` folder in the project root will continue to work, even after changing the `project_serena_folder_location` setting. + (ls-specific-settings)= ### Language Server-Specific Settings @@ -180,8 +213,10 @@ You can change this location by setting the `SERENA_HOME` environment variable t Most users will not need to adjust these settings. ::: -Under the key `ls_specific_settings` in `serena_config.yml`, you can you pass per-language, -language server-specific configuration. +Under the key `ls_specific_settings` in `serena_config.yml`, you can you pass global per-language, +language server-specific configuration. You can use the same key in the project configuration files (`project.yml` +and `project.local.yml` ) to override or extend the global settings for a specific project. +The settings are merged on top-level, meaning that project-level settings for a language will replace global settings for the same language. Structure: @@ -197,6 +232,7 @@ Most settings are currently undocumented. Please refer to the implementation to determine supported settings. ::: +(override-ls-path)= #### Overriding the Language Server Path Some language servers, particularly those that use a single core path for the language server (e.g. the main executable), @@ -322,6 +358,29 @@ ls_specific_settings: jvm_options: "-Xmx4G -XX:+UseG1GC" # JVM options (default: -Xmx2G). Set to "" to disable. ``` +#### Luau + +Serena uses [`luau-lsp`](https://github.com/JohnnyMorganz/luau-lsp) for Luau support. + +**Runtime Requirements:** + +- `luau-lsp` is used from PATH if available. +- Otherwise, Serena downloads the pinned `luau-lsp` release for the current platform. + +**Configuration:** + +```yaml +ls_specific_settings: + luau: + ls_path: "/path/to/luau-lsp" # Optional: override the language server executable + platform: "roblox" # "roblox" (default) or "standard" + roblox_security_level: "PluginSecurity" # Roblox only: None, PluginSecurity, LocalUserSecurity, RobloxScriptSecurity +``` + +Notes: +- In `roblox` mode, Serena downloads Roblox definitions and Roblox API docs and passes them to `luau-lsp`. +- In `standard` mode, Serena skips Roblox definitions and only downloads the standard Luau docs bundle. + #### Pascal (`pasls`) Serena uses [pasls](https://github.com/genericptr/pascal-language-server) (Pascal Language Server) for Pascal/Free Pascal support. diff --git a/docs/03-special-guides/cpp_setup.md b/docs/03-special-guides/cpp_setup.md index 916040d4..c54f2105 100644 --- a/docs/03-special-guides/cpp_setup.md +++ b/docs/03-special-guides/cpp_setup.md @@ -15,6 +15,8 @@ for cross-file reference finding, see below for details. Your project must have a `compile_commands.json` file at the repository root. This file is essential for correct parsing and cross-file reference finding. +You can use a specific clangd or ccls installation (e.g., a custom build or a version provided by your project), +by specifying the path in your configuration, see [ls-specific-settings](ls-specific-settings) for details. ## compile_commands.json Requirements @@ -27,7 +29,7 @@ For reliable cross-file reference finding with clangd, your `compile_commands.js ### With clangd -Serena automatically downloads and manages clangd. Since clangd does not properly work with relative paths in `compile_commands.json`, +By default, Serena automatically downloads and manages clangd. Since clangd does not properly work with relative paths in `compile_commands.json`, Serena will detect them and transform them into absolute paths automatically (writing a new `compile_commands.json` file), if needed. #### Customizing the Compilation Database Location diff --git a/docs/03-special-guides/ocaml_setup_guide_for_serena.md b/docs/03-special-guides/ocaml_setup_guide_for_serena.md new file mode 100644 index 00000000..068fd057 --- /dev/null +++ b/docs/03-special-guides/ocaml_setup_guide_for_serena.md @@ -0,0 +1,88 @@ +# OCaml Setup Guide for Serena + +This guide explains how to set up an OCaml project so that Serena can provide code intelligence via ocaml-lsp-server (ocamllsp). + +Unlike some other languages, Serena does not download the OCaml language server automatically. You must install it yourself via opam, as OCaml tooling is compiled from source against your specific environment. + +--- +## Prerequisites + +Install the following on your system and ensure they are available on `PATH`: + +- **opam** (OCaml package manager) + - macOS: `brew install opam` + - Ubuntu/Debian: `sudo apt install opam` + - Fedora: `sudo dnf install opam` + - Other: https://opam.ocaml.org/doc/Install.html +- **OCaml compiler** (via opam) + - OCaml < 5.1 or >= 5.1.1 (OCaml 5.1.0 is **not supported** by ocaml-lsp-server) + - Recommended: OCaml 4.14.x (stable) or 5.2+ (for cross-file references) +- **ocaml-lsp-server** (via opam) +- **dune** (build system, via opam) + +--- +## Installation + +1. Initialize opam if you haven't already: + ```bash + opam init + eval $(opam env) + ``` + +2. Create an opam switch with a compatible OCaml version: + ```bash + # For cross-file reference support (recommended) + opam switch create serena-ocaml ocaml-base-compiler.5.2.1 + eval $(opam env) + + # Or for stable OCaml 4.14.x + opam switch create serena-ocaml ocaml-base-compiler.4.14.2 + eval $(opam env) + ``` + +3. Install the language server and build tools: + ```bash + opam install ocaml-lsp-server dune + ``` + +4. Verify the installation: + ```bash + opam exec -- ocamllsp --version + opam exec -- ocaml -version + ``` + +--- +## Cross-File References + +Cross-file reference support (finding all usages of a symbol across your project) requires: + +- OCaml >= 5.2 +- ocaml-lsp-server >= 1.23.0 +- dune >= 3.16.0 + +When these requirements are met, Serena automatically builds the cross-file index during startup via `dune build @ocaml-index`. Without these versions, references are limited to the current file. + +--- +## Using Serena with OCaml + +- Serena automatically detects OCaml files (`*.ml`, `*.mli`) and Reason files (`*.re`, `*.rei`). +- The language server is started via `opam exec -- ocamllsp`, so your opam environment must be configured. +- Ensure your project builds successfully with `dune build` before using Serena for best results. + +--- +## Troubleshooting + +| Problem | Solution | +|---------|----------| +| "opam not found" | Install opam and add it to PATH | +| "OCaml 5.1.0 is incompatible" | Create a new switch: `opam switch create ocaml-base-compiler.5.2.1` | +| "ocaml-lsp-server not found" | `opam install ocaml-lsp-server` | +| Cross-file refs not working | Ensure OCaml >= 5.2 and ocaml-lsp-server >= 1.23.0; run `dune build` first | +| Stale index | Rebuild with `dune build @ocaml-index` | + +--- +## Reference + +- opam: https://opam.ocaml.org +- ocaml-lsp-server: https://github.com/ocaml/ocaml-lsp +- Project-wide occurrences: https://discuss.ocaml.org/t/ann-project-wide-occurrences-in-merlin-and-lsp/14847 diff --git a/docs/_config.yml b/docs/_config.yml index 89a0f6c2..999c6a62 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -12,7 +12,7 @@ title : Serena Documentation # The title of the book. Wil author : Oraios AI & Oraios Software # The author of the book copyright : "2025 by Serena contributors" # Copyright year to be placed in the footer # Patterns to skip when building the book. Can be glob-style (e.g. "*skip.ipynb") -exclude_patterns : ['**.ipynb_checkpoints', '.DS_Store', 'Thumbs.db', '_build', 'jupyter_execute', '.jupyter_cache', '.pytest_cache', 'docs/autogen_rst.py', 'docs/create_toc.py'] +exclude_patterns : ['**.ipynb_checkpoints', '.DS_Store', 'Thumbs.db', '_build', 'jupyter_execute', '.jupyter_cache', '.pytest_cache', 'docs/autogen_docs.py', 'docs/create_toc.py'] # Auto-exclude files not in the toc only_build_toc_files : true diff --git a/docs/autogen_rst.py b/docs/autogen_docs.py similarity index 80% rename from docs/autogen_rst.py rename to docs/autogen_docs.py index e59c2f70..5b4ca422 100644 --- a/docs/autogen_rst.py +++ b/docs/autogen_docs.py @@ -4,6 +4,8 @@ import shutil from pathlib import Path from typing import Optional, List +from sensai.util.string import TextBuilder + log = logging.getLogger(os.path.basename(__file__)) TOP_LEVEL_PACKAGE = "serena" @@ -143,10 +145,38 @@ def make_rst(src_root, rst_root, clean=False, overwrite=False, package_prefix="" write_to_file(module_template(module_qualname), module_rst_path) +def autogen_tool_list(target_filename = "01-about/035_tools.md"): + from serena.tools import ToolRegistry + + target_file = Path(__file__).parent / target_filename + with open(target_file, "w") as f: + f.write("\n\n") + f.write("# Tools\n\n") + f.write("Find the full list of Serena's tools below. \n") + f.write("Note that in most configurations, only a subset of these tools will be enabled simultaneously.\n") + f.write("Tools marked as optional are disabled by default.\n\n") + tools_by_module = ToolRegistry().get_registered_tools_by_module() + priority_modules = {"serena.tools.symbol_tools": 1, "serena.tools.jetbrains_tools": 2} + + text = TextBuilder() + sorted_modules = sorted(tools_by_module.keys(), key=lambda m: (priority_modules.get(m, 3), m)) + for module in sorted_modules: + tools = tools_by_module[module] + module = module.replace("serena.tools.", "") + text.with_line(f"* **{module}**") + for tool in tools: + info = " *(optional)*" if tool.is_optional else "" + text.with_line(f"* `{tool.tool_name}`{info}: {tool.class_docstring}", indent=2) + f.write(text.build()) + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) docs_root = Path(__file__).parent enable_module_docs = False + + autogen_tool_list() + if enable_module_docs: make_rst( docs_root / ".." / "src" / "serena", diff --git a/flake.nix b/flake.nix index e311aa0d..e8d9ccfa 100644 --- a/flake.nix +++ b/flake.nix @@ -77,7 +77,16 @@ packages = { serena-env = pythonSet.mkVirtualEnv "serena-env" workspace.deps.default; - serena = pkgs.runCommand "serena" {} '' + serena = pkgs.runCommand "serena" { + meta = { + description = "A powerful coding agent toolkit providing semantic retrieval and editing capabilities (MCP server & Agno integration)"; + homepage = "https://oraios.github.io/serena"; + changelog = "https://github.com/oraios/serena/blob/main/CHANGELOG.md"; + mainProgram = "serena"; + license = pkgs.lib.licenses.mit; + platforms = lib.platforms.all; + }; + } '' mkdir -p $out/bin ln -s ${packages.serena-env}/bin/serena $out/bin/serena ''; diff --git a/pyproject.toml b/pyproject.toml index f4ad1be8..93aba77d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,36 +8,44 @@ version = "0.1.4" description = "" authors = [{ name = "Oraios AI", email = "info@oraios-ai.de" }] readme = "README.md" -requires-python = ">=3.11, <3.12" +requires-python = ">=3.11, <3.15" classifiers = [ "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.11", ] dependencies = [ - "requests>=2.32.3,<3", - "pyright>=1.1.396,<2", - "fortls>=3.2.2", - "overrides>=7.7.0,<8", - "python-dotenv>=1.0.0, <2", - "mcp==1.23.0", - "flask>=3.0.0", - "sensai-utils>=1.5.0", - "pydantic>=2.10.6", - "types-pyyaml>=6.0.12.20241230", - "pyyaml>=6.0.2", + "requests==2.32.5", + "pyright==1.1.403", + "fortls==3.2.2", + "overrides==7.7.0", + "python-dotenv==1.2.1", + "mcp==1.26.0", + "flask==3.1.3", # bumped from 3.1.1 for CVE fix (also fixes werkzeug alert) + "sensai-utils==1.5.0", + "pydantic==2.12.5", + "types-pyyaml==6.0.12.20250516", + "pyyaml==6.0.2", "ruamel.yaml==0.18.14", - "jinja2>=3.1.6", - "dotenv>=0.9.9", - "pathspec>=0.12.1", - "psutil>=7.0.0", - "docstring_parser>=0.16", - "joblib>=1.5.1", - "tqdm>=4.67.1", - "tiktoken>=0.9.0", - "anthropic>=0.54.0", - "beautifulsoup4>=4.14.2", + "jinja2==3.1.6", + "dotenv==0.9.9", + "pathspec==0.12.1", + "psutil==7.0.0", + "docstring_parser==0.17.0", + "joblib==1.5.1", + "tqdm==4.67.1", + "tiktoken==0.12.0", + "anthropic==0.59.0", + "beautifulsoup4==4.14.2", "pywebview", "pystray>=0.19.5", + # Transitive deps pinned for security (dependabot alerts). + # Exact pins because uvx installs from git, ignoring the lock file. + "urllib3==2.6.3", + "werkzeug==3.1.7", + "starlette==1.0.0", + "python-multipart==0.0.22", + "filelock==3.25.2", + "cryptography==46.0.5", ] [tool.uv.sources] @@ -59,29 +67,35 @@ text = "MIT" [project.optional-dependencies] dev = [ - "black[jupyter]>=23.7.0, <26", # black 26 is incompatible with our pathspec version - "jinja2", + "jinja2==3.1.6", # In version 1.0.4 we get a NoneType error related to some config conversion (yml_analytics is None and should be a list) - "mypy>=1.16.1", - "poethepoet>=0.20.0", - "pytest>=8.0.2", - "pytest-xdist>=3.5.0", - "ruff>=0.0.285", - "toml-sort>=0.24.2", - "types-pyyaml>=6.0.12.20241230", - "syrupy>=4.9.1", - "types-requests>=2.32.4.20241230", + "mypy==1.17.0", + "poethepoet==0.36.0", + "pytest==8.4.1", + "pytest-xdist==3.8.0", + "ruff==0.12.5", + "toml-sort==0.24.2", + "types-pyyaml==6.0.12.20250516", + "syrupy==4.9.1", + "types-requests==2.32.4.20250809", # docs - "sphinx>=7,<8", - "sphinx_rtd_theme>=0.5.1", + "sphinx==7.4.7", + "sphinx_rtd_theme==2.0.0", "sphinx-toolbox==3.7.0", - "jupyter-book>=1,<2", - "nbsphinx", - "pyinstrument", - "pytest-timeout>=2.4.0", + "jupyter-book==1.0.4.post1", + "nbsphinx==0.9.7", + "pyinstrument==5.1.1", + "pytest-timeout==2.4.0", + # Transitive dev deps pinned for security (dependabot alerts). + # Exact pins because uvx installs from git, ignoring the lock file. + "tornado==6.5.5", + "nbconvert==7.17.0", + "wheel==0.46.3", + "pyasn1==0.6.3", + "PyJWT==2.10.1", ] -agno = ["agno>=2.2.1", "sqlalchemy>=2.0.40"] -google = ["google-genai>=1.8.0"] +agno = ["agno==2.5.10", "sqlalchemy==2.0.41"] # agno bumped for session state overwrite CVE +google = ["google-genai==1.27.0"] [project.urls] Homepage = "https://github.com/oraios/serena" @@ -89,15 +103,6 @@ Homepage = "https://github.com/oraios/serena" [tool.hatch.build.targets.wheel] packages = ["src/serena", "src/interprompt", "src/solidlsp"] -[tool.black] -line-length = 140 -target-version = ["py311"] -exclude = ''' -/( - src/solidlsp/language_servers/.*/static|src/multilspy -)/ -''' - [tool.doc8] max-line-length = 1000 @@ -140,22 +145,22 @@ type = "simple" # For custom markers, one can either adjust the env var or just use -m option in the command line, # as the second -m option will override the first one. test = "pytest test -vv" -_black_check = "black --check src scripts test" _ruff_check = "ruff check src scripts test" -_black_format = "black src scripts test" -_ruff_format = "ruff check --fix src scripts test" -lint = ["_black_check", "_ruff_check"] -format = ["_ruff_format", "_black_format"] +_ruff_format_check = "ruff format --check src scripts test" +_ruff_fix = "ruff check --fix src scripts test" +_ruff_format = "ruff format src scripts test" +lint = ["_ruff_format_check", "_ruff_check"] +format = ["_ruff_fix", "_ruff_format"] _mypy_core = "mypy src/serena src/solidlsp" _mypy_test = "mypy --disable-error-code no-untyped-def test" type-check = ["_mypy_core", "_mypy_test"] # docs -_autogen_rst = "python docs/autogen_rst.py" +_autogen_docs = "python docs/autogen_docs.py" _sphinx_build = "sphinx-build -b html docs docs/_build -W --keep-going" _jb_generate_toc = "python docs/create_toc.py" _jb_generate_config = "jupyter-book config sphinx docs/" doc-clean = "rm -rf docs/_build docs/03_api" -doc-generate-files = ["_autogen_rst", "_jb_generate_toc", "_jb_generate_config"] +doc-generate-files = ["_autogen_docs", "_jb_generate_toc", "_jb_generate_config"] doc-build = ["doc-clean", "doc-generate-files", "_sphinx_build"] [tool.ruff] @@ -312,9 +317,11 @@ markers = [ "ruby: language server running for Ruby (uses ruby-lsp)", "zig: language server running for Zig", "lua: language server running for Lua", + "luau: language server running for Luau", "nix: language server running for Nix", "dart: language server running for Dart", "erlang: language server running for Erlang", + "ocaml: language server running for OCaml and Reason", "scala: language server running for Scala", "al: language server running for AL (Microsoft Dynamics 365 Business Central)", "fsharp: language server running for F#", @@ -331,11 +338,15 @@ markers = [ "toml: language server running for TOML", "matlab: language server running for MATLAB (requires MATLAB R2021b+)", "systemverilog: language server running for SystemVerilog (uses verible-verilog-ls)", + "hlsl: language server running for HLSL shaders (uses shader-language-server)", + "lean4: language server running for Lean 4", + "solidity: language server running for Solidity (uses @nomicfoundation/solidity-language-server)", + "ansible: language server running for Ansible (uses @ansible/ansible-language-server)", ] [tool.codespell] # Ref: https://github.com/codespell-project/codespell#using-a-config-file skip = '.git*,*.svg,*.lock,*.min.*' check-hidden = true -# ignore-regex = '' +ignore-regex = '\.\w+' ignore-words-list = 'paket' diff --git a/roadmap.md b/roadmap.md deleted file mode 100644 index 4b936b30..00000000 --- a/roadmap.md +++ /dev/null @@ -1,63 +0,0 @@ -# Roadmap - -This document gives an overview of the ongoing and future development of Serena. -If you have a proposal or want to discuss something, feel free to open a discussion -on Github. For a summary of the past development, see the [changelog](/CHANGELOG.md). - -Want to see us reach our goals faster? You can help out with an issue, start a discussion, or -inform us about funding opportunities so that we can devote more time to the project. - -## Overall Goals - -Serena has the potential to be the go-to tool for most LLM coding tasks, since it is -unique in its ability to be used as MCP Server in any kind of environment -while still being a capable agent. We want to achieve the following goals in terms of functionality: - -1. Top performance (comparable to API-based coding agents) when used through official (free) clients like Claude Desktop. -1. Lowering API costs and potentially improving performance of coding clients (Claude Code, Codex, Cline, Roo, Cursor/Windsurf/VSCode etc). -1. Transparency and simplicity of use. Achieved through the dashboard/logging GUI. -1. Integrations with major frameworks that don't accept MCP. Usable as a library. - -Apart from the functional goals, we have the goal of having great code design, so that Serena can be viewed -as a reference for how to implement MCP Servers. Such projects are an emerging technology, and -best practices are yet to be determined. We will share our experiences in [lessons learned](/lessons_learned.md). - - -## Immediate/Ongoing - -- Support for projects using multiple programming languages. -- Evaluate whether `ReplaceLinesTool` can be removed in favor of a more reliable and performant editing approach. -- Generally experiment with various approaches to editing tools -- Manual evaluation on selected tasks from SWE-verified -- Manual evaluation of cost-lowering and performance when used within popular non-MCP agents -- Improvements in prompts, in particular giving examples and extending modes and contexts - -## Upcoming - -- Publishing Serena as a package that can also be used as library -- Use linting and type-hierarchy from the LSP in tools -- Tools for refactoring (rename, move) - speculative, maybe won't do this. -- Tracking edits and rolling them back with the dashboard -- Improve configurability and safety of shell tool. Maybe autogeneration of tools from a list of commands and descriptions. -- Transparent comparison with DesktopCommander and ... -- Automatic evaluation using OpenHands, submission to SWE-Bench -- Evaluation whether incorporating other MCPs increases performance or usability (memory bank is a candidate) -- More documentation and best practices - -## Stretch - -- Allow for sandboxing and parallel instances of Serena, maybe use openhands or codex for that -- Incorporate a verifier model or generally a second model (maybe for applying edits) as a tool. -- Building on the above, allow for the second model itself to be reachable through an MCP server, so it can be used for free -- Tracking edits performed with shell tools - -## Beyond Serena - -The technologies and approaches taken in Serena can be used for various research and service ideas. Some thought that we had are: - -- PR and issue assistant working with GitHub, similar to how [OpenHands](https://github.com/All-Hands-AI/OpenHands) - and [qodo](https://github.com/qodo-ai/pr-agent) operate. Should be callable through @serena -- Tuning a coding LLM with Serena's tools with RL on one-shot tasks. We would need compute-funding for that -- Develop a web app to quantitatively compare the performance of various agents by scraping PRs and manually crafted metadata. - The main metric for coding agents should be *developer experience*, and that is hard to grasp and is poorly correlated with - performance on current benchmarks. \ No newline at end of file diff --git a/scripts/print_language_list.py b/scripts/print_language_list.py index a70acc2e..6a156731 100644 --- a/scripts/print_language_list.py +++ b/scripts/print_language_list.py @@ -7,7 +7,7 @@ from solidlsp.ls_config import Language if __name__ == "__main__": lang_strings = sorted([l.value for l in Language]) max_len = max(len(s) for s in lang_strings) - fmt = f"%-{max_len+2}s" + fmt = f"%-{max_len + 2}s" for i, l in enumerate(lang_strings): if i % 5 == 0: print("\n# ", end="") diff --git a/src/serena/agent.py b/src/serena/agent.py index 73202d35..b495fa59 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -7,14 +7,15 @@ import os import platform import subprocess import sys -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager from logging import Logger from typing import TYPE_CHECKING, Optional, TypeVar import webview from sensai.util import logging from sensai.util.logging import LogTime -from sensai.util.string import TextBuilder +from sensai.util.string import dict_string from interprompt.jinja_template import JinjaTemplate from serena import serena_version @@ -30,8 +31,8 @@ from serena.config.serena_config import ( ToolInclusionDefinition, ) from serena.dashboard import SerenaDashboardAPI, SerenaDashboardViewer -from serena.ls_manager import LanguageServerManager, LanguageServerManagerInitialisationError -from serena.project import Project +from serena.ls_manager import LanguageServerManager +from serena.project import MemoriesManager, Project from serena.prompt_factory import SerenaPromptFactory from serena.task_executor import TaskExecutor from serena.tools import ActivateProjectTool, GetCurrentConfigTool, OpenDashboardTool, ReplaceContentTool, Tool, ToolMarker, ToolRegistry @@ -256,7 +257,6 @@ class SerenaAgent: # project-specific instances, which will be initialized upon project activation self._active_project: Project | None = None - self._lsp_init_error: LanguageServerManagerInitialisationError | None = None # determine registered project to be activated (if any) registered_project_to_activate: RegisteredProject | None = ( @@ -315,7 +315,7 @@ class SerenaAgent: # log fundamental information log.info( f"Starting Serena server (version={serena_version()}, process id={os.getpid()}, parent process id={os.getppid()}; " - f"language backend={self.serena_config.language_backend.name})" + f"language backend={self.serena_config.language_backend.name}); Python version={platform.python_version()}, platform={platform.platform()}" ) log.info("Configuration file: %s", self.serena_config.config_file_path) log.info("Available projects: {}".format(", ".join(self.serena_config.project_names))) @@ -475,17 +475,8 @@ class SerenaAgent: return None def get_language_server_manager_or_raise(self) -> LanguageServerManager: - language_server_manager = self.get_language_server_manager() - if language_server_manager is None: - msg = TextBuilder("The language server manager is not initialized, indicating a problem during project initialisation.") - if self._lsp_init_error is not None: - msg.with_text(str(self._lsp_init_error)) - msg.with_text("For details, please check the logs. " + self.get_log_inspection_instructions()) - msg.with_text( - "IMPORTANT: Stop, do not attempt workarounds. Inform the user and wait for further instructions before you continue!" - ) - raise Exception(msg.build()) - return language_server_manager + active_project = self.get_active_project_or_raise() + return active_project.get_language_server_manager_or_raise() def get_log_inspection_instructions(self) -> str: if self.serena_config.web_dashboard: @@ -493,7 +484,7 @@ class SerenaAgent: else: log_path = SerenaPaths().last_returned_log_file_path if log_path is not None: - return f"Find the current log file here: f{log_path}" + return f"Find the current log file here: {log_path}" else: return "Unfortunately, logs are not available. We recommend enabling the web dashboard/logging in general." @@ -569,15 +560,6 @@ class SerenaAgent: start_new_session=True, # Detach from parent process ) - def get_project_root(self) -> str: - """ - :return: the root directory of the active project (if any); raises a ValueError if there is no active project - """ - project = self.get_active_project() - if project is None: - raise ValueError("Cannot get project root if no project is active.") - return project.project_root - def get_exposed_tool_instances(self) -> list["Tool"]: """ :return: the tool instances which are exposed (e.g. to the MCP client). @@ -629,12 +611,17 @@ class SerenaAgent: def create_system_prompt(self) -> str: available_tools = self._active_tools available_markers = available_tools.tool_marker_names + global_memories = MemoriesManager( + serena_data_folder=None, read_only_memory_patterns=self.serena_config.read_only_memory_patterns + ).list_global_memories() + global_memories_str = dict_string(global_memories.to_dict()) if len(global_memories) > 0 else "" log.info("Generating system prompt with available_tools=(see active tools), available_markers=%s", available_markers) system_prompt = self.prompt_factory.create_system_prompt( context_system_prompt=self._format_prompt(self._context.prompt), mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self.get_active_modes()], available_tools=available_tools.tool_names, available_markers=available_markers, + global_memories_list=global_memories_str, ) # If a project is active at startup, append its activation message @@ -730,7 +717,13 @@ class SerenaAgent: f"(2) Configure one MCP server per backend in your client." ) + # shut down the previously active project to release its language server processes + if self._active_project is not None: + log.info(f"Shutting down previously active project '{self._active_project.project_name}' before switching") + self._active_project.shutdown() + self._active_project = project + project.set_agent(self) if update_active_modes: self._update_active_modes() @@ -744,7 +737,7 @@ class SerenaAgent: self.reset_language_server_manager() # initialize the language server in the background (if in language server mode) - if self.is_using_language_server(): + if self.get_language_backend().is_lsp(): self.issue_task(init_language_server_manager) if self._project_activation_callback is not None: @@ -841,26 +834,7 @@ class SerenaAgent: """ Starts/resets the language server manager for the current project """ - tool_timeout = self.serena_config.tool_timeout - if tool_timeout is None or tool_timeout < 0: - ls_timeout = None - else: - if tool_timeout < 10: - raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds") - ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout - - # instantiate and start the necessary language servers - try: - self._lsp_init_error = None - self.get_active_project_or_raise().create_language_server_manager( - log_level=self.serena_config.log_level, - ls_timeout=ls_timeout, - trace_lsp_communication=self.serena_config.trace_lsp_communication, - ls_specific_settings=self.serena_config.ls_specific_settings, - ) - except LanguageServerManagerInitialisationError as e: - self._lsp_init_error = e - raise + self.get_active_project_or_raise().create_language_server_manager() def add_language(self, language: Language) -> None: """ @@ -894,7 +868,8 @@ class SerenaAgent: """ Shuts down the agent, freeing resources and stopping background tasks. """ - if not hasattr(self, "_is_initialized"): + # guard against __del__ being called on a partially constructed instance + if not hasattr(self, "_active_project"): return log.info("SerenaAgent is shutting down ...") if self._active_project is not None: @@ -914,3 +889,17 @@ class SerenaAgent: if ls_manager is None: return [] return ls_manager.get_active_languages() + + @contextmanager + def active_project_context(self, project: Project) -> Iterator[None]: + """ + Context manager for temporarily setting/overriding the active project + + :param project: the project to be active + """ + original_project = self._active_project + self._active_project = project + try: + yield + finally: + self._active_project = original_project diff --git a/src/serena/cli.py b/src/serena/cli.py index 61546b1d..7dc3bf8f 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -350,6 +350,61 @@ class TopLevelCommands(AutoRegisteringGroup): else: print(f"{prefix}\n{instr}\n{postfix}") + @staticmethod + @click.command( + "start-project-server", + help="Starts the Serena project server, which exposes project querying capabilities via HTTP.", + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.option( + "--host", + type=str, + default="127.0.0.1", + show_default=True, + help="Listen address for the project server.", + ) + @click.option( + "--port", + type=int, + default=None, + help="Listen port for the project server (default: ProjectServer.PORT).", + ) + @click.option( + "--log-level", + type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), + default=None, + help="Override log level in config.", + ) + def start_project_server( + host: str, + port: int | None, + log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None, + ) -> None: + from serena.project_server import ProjectServer + + # initialize logging + Logger.root.setLevel(logging.INFO) + formatter = logging.Formatter(SERENA_LOG_FORMAT) + stderr_handler = logging.StreamHandler(stream=sys.stderr) + stderr_handler.formatter = formatter + Logger.root.addHandler(stderr_handler) + log_path = SerenaPaths().get_next_log_file_path("project-server") + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.formatter = formatter + Logger.root.addHandler(file_handler) + + if log_level is not None: + Logger.root.setLevel(logging.getLevelNamesMapping()[log_level]) + + log.info("Starting Serena project server") + log.info("Storing logs in %s", log_path) + + server = ProjectServer() + run_kwargs: dict[str, Any] = {"host": host} + if port is not None: + run_kwargs["port"] = port + server.run(**run_kwargs) + @staticmethod @click.command( "dashboard-viewer", @@ -560,7 +615,8 @@ class ProjectCommands(AutoRegisteringGroup): :return: the RegisteredProject instance """ project_root = Path(project_path).resolve() - yml_path = ProjectConfig.path_to_project_yml(project_root) + serena_config = SerenaConfig.from_config_file() + yml_path = serena_config.get_project_yml_location(str(project_root)) if os.path.exists(yml_path): raise FileExistsError(f"Project file {yml_path} already exists.") @@ -574,14 +630,14 @@ class ProjectCommands(AutoRegisteringGroup): raise ValueError(f"Unknown language '{lang}'. Supported: {all_langs}") generated_conf = ProjectConfig.autogenerate( - project_root=project_path, project_name=name, languages=languages if languages else None, interactive=True + project_root=project_path, + serena_config=serena_config, + project_name=name, + languages=languages if languages else None, + interactive=True, ) - yml_path = ProjectConfig.path_to_project_yml(project_path) languages_str = ", ".join([lang.value for lang in generated_conf.languages]) if generated_conf.languages else "N/A" click.echo(f"Generated project with languages {{{languages_str}}} at {yml_path}.") - - # add to SerenaConfig's list of registered projects - serena_config = SerenaConfig.from_config_file() registered_project = serena_config.get_registered_project(str(project_root)) if registered_project is None: registered_project = RegisteredProject(str(project_root), generated_conf) @@ -656,9 +712,7 @@ class ProjectCommands(AutoRegisteringGroup): serena_config = SerenaConfig.from_config_file() proj = registered_project.get_project_instance(serena_config=serena_config) click.echo(f"Indexing symbols in {proj} …") - ls_mgr = proj.create_language_server_manager( - log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings - ) + ls_mgr = proj.create_language_server_manager() try: log_file = os.path.join(proj.project_root, ".serena", "logs", "indexing.txt") @@ -769,6 +823,9 @@ class ProjectCommands(AutoRegisteringGroup): logging.configure(level=logging.INFO) project_path = os.path.abspath(project) serena_config = SerenaConfig.from_config_file() + serena_config.language_backend = LanguageBackend.LSP + serena_config.gui_log_window = False + serena_config.web_dashboard = False proj = Project.load(project_path, serena_config=serena_config) # Create log file with timestamp @@ -783,9 +840,7 @@ class ProjectCommands(AutoRegisteringGroup): try: # Create SerenaAgent with dashboard disabled log.info("Creating SerenaAgent with disabled dashboard...") - serena_config = SerenaConfig.from_config_file() - serena_config.gui_log_window = False - serena_config.web_dashboard = False + agent = SerenaAgent(project=project_path, serena_config=serena_config) log.info("SerenaAgent created successfully") @@ -819,7 +874,7 @@ class ProjectCommands(AutoRegisteringGroup): # Test 1: Get symbols overview log.info("Testing GetSymbolsOverviewTool on file: %s", target_file) overview_data = agent.execute_task(lambda: overview_tool.get_symbol_overview(target_file)) - log.info("GetSymbolsOverviewTool returned %d symbols", len(overview_data)) + log.info(f"GetSymbolsOverviewTool returned: {overview_data}") if not overview_data: log.error("No symbols found in file %s", target_file) @@ -840,8 +895,8 @@ class ProjectCommands(AutoRegisteringGroup): selected_symbol = overview_data[0] log.info("No class or function found, using first available symbol") - symbol_name = selected_symbol.get("name_path", "unknown") - symbol_kind = selected_symbol.get("kind", "unknown") + symbol_name = selected_symbol["name"] + symbol_kind = selected_symbol["kind"] log.info("Using symbol for testing: %s (kind: %s)", symbol_name, symbol_kind) # Test 2: FindSymbolTool diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index b5075c38..430c8c2a 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -4,7 +4,7 @@ import os from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator, Reversible from contextlib import contextmanager -from typing import TYPE_CHECKING, Generic, Optional, TypeVar, cast +from typing import Generic, TypeVar, cast from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol @@ -12,29 +12,17 @@ from solidlsp import SolidLanguageServer, ls_types from solidlsp.ls import LSPFileBuffer from solidlsp.ls_utils import PathUtils, TextUtils -from .constants import DEFAULT_SOURCE_FILE_ENCODING from .project import Project -if TYPE_CHECKING: - from .agent import SerenaAgent - - log = logging.getLogger(__name__) TSymbol = TypeVar("TSymbol", bound=Symbol) class CodeEditor(Generic[TSymbol], ABC): - def __init__(self, project_root: str, agent: Optional["SerenaAgent"] = None) -> None: - self.project_root = project_root - self.agent = agent - - # set encoding based on active project, if available - encoding = DEFAULT_SOURCE_FILE_ENCODING - if agent is not None: - project = agent.get_active_project() - if project is not None: - encoding = project.project_config.encoding - self.encoding = encoding + def __init__(self, project: Project) -> None: + self.project_root = project.project_root + self.encoding = project.project_config.encoding + self.newline = project.line_ending.newline_str class EditedFile(ABC): def __init__(self, relative_path: str) -> None: @@ -82,7 +70,7 @@ class CodeEditor(Generic[TSymbol], ABC): def _save_edited_file(self, edited_file: "CodeEditor.EditedFile") -> None: abs_path = os.path.join(self.project_root, edited_file.relative_path) new_contents = edited_file.get_contents() - with open(abs_path, "w", encoding=self.encoding) as f: + with open(abs_path, "w", encoding=self.encoding, newline=self.newline) as f: f.write(new_contents) @abstractmethod @@ -244,8 +232,8 @@ class CodeEditor(Generic[TSymbol], ABC): class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): - def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, agent: Optional["SerenaAgent"] = None): - super().__init__(project_root=symbol_retriever.get_root_path(), agent=agent) + def __init__(self, symbol_retriever: LanguageServerSymbolRetriever): + super().__init__(project=symbol_retriever.project) self._symbol_retriever = symbol_retriever def _get_language_server(self, relative_path: str) -> SolidLanguageServer: @@ -379,9 +367,9 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]): - def __init__(self, project: Project, agent: Optional["SerenaAgent"] = None) -> None: + def __init__(self, project: Project) -> None: self._project = project - super().__init__(project_root=project.project_root, agent=agent) + super().__init__(project) class EditedFile(CodeEditor.EditedFile): def __init__(self, relative_path: str, project: Project): diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 432e8a65..dcfe6f90 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -4,6 +4,7 @@ The Serena Model Context Protocol (MCP) Server import dataclasses import os +import re import shutil from collections.abc import Iterator, Sequence from copy import deepcopy @@ -22,6 +23,7 @@ from sensai.util.string import ToStringMixin from serena.constants import ( DEFAULT_SOURCE_FILE_ENCODING, + PROJECT_LOCAL_TEMPLATE_FILE, PROJECT_TEMPLATE_FILE, REPO_ROOT, SERENA_CONFIG_TEMPLATE_FILE, @@ -85,6 +87,12 @@ class SerenaPaths: """ file containing the ID of the last read news snippet """ + global_memories_path = Path(os.path.join(self.serena_user_home_dir, "memories", "global")) + global_memories_path.mkdir(parents=True, exist_ok=True) + self.global_memories_path = global_memories_path + """ + directory where global memories are stored, i.e. memories that are available across all projects + """ self.last_returned_log_file_path: str | None = None """ the path to the last log file returned by `get_next_log_file_path`. If this is not None, the logs @@ -98,7 +106,7 @@ class SerenaPaths: """ log_dir = os.path.join(self.serena_user_home_dir, "logs", datetime.now().strftime("%Y-%m-%d")) os.makedirs(log_dir, exist_ok=True) - self.last_returned_log_file_path = os.path.join(log_dir, prefix + "_" + datetime_tag() + ".txt") + self.last_returned_log_file_path = os.path.join(log_dir, prefix + "_" + datetime_tag() + f"_{os.getpid()}" + ".txt") return self.last_returned_log_file_path # TODO: Paths from constants.py should be moved here @@ -167,6 +175,41 @@ class LanguageBackend(Enum): return backend raise ValueError(f"Unknown language backend '{backend_str}': valid values are {[b.value for b in LanguageBackend]}") + def is_lsp(self) -> bool: + return self == LanguageBackend.LSP + + def is_jetbrains(self) -> bool: + return self == LanguageBackend.JETBRAINS + + +class LineEnding(Enum): + """Line ending convention for file writes.""" + + LF = "lf" + CRLF = "crlf" + NATIVE = "native" + + @property + def newline_str(self) -> str | None: + """The newline parameter value for :func:`open` and :meth:`Path.write_text`. + + Returns ``None`` for native mode (platform default). + """ + if self is LineEnding.LF: + return "\n" + elif self is LineEnding.CRLF: + return "\r\n" + return None + + @classmethod + def from_str(cls, value: str) -> "LineEnding": + """Parse a string value into a :class:`LineEnding`.""" + try: + return cls(value.lower()) + except ValueError as e: + valid = [le.value for le in cls] + raise ValueError(f"Invalid line_ending: {value!r}. Valid values are: {valid}") from e + @dataclass class SharedConfig(ModeSelectionDefinition, ToolInclusionDefinition, ToStringMixin): @@ -177,14 +220,22 @@ class SharedConfig(ModeSelectionDefinition, ToolInclusionDefinition, ToStringMix symbol_info_budget: float | None = None language_backend: LanguageBackend | None = None + line_ending: LineEnding | None = None + read_only_memory_patterns: list[str] = field(default_factory=list) + ignored_memory_patterns: list[str] = field(default_factory=list) + ls_specific_settings: dict = field(default_factory=dict) + """Advanced configuration option allowing to configure language server implementation specific options, see SolidLSPSettings for more info.""" class SerenaConfigError(Exception): pass -def get_serena_managed_in_project_dir(project_root: str | Path) -> str: - return os.path.join(project_root, SERENA_MANAGED_DIR_NAME) +DEFAULT_PROJECT_SERENA_FOLDER_LOCATION = "$projectDir/" + SERENA_MANAGED_DIR_NAME +""" +The default template for the project Serena folder location. +Uses $projectDir and $projectFolderName as placeholders. +""" @dataclass(kw_only=True) @@ -197,7 +248,12 @@ class ProjectConfig(SharedConfig): initial_prompt: str = "" encoding: str = DEFAULT_SOURCE_FILE_ENCODING - SERENA_DEFAULT_PROJECT_FILE = "project.yml" + # internal fields which are not mapped to/from the configuration file (must start with "_") + _local_override_keys: list[str] = field(default_factory=list) + + # class-level constants + SERENA_PROJECT_FILE = "project.yml" + SERENA_LOCAL_PROJECT_FILE = "project.local.yml" FIELDS_WITHOUT_DEFAULTS = {"project_name", "languages"} YAML_COMMENT_NORMALISATION = YamlCommentNormalisation.LEADING """ @@ -212,6 +268,7 @@ class ProjectConfig(SharedConfig): def autogenerate( cls, project_root: str | Path, + serena_config: "SerenaConfig", project_name: str | None = None, languages: list[Language] | None = None, save_to_disk: bool = True, @@ -221,6 +278,7 @@ class ProjectConfig(SharedConfig): Autogenerate a project configuration for a given project root. :param project_root: the path to the project root + :param serena_config: the global Serena configuration :param project_name: the name of the project; if None, the name of the project will be the name of the directory containing the project :param languages: the languages of the project; if None, they will be determined automatically @@ -233,70 +291,79 @@ class ProjectConfig(SharedConfig): raise FileNotFoundError(f"Project root not found: {project_root}") with LogTime("Project configuration auto-generation", logger=log): log.info("Project root: %s", project_root) - project_name = project_name or project_root.name + project_folder_name = project_root.name + project_name = project_name or project_folder_name if languages is None: # determine languages automatically log.info("Determining programming languages used in the project") language_composition = determine_programming_language_composition(str(project_root)) log.info("Language composition: %s", language_composition) if len(language_composition) == 0: - language_values = ", ".join([lang.value for lang in Language]) - raise ValueError( - f"No source files found in {project_root}\n\n" - f"To use Serena with this project, you need to either\n" - f" 1. specify a programming language by adding parameters --language \n" - f" when creating the project via the Serena CLI command OR\n" - f" 2. add source files in one of the supported languages first.\n\n" - f"Supported languages are: {language_values}\n" - f"Read the documentation for more information." + log.warning( + "No source files for supported language servers were found in %s. " + "Creating project with no configured languages. " + "Symbol-related tools (e.g. find_symbol, get_symbols_overview) will not work " + "when using the LSP backend. You can add languages later via the Serena dashboard " + "or by manually editing the project configuration.", + project_root, ) - # sort languages by number of files found - languages_and_percentages = sorted( - language_composition.items(), key=lambda item: (item[1], item[0].get_priority()), reverse=True - ) - # find the language with the highest percentage and enable it - top_language_pair = languages_and_percentages[0] - other_language_pairs = languages_and_percentages[1:] - languages_to_use: list[str] = [top_language_pair[0].value] - # if in interactive mode, ask the user which other languages to enable - if len(other_language_pairs) > 0 and interactive: - print( - "Detected and enabled main language '%s' (%.2f%% of source files)." - % (top_language_pair[0].value, top_language_pair[1]) + languages_to_use: list[str] = [] + else: + # sort languages by number of files found + languages_and_percentages = sorted( + language_composition.items(), key=lambda item: (item[1], item[0].get_priority()), reverse=True ) - print(f"Additionally detected {len(other_language_pairs)} other language(s).\n") - print("Note: Enable only languages you need symbolic retrieval/editing capabilities for.") - print(" Additional language servers use resources and some languages may require additional") - print(" system-level installations/configuration (see Serena documentation).") - print("\nWhich additional languages do you want to enable?") - for lang, perc in other_language_pairs: - enable = ask_yes_no("Enable %s (%.2f%% of source files)?" % (lang.value, perc), default=False) - if enable: - languages_to_use.append(lang.value) - print() + # find the language with the highest percentage and enable it + top_language_pair = languages_and_percentages[0] + other_language_pairs = languages_and_percentages[1:] + languages_to_use = [top_language_pair[0].value] + # if in interactive mode, ask the user which other languages to enable + if len(other_language_pairs) > 0 and interactive: + print( + "Detected and enabled main language '%s' (%.2f%% of source files)." + % (top_language_pair[0].value, top_language_pair[1]) + ) + print(f"Additionally detected {len(other_language_pairs)} other language(s).\n") + print("Note: Enable only languages you need symbolic retrieval/editing capabilities for.") + print(" Additional language servers use resources and some languages may require additional") + print(" system-level installations/configuration (see Serena documentation).") + print("\nWhich additional languages do you want to enable?") + for lang, perc in other_language_pairs: + enable = ask_yes_no("Enable %s (%.2f%% of source files)?" % (lang.value, perc), default=False) + if enable: + languages_to_use.append(lang.value) + print() log.info("Using languages: %s", languages_to_use) else: languages_to_use = [lang.value for lang in languages] - config_with_comments, _ = cls._load_yaml(PROJECT_TEMPLATE_FILE) + config_with_comments, _ = cls._load_yaml_dict(PROJECT_TEMPLATE_FILE) config_with_comments["project_name"] = project_name config_with_comments["languages"] = languages_to_use + if save_to_disk: - project_yml_path = cls.path_to_project_yml(project_root) + project_yml_path = serena_config.get_project_yml_location(str(project_root)) log.info("Saving project configuration to %s", project_yml_path) save_yaml(project_yml_path, config_with_comments) - return cls._from_dict(config_with_comments) + project_local_yml_path = os.path.join(os.path.dirname(project_yml_path), cls.SERENA_LOCAL_PROJECT_FILE) + shutil.copy(PROJECT_LOCAL_TEMPLATE_FILE, project_local_yml_path) + + return cls._from_dict(config_with_comments, local_override_keys=[]) @classmethod - def path_to_project_yml(cls, project_root: str | Path) -> str: - return os.path.join(project_root, cls.rel_path_to_project_yml()) + def default_project_yml_path(cls, project_root: str | Path) -> str: + """ + :return: the default path to the project.yml file (inside ``$projectDir/.serena/``). + This is suitable as a fallback when no ``SerenaConfig`` is available to resolve + a potentially customised location. + """ + return os.path.join(str(project_root), SERENA_MANAGED_DIR_NAME, cls.SERENA_PROJECT_FILE) @classmethod - def rel_path_to_project_yml(cls) -> str: - return os.path.join(SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE) - - @classmethod - def _load_yaml( - cls, yml_path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE + def _load_yaml_dict( + cls, + yml_path: str, + comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE, + apply_defaults: bool = True, ) -> tuple[CommentedMap, bool]: """ Load the project configuration as a CommentedMap, preserving comments and ensuring @@ -304,24 +371,33 @@ class ProjectConfig(SharedConfig): and backward compatibility adjustments. :param yml_path: the path to the project.yml file + :param comment_normalisation: the strategy to use for normalising comments in the loaded YAML + :param apply_defaults: whether to apply default values for missing fields :return: a tuple `(dict, was_complete)` where dict is a CommentedMap representing a full project configuration and `was_complete` indicates whether the loaded configuration - was complete (i.e., did not require any default values to be applied) + was complete (i.e., did not require any default values to be applied) for the case where + `apply_defaults` is True; If `apply_defaults` is False, the returned dict may be incomplete + and `was_complete` will always be True. """ data = load_yaml(yml_path, comment_normalisation=comment_normalisation) # apply defaults was_complete = True - for field_info in dataclasses.fields(cls): - key = field_info.name - if key in cls.FIELDS_WITHOUT_DEFAULTS: - continue - if key not in data: - was_complete = False - default_value = get_dataclass_default(cls, key) - data.setdefault(key, default_value) + if apply_defaults: + for field_info in dataclasses.fields(cls): + key = field_info.name + if key.startswith("_"): + continue + if key in cls.FIELDS_WITHOUT_DEFAULTS: + continue + if key not in data: + was_complete = False + default_value = get_dataclass_default(cls, key) + data.setdefault(key, default_value) - # backward compatibility: handle single "language" field + # backward compatibility + # NOTE: This must also work for project.local.yml files, which may be highly incomplete + # * handle single "language" field if "languages" not in data and "language" in data: data["languages"] = [data["language"]] del data["language"] @@ -329,9 +405,13 @@ class ProjectConfig(SharedConfig): return data, was_complete @classmethod - def _from_dict(cls, data: dict[str, Any]) -> Self: + def _from_dict(cls, data: dict[str, Any], local_override_keys: list[str]) -> Self: """ Create a ProjectConfig instance from a (full) configuration dictionary + + :param data: the configuration dictionary; must contain all required fields and use the same field names as + the ProjectConfig dataclass + :param local_override_keys: the list of keys that have been overridden from project.local.yml """ lang_name_mapping = {"javascript": "typescript"} languages: list[Language] = [] @@ -362,6 +442,9 @@ class ProjectConfig(SharedConfig): language_backend_value = data.get("language_backend") language_backend = LanguageBackend.from_str(language_backend_value) if language_backend_value else None + line_ending_value = data.get("line_ending") + line_ending = LineEnding.from_str(line_ending_value) if line_ending_value else None + return cls( project_name=data["project_name"], languages=languages, @@ -370,13 +453,18 @@ class ProjectConfig(SharedConfig): fixed_tools=data["fixed_tools"], included_optional_tools=data["included_optional_tools"], read_only=data["read_only"], + read_only_memory_patterns=data.get("read_only_memory_patterns", []), + ignored_memory_patterns=data.get("ignored_memory_patterns", []), ignore_all_files_in_gitignore=data["ignore_all_files_in_gitignore"], initial_prompt=data["initial_prompt"], encoding=data["encoding"], + line_ending=line_ending, language_backend=language_backend, base_modes=data["base_modes"], default_modes=data["default_modes"], symbol_info_budget=symbol_info_budget, + ls_specific_settings=data.get("ls_specific_settings", {}), + _local_override_keys=local_override_keys, ) def _to_yaml_dict(self) -> dict: @@ -384,59 +472,117 @@ class ProjectConfig(SharedConfig): :return: a yaml-serializable dictionary representation of this configuration """ d = dataclasses.asdict(self) + + # drop internal fields starting with underscore + keys = list(d.keys()) + for k in keys: + if k.startswith("_"): + del d[k] + + # map fields using non-primitive types to a YAML-compatible representation d["languages"] = [lang.value for lang in self.languages] d["language_backend"] = self.language_backend.value if self.language_backend is not None else None + d["line_ending"] = self.line_ending.value if self.line_ending is not None else None + return d @classmethod - def load(cls, project_root: Path | str, autogenerate: bool = False) -> Self: + def _project_local_yml_path(cls, project_yml_path: str) -> str: + return os.path.join(os.path.dirname(project_yml_path), cls.SERENA_LOCAL_PROJECT_FILE) + + @classmethod + def load(cls, project_root: Path | str, serena_config: "SerenaConfig", autogenerate: bool = False) -> Self: """ Load a ProjectConfig instance from the path to the project root. + + :param project_root: the path to the project root + :param serena_config: the global Serena configuration + :param autogenerate: whether to auto-generate the configuration if it does not exist """ project_root = Path(project_root) - yaml_path = project_root / cls.rel_path_to_project_yml() + project_folder_name = project_root.name + yaml_path = serena_config.get_project_yml_location(project_root) + log.debug("Loading project configuration from %s", yaml_path) # auto-generate if necessary - if not yaml_path.exists(): + if not os.path.exists(yaml_path): if autogenerate: - return cls.autogenerate(project_root) + return cls.autogenerate(project_root, serena_config) else: raise FileNotFoundError(f"Project configuration file not found: {yaml_path}") # load the configuration dictionary - yaml_data, was_complete = cls._load_yaml(str(yaml_path)) + yaml_data, was_complete = cls._load_yaml_dict(str(yaml_path)) if "project_name" not in yaml_data: - yaml_data["project_name"] = project_root.name + yaml_data["project_name"] = project_folder_name + + # apply overrides from project.local.yml, if present + local_yaml_path = cls._project_local_yml_path(str(yaml_path)) + local_override_keys = [] + if os.path.exists(local_yaml_path): + local_yaml_data, _ = cls._load_yaml_dict(local_yaml_path, apply_defaults=False) + if local_yaml_data: + local_override_keys = list(local_yaml_data.keys()) + log.debug( + "Applying project configuration overrides from %s with keys %s", + local_yaml_path, + local_override_keys, + ) + yaml_data.update(local_yaml_data) # instantiate the ProjectConfig - project_config = cls._from_dict(yaml_data) + project_config = cls._from_dict(yaml_data, local_override_keys=local_override_keys) # if the configuration was incomplete, re-save it to disk if not was_complete: log.info("Project configuration in %s was incomplete, re-saving with default values for missing fields", yaml_path) - project_config.save(project_root) + project_config.save(str(yaml_path), save_project_local_yml=False) return project_config - def save(self, project_root: Path | str) -> None: + def save(self, project_yml_path: str, save_project_local_yml: bool = True) -> None: """ - Saves the project configuration to disk. + Saves the project configuration to disk, updating both the project.yml file and, optionally, + the project.local.yml file to reflect overridden keys. - :param project_root: the root directory of the project + Keys that are overridden by project.local.yml are not updated in project.yml. + Only keys that are overridden are updated in project.local.yml. + + :param project_yml_path: the path to the project.yml file + :param save_project_local_yml: whether to also update the project.local.yml file to reflect overridden keys """ - config_path = self.path_to_project_yml(project_root) + config_path = project_yml_path log.info("Saving updated project configuration to %s", config_path) - # load original commented map and update it with current values - config_with_comments, _ = self._load_yaml(config_path, self.YAML_COMMENT_NORMALISATION) - config_with_comments.update(self._to_yaml_dict()) + # get the current configuration as a dictionary + cur_dict = self._to_yaml_dict() + + # load commented map from the original file and update all non-overridden keys + config_with_comments, _ = self._load_yaml_dict(config_path, self.YAML_COMMENT_NORMALISATION) + for key in cur_dict: + if key not in self._local_override_keys: + config_with_comments[key] = cur_dict[key] # transfer missing comments from the template file - template_config, _ = self._load_yaml(PROJECT_TEMPLATE_FILE, self.YAML_COMMENT_NORMALISATION) + template_config, _ = self._load_yaml_dict(PROJECT_TEMPLATE_FILE, self.YAML_COMMENT_NORMALISATION) transfer_missing_yaml_comments(template_config, config_with_comments, self.YAML_COMMENT_NORMALISATION) + # save project.yml save_yaml(config_path, config_with_comments) + # update project.local.yml to reflect overridden keys if necessary + if save_project_local_yml: + project_local_yml_path = self._project_local_yml_path(project_yml_path) + if self._local_override_keys and os.path.exists(project_local_yml_path): + log.info("Saving updated local project configuration to %s", project_local_yml_path) + local_config_with_comments, _ = self._load_yaml_dict( + project_local_yml_path, comment_normalisation=YamlCommentNormalisation.NONE, apply_defaults=False + ) + for key in self._local_override_keys: + if key in cur_dict: + local_config_with_comments[key] = cur_dict[key] + save_yaml(project_local_yml_path, local_config_with_comments) + class RegisteredProject(ToStringMixin): def __init__( @@ -472,8 +618,8 @@ class RegisteredProject(ToStringMixin): ) @classmethod - def from_project_root(cls, project_root: str | Path) -> "RegisteredProject": - project_config = ProjectConfig.load(project_root) + def from_project_root(cls, project_root: str | Path, serena_config: "SerenaConfig") -> "RegisteredProject": + project_config = ProjectConfig.load(project_root, serena_config=serena_config) return RegisteredProject( project_root=str(project_root), project_config=project_config, @@ -488,7 +634,7 @@ class RegisteredProject(ToStringMixin): """ return self.project_root.samefile(Path(path).resolve()) - def get_project_instance(self, serena_config: "SerenaConfig | None") -> "Project": + def get_project_instance(self, serena_config: "SerenaConfig") -> "Project": """ Returns the project instance for this registered project, loading it if necessary. """ @@ -524,11 +670,6 @@ class SerenaConfig(SharedConfig): jetbrains_plugin_server_address: str = "127.0.0.1" tool_timeout: float = DEFAULT_TOOL_TIMEOUT - language_backend: LanguageBackend = LanguageBackend.LSP - """ - the language backend to use for code understanding features - """ - token_count_estimator: str = RegisteredTokenCountEstimator.CHAR_COUNT.name """Only relevant if `record_tool_usage` is True; the name of the token count estimator to use for tool usage statistics. See the `RegisteredTokenCountEstimator` enum for available options. @@ -542,15 +683,29 @@ class SerenaConfig(SharedConfig): Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default through the global configuration. """ - ls_specific_settings: dict = field(default_factory=dict) - """Advanced configuration option allowing to configure language server implementation specific options, see SolidLSPSettings for more info.""" ignored_paths: list[str] = field(default_factory=list) """List of paths to ignore across all projects. Same syntax as gitignore, so you can use * and **. These patterns are merged additively with each project's own ignored_paths.""" + project_serena_folder_location: str = DEFAULT_PROJECT_SERENA_FOLDER_LOCATION + """ + Template for the location of the per-project .serena data folder (memories, caches, etc.). + Supports the following placeholders: + - $projectDir: the absolute path to the project root directory + - $projectFolderName: the name of the project folder + Examples: + - "$projectDir/.serena" (default, stores data inside the project) + - "/projects-metadata/$projectFolderName/.serena" (stores data in a central location) + """ + # settings with overridden defaults + language_backend: LanguageBackend = LanguageBackend.LSP + """ + the language backend to use for code understanding features + """ default_modes: Sequence[str] | None = ("interactive", "editing") + line_ending: LineEnding = LineEnding.NATIVE symbol_info_budget: float = 10.0 """ Time budget (seconds) for requests when tools request include_info (currently @@ -559,7 +714,6 @@ class SerenaConfig(SharedConfig): If the budget is exceeded, Serena stops issuing further requests and returns partial info results. 0 disables the budget (no early stopping). Negative values are invalid. """ - # *** fields that are NOT mapped to/from the configuration file *** _loaded_commented_yaml: CommentedMap | None = None @@ -572,7 +726,7 @@ class SerenaConfig(SharedConfig): # *** static members *** CONFIG_FILE = "serena_config.yml" - CONFIG_FIELDS_WITH_TYPE_CONVERSION = {"projects", "language_backend"} + CONFIG_FIELDS_WITH_TYPE_CONVERSION = {"projects", "language_backend", "line_ending"} # *** methods *** @classmethod @@ -675,15 +829,20 @@ class SerenaConfig(SharedConfig): instance.projects = [] for path in loaded_commented_yaml["projects"]: path = Path(path).resolve() - if not path.exists() or (path.is_dir() and not (path / ProjectConfig.rel_path_to_project_yml()).exists()): - log.warning(f"Project path {path} does not exist or does not contain a project configuration file, skipping.") + try: + path_exists = path.exists() + except OSError as e: + log.warning(f"Project path {path} is not accessible ({e}), skipping.") + continue + if not path_exists or (path.is_dir() and not os.path.isfile(instance.get_project_yml_location(str(path)))): + log.warning(f"Project path {path} does not exist or no associated project configuration file found, skipping.") continue if path.is_file(): path = cls._migrate_out_of_project_config_file(path) if path is None: continue num_migrations += 1 - project_config = ProjectConfig.load(path) + project_config = ProjectConfig.load(path, serena_config=instance) # instance is sufficiently populated project = RegisteredProject( project_root=str(path), project_config=project_config, @@ -704,6 +863,14 @@ class SerenaConfig(SharedConfig): del loaded_commented_yaml["jetbrains"] instance.language_backend = language_backend + # determine line ending + line_ending_value = loaded_commented_yaml.get("line_ending") + if line_ending_value: + instance.line_ending = LineEnding.from_str(line_ending_value) + else: + num_migrations += 1 + instance.line_ending = get_dataclass_default(SerenaConfig, "line_ending") + # migrate deprecated "gui_log_level" field if necessary if "gui_log_level" in loaded_commented_yaml: num_migrations += 1 @@ -711,6 +878,14 @@ class SerenaConfig(SharedConfig): instance.log_level = loaded_commented_yaml["gui_log_level"] del loaded_commented_yaml["gui_log_level"] + # migrate "edit_global_memories" + if "edit_global_memories" in loaded_commented_yaml: + num_migrations += 1 + edit_global_memories = loaded_commented_yaml["edit_global_memories"] + if not edit_global_memories: + instance.read_only_memory_patterns.append("global/.*") + del loaded_commented_yaml["edit_global_memories"] + # re-save the configuration file if any migrations were performed if num_migrations > 0: log.info("Legacy configuration was migrated; re-saving configuration file") @@ -736,7 +911,7 @@ class SerenaConfig(SharedConfig): with open(path, "a", encoding=SERENA_FILE_ENCODING) as f: f.write(f"\nproject_name: {project_name}") project_root = project_config_data["project_root"] - shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml())) + shutil.move(str(path), ProjectConfig.default_project_yml_path(project_root)) return Path(project_root).resolve() except Exception as e: log.error(f"Error migrating configuration file: {e}") @@ -765,7 +940,7 @@ class SerenaConfig(SharedConfig): return project_candidates[0] elif len(project_candidates) > 1: raise ValueError( - f"Multiple projects found with name '{project_root_or_name}'. Please activate it by location instead. " + f"Multiple projects found with name '{project_root_or_name}'. Please reference it by location instead. " f"Locations: {[p.project_root for p in project_candidates]}" ) # no project found by name; check if it's a path @@ -775,9 +950,9 @@ class SerenaConfig(SharedConfig): return project # no registered project found; auto-register if project configuration exists if autoregister: - config_path = ProjectConfig.path_to_project_yml(project_root_or_name) + config_path = self.get_project_yml_location(project_root_or_name) if os.path.isfile(config_path): - registered_project = RegisteredProject.from_project_root(project_root_or_name) + registered_project = RegisteredProject.from_project_root(project_root_or_name, serena_config=self) self.add_registered_project(registered_project) return registered_project # nothing found @@ -820,7 +995,7 @@ class SerenaConfig(SharedConfig): f"Project with path {project_root} was already added with name '{already_registered_project.project_name}'." ) - project_config = ProjectConfig.load(project_root, autogenerate=True) + project_config = ProjectConfig.load(project_root, serena_config=self, autogenerate=True) new_project = Project( project_root=str(project_root), @@ -863,6 +1038,9 @@ class SerenaConfig(SharedConfig): # convert language backend to string commented_yaml["language_backend"] = self.language_backend.value + # convert line ending to string + commented_yaml["line_ending"] = self.line_ending.value + # transfer comments from the template file # NOTE: The template file now uses leading comments, but we previously used trailing comments, # so we apply a conversion, which detects the old style and transforms it. @@ -873,6 +1051,78 @@ class SerenaConfig(SharedConfig): save_yaml(self.config_file_path, commented_yaml) + @staticmethod + def _resolve_serena_folder_location(template: str, placeholders: dict[str, str]) -> str: + """ + Resolves a folder location template by replacing known ``$placeholder`` tokens + and raising on any unrecognised ones. + + :param template: the template string (e.g. ``"$projectDir/.serena"``) + :param placeholders: mapping from placeholder name (without ``$``) to replacement value + :return: the resolved absolute path + :raises SerenaConfigError: if the template contains an unknown ``$placeholder`` + """ + + def _replace(match: re.Match[str]) -> str: + name = match.group(1) + if name not in placeholders: + raise SerenaConfigError( + f"Unknown placeholder '${name}' in project_serena_folder_location. " + f"Supported placeholders: {', '.join('$' + k for k in placeholders)}" + ) + return placeholders[name] + + result = re.sub(r"\$([A-Za-z_]\w*)", _replace, template) + return os.path.abspath(result) + + def get_configured_project_serena_folder(self, project_root: str | Path) -> str: + """ + Returns the resolved absolute path to the .serena data folder for a project, + applying placeholder substitution to ``project_serena_folder_location`` + without any fallback logic. + + :param project_root: the absolute path to the project root directory + :return: the resolved absolute path to the project's .serena folder + :raises SerenaConfigError: if the template contains an unknown placeholder + """ + project_folder_name = Path(project_root).name + placeholders = { + "projectDir": str(project_root), + "projectFolderName": project_folder_name, + } + return self._resolve_serena_folder_location(self.project_serena_folder_location, placeholders) + + def get_project_serena_folder(self, project_root: str | Path) -> str: + """ + Resolves the location of the project's .serena data folder using fallback logic: + + 1. If the folder exists at the configured path (``project_serena_folder_location``), use it. + 2. Otherwise, if it exists at the default location inside the project root, use that. + 3. If neither exists, return the configured path (for creation). + + :param project_root: the absolute path to the project root directory + :return: the resolved absolute path to the .serena data folder + :raises SerenaConfigError: if the configured template contains an unknown placeholder + """ + configured_path = self.get_configured_project_serena_folder(project_root) + if os.path.isdir(configured_path): + return configured_path + default_path = os.path.join(str(project_root), SERENA_MANAGED_DIR_NAME) + if configured_path != default_path and os.path.isdir(default_path): + return default_path + return configured_path + + def get_project_yml_location(self, project_root: str | Path) -> str: + """ + Returns the resolved absolute path to the project.yml configuration file, + based on the resolved .serena data folder (with fallback logic). + + :param project_root: the absolute path to the project root directory + :return: the resolved absolute path to the project's project.yml file + """ + serena_folder = self.get_project_serena_folder(project_root) + return os.path.join(serena_folder, ProjectConfig.SERENA_PROJECT_FILE) + def propagate_settings(self) -> None: """ Propagate settings from this configuration to individual components that are statically configured diff --git a/src/serena/constants.py b/src/serena/constants.py index 6d90a539..4d96bcff 100644 --- a/src/serena/constants.py +++ b/src/serena/constants.py @@ -25,6 +25,7 @@ SERENA_FILE_ENCODING = "utf-8" """The encoding used for Serena's own files, such as configuration files and memories.""" PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.yml") +PROJECT_LOCAL_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.local.template.yml") SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml") SERENA_LOG_FORMAT = "%(levelname)-5s %(asctime)-15s [%(threadName)s] %(name)s:%(funcName)s:%(lineno)d - %(message)s" diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index f5675da3..9143bcd5 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Self import pystray import webview -from flask import Flask, Response, request, send_from_directory +from flask import Flask, Response, redirect, request, send_from_directory from PIL import Image from pydantic import BaseModel from pystray import MenuItem as Item @@ -94,6 +94,11 @@ class RequestDeleteMemory(BaseModel): memory_name: str +class RequestRenameMemory(BaseModel): + old_name: str + new_name: str + + class ResponseGetSerenaConfig(BaseModel): content: str @@ -148,6 +153,10 @@ class SerenaDashboardAPI: return self._memory_log_handler def _setup_routes(self) -> None: + @self._app.route("/") + def redirect_to_dashboard() -> Response: + return redirect("/dashboard/") # type: ignore[return-value] + # Static files @self._app.route("/dashboard/") def serve_dashboard(filename: str) -> Response: @@ -274,6 +283,18 @@ class SerenaDashboardAPI: except Exception as e: return {"status": "error", "message": str(e)} + @self._app.route("/rename_memory", methods=["POST"]) + def rename_memory() -> dict[str, str]: + request_data = request.get_json() + if not request_data: + return {"status": "error", "message": "No data provided"} + request_rename_memory = RequestRenameMemory.model_validate(request_data) + try: + result_message = self._rename_memory(request_rename_memory) + return {"status": "success", "message": result_message} + except Exception as e: + return {"status": "error", "message": str(e)} + @self._app.route("/get_serena_config", methods=["GET"]) def get_serena_config() -> dict[str, Any]: try: @@ -489,7 +510,7 @@ class SerenaDashboardAPI: # Get available memories if ReadMemoryTool is active available_memories = None if self._agent.tool_is_active("read_memory") and project is not None: - available_memories = project.memories_manager.list_memories() + available_memories = project.memories_manager.list_memories().get_full_list() # Get list of languages for the active project languages = [] @@ -512,7 +533,7 @@ class SerenaDashboardAPI: available_modes=available_modes, available_contexts=available_contexts, available_memories=available_memories, - jetbrains_mode=not self._agent.is_using_language_server(), + jetbrains_mode=self._agent.get_language_backend().is_jetbrains(), languages=languages, encoding=encoding, current_client=Tool.get_last_tool_call_client_str(), @@ -561,8 +582,7 @@ class SerenaDashboardAPI: project = self._agent.get_active_project() if project is None: raise ValueError("No active project") - - project.memories_manager.save_memory(request_save_memory.memory_name, request_save_memory.content) + project.memories_manager.save_memory(request_save_memory.memory_name, request_save_memory.content, is_tool_context=False) self._agent.execute_task(run, logged=True, name="SaveMemory") @@ -571,11 +591,22 @@ class SerenaDashboardAPI: project = self._agent.get_active_project() if project is None: raise ValueError("No active project") - - project.memories_manager.delete_memory(request_delete_memory.memory_name) + project.memories_manager.delete_memory(request_delete_memory.memory_name, is_tool_context=False) self._agent.execute_task(run, logged=True, name="DeleteMemory") + def _rename_memory(self, request_rename_memory: RequestRenameMemory) -> str: + def run() -> str: + project = self._agent.get_active_project() + if project is None: + raise ValueError("No active project") + + return project.memories_manager.move_memory( + request_rename_memory.old_name, request_rename_memory.new_name, is_tool_context=False + ) + + return self._agent.execute_task(run, logged=True, name="RenameMemory") + def _get_serena_config(self) -> ResponseGetSerenaConfig: config_path = self._agent.serena_config.config_file_path if config_path is None or not os.path.exists(config_path): diff --git a/src/serena/generated/generated_prompt_factory.py b/src/serena/generated/generated_prompt_factory.py index 3a5e1442..9129318c 100644 --- a/src/serena/generated/generated_prompt_factory.py +++ b/src/serena/generated/generated_prompt_factory.py @@ -33,6 +33,12 @@ class PromptFactory(PromptFactoryBase): return self._render_prompt("prepare_for_new_conversation", locals()) def create_system_prompt( - self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, mode_system_prompts: Any + self, + *, + available_markers: Any, + available_tools: Any, + context_system_prompt: Any, + global_memories_list: Any, + mode_system_prompts: Any, ) -> str: return self._render_prompt("system_prompt", locals()) diff --git a/src/serena/jetbrains/jetbrains_plugin_client.py b/src/serena/jetbrains/jetbrains_plugin_client.py index f11dc080..f237a079 100644 --- a/src/serena/jetbrains/jetbrains_plugin_client.py +++ b/src/serena/jetbrains/jetbrains_plugin_client.py @@ -8,6 +8,7 @@ import logging import re import threading from concurrent.futures.thread import ThreadPoolExecutor +from dataclasses import dataclass from pathlib import Path from typing import Any, Literal, Optional, Self, TypeVar, cast @@ -16,6 +17,7 @@ from requests import Response from sensai.util.string import ToStringMixin import serena.jetbrains.jetbrains_types as jb +from serena.config.serena_config import RegisteredProject from serena.jetbrains.jetbrains_types import PluginStatusDTO from serena.project import Project from serena.util.class_decorators import singleton @@ -63,6 +65,12 @@ class ServerNotFoundError(Exception): """Raised when the plugin's service is not found.""" +@dataclass +class MatchedClient: + client: "JetBrainsPluginClient" + registered_project: RegisteredProject + + @singleton class JetBrainsPluginClientManager: """ @@ -73,6 +81,7 @@ class JetBrainsPluginClientManager: def __init__(self) -> None: self._clients: dict[int, "JetBrainsPluginClient"] = {} + self._matched_clients: list[MatchedClient] = [] self._lock = threading.Lock() def _submit_scan(self) -> list[concurrent.futures.Future["JetBrainsPluginClient"]]: @@ -116,6 +125,47 @@ class JetBrainsPluginClientManager: "with the Serena plugin installed and running!" ) + def match_clients(self, registered_projects: list[RegisteredProject]) -> list[MatchedClient]: + """ + Scans for plugin instances and matches them against the given registered projects. + + :param registered_projects: the list of registered projects to match plugin instances against + :return: the list of matched clients with their corresponding registered project + """ + matched_clients = [] + for future in self._submit_scan(): + client = future.result() + if client.project_root is not None: + for rp in registered_projects: + if client.matches(Path(rp.project_root)): + matched_clients.append(MatchedClient(client, rp)) + break + self._matched_clients = matched_clients + return matched_clients + + def get_matched_client( + self, registered_project: RegisteredProject, registered_projects: list[RegisteredProject] + ) -> Optional["JetBrainsPluginClient"]: + """ + Gets the matched client for a given registered project, if any. + + :param registered_project: the registered project to get the matched client for + :param registered_projects: the list of all registered projects (used to perform matching of all clients + if no match is found for the given project) + :return: the matched client or None if no match is found + """ + + def find_match() -> Optional["JetBrainsPluginClient"]: + for matched_client in self._matched_clients: + if matched_client.registered_project.project_root == registered_project.project_root: + return matched_client.client + return None + + match = find_match() + if match is None: + self.match_clients(registered_projects) + return find_match() + class JetBrainsPluginClient(ToStringMixin): """ diff --git a/src/serena/ls_manager.py b/src/serena/ls_manager.py index dde1f428..5c33fa52 100644 --- a/src/serena/ls_manager.py +++ b/src/serena/ls_manager.py @@ -6,7 +6,6 @@ from collections.abc import Iterator from sensai.util.logging import LogTime from serena.config.serena_config import SerenaPaths -from serena.constants import SERENA_MANAGED_DIR_NAME from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language, LanguageServerConfig from solidlsp.settings import SolidLSPSettings @@ -23,6 +22,7 @@ class LanguageServerFactory: def __init__( self, project_root: str, + project_data_path: str, encoding: str, ignored_patterns: list[str], ls_timeout: float | None = None, @@ -30,6 +30,7 @@ class LanguageServerFactory: trace_lsp_communication: bool = False, ): self.project_root = project_root + self.project_data_path = project_data_path self.encoding = encoding self.ignored_patterns = ignored_patterns self.ls_timeout = ls_timeout @@ -51,7 +52,7 @@ class LanguageServerFactory: timeout=self.ls_timeout, solidlsp_settings=SolidLSPSettings( solidlsp_dir=SerenaPaths().serena_user_home_dir, - project_data_relative_path=SERENA_MANAGED_DIR_NAME, + project_data_path=self.project_data_path, ls_specific_settings=self.ls_specific_settings or {}, ), ) @@ -76,8 +77,12 @@ class LanguageServerManager: """ self._language_servers = language_servers self._language_server_factory = language_server_factory - self._default_language_server = next(iter(language_servers.values())) - self._root_path = self._default_language_server.repository_root_path + + @property + def _default_language_server(self) -> SolidLanguageServer: + if len(self._language_servers) == 0: + raise ValueError("No language servers available in the manager") + return next(iter(self._language_servers.values())) @staticmethod def from_languages(languages: list[Language], factory: LanguageServerFactory) -> "LanguageServerManager": @@ -138,25 +143,26 @@ class LanguageServerManager: return LanguageServerManager(language_servers, factory) - def get_root_path(self) -> str: - return self._root_path - def _ensure_functional_ls(self, ls: SolidLanguageServer) -> SolidLanguageServer: if not ls.is_running(): log.warning(f"Language server for language {ls.language} is not running; restarting ...") ls = self.restart_language_server(ls.language) return ls + def _get_suitable_language_server(self, relative_path: str) -> SolidLanguageServer | None: + """:param relative_path: relative path to a file""" + for candidate in self._language_servers.values(): + if not candidate.is_ignored_path(relative_path, ignore_unsupported_files=True): + return candidate + return None + def get_language_server(self, relative_path: str) -> SolidLanguageServer: """:param relative_path: relative path to a file""" ls: SolidLanguageServer | None = None if len(self._language_servers) > 1: if os.path.isdir(relative_path): raise ValueError(f"Expected a file path, but got a directory: {relative_path}") - for candidate in self._language_servers.values(): - if not candidate.is_ignored_path(relative_path, ignore_unsupported_files=True): - ls = candidate - break + ls = self._get_suitable_language_server(relative_path) if ls is None: ls = self._default_language_server return self._ensure_functional_ls(ls) @@ -241,3 +247,6 @@ class LanguageServerManager: for ls in self.iter_language_servers(): if ls.is_running(): ls.save_cache() + + def has_suitable_ls_for_file(self, relative_file_path: str) -> bool: + return self._get_suitable_language_server(relative_file_path) is not None diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 83b8278b..c83dadc7 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -341,8 +341,12 @@ class SerenaMCPFactory: openai_tool_compatible = self.context.name in ["chatgpt", "codex", "oaicompat-agent"] self._set_mcp_tools(mcp_server, openai_tool_compatible=openai_tool_compatible) log.info("MCP server lifetime setup complete") - yield - log.info("MCP server shutting down") + try: + yield + finally: + log.info("MCP server shutting down") + if self.agent is not None: + self.agent.shutdown() def _get_initial_instructions(self) -> str: assert self.agent is not None diff --git a/src/serena/project.py b/src/serena/project.py index 59b89ec4..5995c3e7 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -1,103 +1,228 @@ import json import logging import os +import re +import shutil import threading +from collections.abc import Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal, Optional import pathspec from sensai.util.logging import LogTime -from sensai.util.string import ToStringMixin +from sensai.util.string import TextBuilder, ToStringMixin from serena.config.serena_config import ( - DEFAULT_TOOL_TIMEOUT, ProjectConfig, - get_serena_managed_in_project_dir, + SerenaConfig, + SerenaPaths, ) -from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME +from serena.constants import SERENA_FILE_ENCODING from serena.ls_manager import LanguageServerFactory, LanguageServerManager from serena.util.file_system import GitignoreParser, match_path -from serena.util.text_utils import MatchedConsecutiveLines, search_files +from serena.util.text_utils import ContentReplacer, MatchedConsecutiveLines, search_files from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import FileUtils if TYPE_CHECKING: - from serena.config.serena_config import SerenaConfig + from serena.agent import SerenaAgent log = logging.getLogger(__name__) class MemoriesManager: - def __init__(self, project_root: str): - self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" - self._memory_dir.mkdir(parents=True, exist_ok=True) + GLOBAL_TOPIC = "global" + _global_memory_dir = SerenaPaths().global_memories_path + + def __init__( + self, + serena_data_folder: str | Path | None, + read_only_memory_patterns: Sequence[str] = (), + ignored_memory_patterns: Sequence[str] = (), + ): + """ + :param serena_data_folder: the absolute path to the project's .serena data folder + :param read_only_memory_patterns: whether to allow writing global memories in tool execution contexts + :param ignored_memory_patterns: regex patterns for memories to completely exclude from listing, reading, and writing. + Matching memories will not appear in list_memories or activate_project output and cannot be accessed + via read_memory or write_memory. Use read_file on the raw path to access ignored memory files. + """ + self._project_memory_dir: Path | None = None + if serena_data_folder is not None: + self._project_memory_dir = Path(serena_data_folder) / "memories" + self._project_memory_dir.mkdir(parents=True, exist_ok=True) self._encoding = SERENA_FILE_ENCODING + self._read_only_memory_patterns = [re.compile(pattern) for pattern in set(read_only_memory_patterns)] + self._ignored_memory_patterns = [re.compile(pattern) for pattern in set(ignored_memory_patterns)] + + def _is_read_only_memory(self, name: str) -> bool: + for pattern in self._read_only_memory_patterns: + if pattern.fullmatch(name): + return True + return False + + def _is_ignored_memory(self, name: str) -> bool: + for pattern in self._ignored_memory_patterns: + if pattern.fullmatch(name): + return True + return False + + def _check_not_ignored(self, name: str) -> None: + if self._is_ignored_memory(name): + raise ValueError( + f"Memory '{name}' matches an ignored_memory_patterns pattern and cannot be accessed. " + f"Use the read_file tool on the raw file path instead." + ) + + def _is_global(self, name: str) -> bool: + return name == self.GLOBAL_TOPIC or name.startswith(self.GLOBAL_TOPIC + "/") def get_memory_file_path(self, name: str) -> Path: # Strip .md extension if present name = name.replace(".md", "") - # Split by "/" to handle subdirectories + if self._is_global(name): + if name == self.GLOBAL_TOPIC: + raise ValueError( + f'Bare "{self.GLOBAL_TOPIC}" is not a valid memory name. Use "{self.GLOBAL_TOPIC}/" to address a global memory.' + ) + # Strip "global/" prefix and resolve against global dir + sub_name = name[len(self.GLOBAL_TOPIC) + 1 :] + parts = sub_name.split("/") + filename = f"{parts[-1]}.md" + if len(parts) > 1: + subdir = self._global_memory_dir / "/".join(parts[:-1]) + subdir.mkdir(parents=True, exist_ok=True) + return subdir / filename + return self._global_memory_dir / filename + + # Project-local memory + assert self._project_memory_dir is not None, "Project dir was not passed at initialization" parts = name.split("/") filename = f"{parts[-1]}.md" if len(parts) > 1: # Create subdirectory path - subdir = self._memory_dir / "/".join(parts[:-1]) + subdir = self._project_memory_dir / "/".join(parts[:-1]) subdir.mkdir(parents=True, exist_ok=True) return subdir / filename - return self._memory_dir / filename + return self._project_memory_dir / filename + + def _check_write_access(self, name: str, is_tool_context: bool) -> None: + # in tool context, memories can be read-only + if is_tool_context and self._is_read_only_memory(name): + raise PermissionError(f"Attempted to write to read_only memory: '{name}')") def load_memory(self, name: str) -> str: + self._check_not_ignored(name) memory_file_path = self.get_memory_file_path(name) if not memory_file_path.exists(): return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it." with open(memory_file_path, encoding=self._encoding) as f: return f.read() - def save_memory(self, name: str, content: str) -> str: + def save_memory(self, name: str, content: str, is_tool_context: bool) -> str: + self._check_not_ignored(name) + self._check_write_access(name, is_tool_context) memory_file_path = self.get_memory_file_path(name) with open(memory_file_path, "w", encoding=self._encoding) as f: f.write(content) return f"Memory {name} written." - def list_memories(self, topic: str = "") -> list[str]: + class MemoriesList: + def __init__(self) -> None: + self.memories: list[str] = [] + self.read_only_memories: list[str] = [] + + def __len__(self) -> int: + return len(self.memories) + len(self.read_only_memories) + + def add(self, memory_name: str, is_read_only: bool) -> None: + if is_read_only: + self.read_only_memories.append(memory_name) + else: + self.memories.append(memory_name) + + def extend(self, other: "MemoriesManager.MemoriesList") -> None: + self.memories.extend(other.memories) + self.read_only_memories.extend(other.read_only_memories) + + def to_dict(self) -> dict[str, list[str]]: + result = {} + if self.memories: + result["memories"] = sorted(self.memories) + if self.read_only_memories: + result["read_only_memories"] = sorted(self.read_only_memories) + return result + + def get_full_list(self) -> list[str]: + return sorted(self.memories + self.read_only_memories) + + def _list_memories(self, search_dir: Path, base_dir: Path, prefix: str = "") -> MemoriesList: + result = self.MemoriesList() + if not search_dir.exists(): + return result + for md_file in search_dir.rglob("*.md"): + rel = str(md_file.relative_to(base_dir).with_suffix("")).replace(os.sep, "/") + memory_name = prefix + rel + if self._is_ignored_memory(memory_name): + continue + result.add(memory_name, is_read_only=self._is_read_only_memory(memory_name)) + return result + + def list_global_memories(self, subtopic: str = "") -> MemoriesList: + dir_path = self._global_memory_dir + if subtopic: + dir_path = dir_path / subtopic.replace("/", os.sep) + return self._list_memories(dir_path, self._global_memory_dir, self.GLOBAL_TOPIC + "/") + + def list_project_memories(self, topic: str = "") -> MemoriesList: + assert self._project_memory_dir is not None, "Project dir was not passed at initialization" + dir_path = self._project_memory_dir + if topic: + dir_path = dir_path / topic.replace("/", os.sep) + return self._list_memories(dir_path, self._project_memory_dir) + + def list_memories(self, topic: str = "") -> MemoriesList: """ - List memories, optionally filtered by topic. + Lists all memories, optionally filtered by topic. + If the topic is omitted, both global and project-specific memories are returned. """ - memories = [] + memories: MemoriesManager.MemoriesList if topic: - # Only list memories in specified subdirectory - search_dir = self._memory_dir / topic.replace("/", os.sep) - if not search_dir.exists(): - return [] + if self._is_global(topic): + topic_parts = topic.split("/") + subtopic = "/".join(topic_parts[1:]) + memories = self.list_global_memories(subtopic=subtopic) + else: + memories = self.list_project_memories(topic=topic) else: - search_dir = self._memory_dir + memories = self.list_project_memories() + memories.extend(self.list_global_memories()) - # Recursively find all .md files - for md_file in search_dir.rglob("*.md"): - # Calculate relative path as memory name - rel_path = md_file.relative_to(self._memory_dir) - name = str(rel_path.with_suffix("")).replace(os.sep, "/") - memories.append(name) + return memories - # Sort alphabetically by name - return sorted(memories) - - def delete_memory(self, name: str) -> str: + def delete_memory(self, name: str, is_tool_context: bool) -> str: + self._check_not_ignored(name) + self._check_write_access(name, is_tool_context) memory_file_path = self.get_memory_file_path(name) if not memory_file_path.exists(): return f"Memory {name} not found." memory_file_path.unlink() return f"Memory {name} deleted." - def rename_memory(self, old_name: str, new_name: str) -> str: + def move_memory(self, old_name: str, new_name: str, is_tool_context: bool) -> str: """ Rename or move a memory file. + Moving between global and project scope (e.g. "global/foo" -> "bar") is supported. """ + self._check_not_ignored(old_name) + self._check_not_ignored(new_name) + self._check_write_access(new_name, is_tool_context) + old_path = self.get_memory_file_path(old_name) new_path = self.get_memory_file_path(new_name) @@ -106,38 +231,79 @@ class MemoriesManager: if new_path.exists(): raise FileExistsError(f"Memory {new_name} already exists.") - # Ensure target directory exists new_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(old_path, new_path) - # Move/rename the file - old_path.rename(new_path) return f"Memory renamed from {old_name} to {new_name}." + def edit_memory( + self, name: str, needle: str, repl: str, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, is_tool_context: bool + ) -> str: + """ + Edit a memory by replacing content matching a pattern. + + :param name: the memory name + :param needle: the string or regex to search for + :param repl: the replacement string + :param mode: "literal" or "regex" + :param allow_multiple_occurrences: + """ + self._check_not_ignored(name) + self._check_write_access(name, is_tool_context) + memory_file_path = self.get_memory_file_path(name) + if not memory_file_path.exists(): + raise FileNotFoundError(f"Memory {name} not found.") + with open(memory_file_path, encoding=self._encoding) as f: + original_content = f.read() + replacer = ContentReplacer(mode=mode, allow_multiple_occurrences=allow_multiple_occurrences) + updated_content = replacer.replace(original_content, needle, repl) + with open(memory_file_path, "w", encoding=self._encoding) as f: + f.write(updated_content) + return f"Memory {name} edited successfully." + class Project(ToStringMixin): def __init__( self, + *, project_root: str, project_config: ProjectConfig, + serena_config: SerenaConfig, is_newly_created: bool = False, - serena_config: "SerenaConfig | None" = None, ): + assert serena_config is not None self.project_root = project_root self.project_config = project_config - self.memories_manager = MemoriesManager(project_root) + self.serena_config = serena_config + self._serena_data_folder = serena_config.get_project_serena_folder(self.project_root) + log.info("Serena project data folder: %s", self._serena_data_folder) + + read_only_memory_patterns = serena_config.read_only_memory_patterns + project_config.read_only_memory_patterns + ignored_memory_patterns = serena_config.ignored_memory_patterns + project_config.ignored_memory_patterns + self.memories_manager = MemoriesManager( + self._serena_data_folder, + read_only_memory_patterns=read_only_memory_patterns, + ignored_memory_patterns=ignored_memory_patterns, + ) + + # resolve line ending (project -> global) + self.line_ending = project_config.line_ending or serena_config.line_ending + self.language_server_manager: LanguageServerManager | None = None + self._language_server_manager_init_error: Exception | None = None self._is_newly_created = is_newly_created + self._agent: Optional["SerenaAgent"] = None # create .gitignore file in the project's Serena data folder if not yet present - serena_data_gitignore_path = os.path.join(self.path_to_serena_data_folder(), ".gitignore") + serena_data_gitignore_path = os.path.join(self._serena_data_folder, ".gitignore") if not os.path.exists(serena_data_gitignore_path): os.makedirs(os.path.dirname(serena_data_gitignore_path), exist_ok=True) log.info(f"Creating .gitignore file in {serena_data_gitignore_path}") with open(serena_data_gitignore_path, "w", encoding="utf-8") as f: f.write(f"/{SolidLanguageServer.CACHE_FOLDER_NAME}\n") + f.write(f"/{ProjectConfig.SERENA_LOCAL_PROJECT_FILE}\n") # prepare ignore spec asynchronously, ensuring immediate project activation. - self._serena_config = serena_config self.__ignored_patterns: list[str] self.__ignore_spec: pathspec.PathSpec self._ignore_spec_available = threading.Event() @@ -145,9 +311,8 @@ class Project(ToStringMixin): def _gather_ignorespec(self) -> None: with LogTime(f"Gathering ignore spec for project {self.project_config.project_name}", logger=log): - # gather ignored paths from the global configuration, project configuration, and gitignore files - global_ignored_paths = self._serena_config.ignored_paths if self._serena_config else [] + global_ignored_paths = self.serena_config.ignored_paths ignored_patterns = list(global_ignored_paths) + list(self.project_config.ignored_paths) if len(global_ignored_paths) > 0: log.info(f"Using {len(global_ignored_paths)} ignored paths from the global configuration.") @@ -166,7 +331,7 @@ class Project(ToStringMixin): # Set up the pathspec matcher for the ignored paths # for all absolute paths in ignored_paths, convert them to relative paths processed_patterns = [] - for pattern in set(ignored_patterns): + for pattern in ignored_patterns: # Normalize separators (pathspec expects forward slashes) pattern = pattern.replace(os.path.sep, "/") processed_patterns.append(pattern) @@ -181,6 +346,9 @@ class Project(ToStringMixin): def _tostring_additional_entries(self) -> dict[str, Any]: return {"root": self.project_root, "name": self.project_name} + def set_agent(self, agent: "SerenaAgent") -> None: + self._agent = agent + @property def project_name(self) -> str: return self.project_config.project_name @@ -189,26 +357,27 @@ class Project(ToStringMixin): def load( cls, project_root: str | Path, - serena_config: "SerenaConfig | None", + serena_config: "SerenaConfig", autogenerate: bool = True, ) -> "Project": + assert serena_config is not None project_root = Path(project_root).resolve() if not project_root.exists(): raise FileNotFoundError(f"Project root not found: {project_root}") - project_config = ProjectConfig.load(project_root, autogenerate=autogenerate) + project_config = ProjectConfig.load(project_root, serena_config=serena_config, autogenerate=autogenerate) return Project(project_root=str(project_root), project_config=project_config, serena_config=serena_config) def save_config(self) -> None: """ Saves the current project configuration to disk. """ - self.project_config.save(self.project_root) + self.project_config.save(self.path_to_project_yml()) def path_to_serena_data_folder(self) -> str: - return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME) + return self._serena_data_folder def path_to_project_yml(self) -> str: - return os.path.join(self.project_root, self.project_config.rel_path_to_project_yml()) + return self.serena_config.get_project_yml_location(self.project_root) def get_activation_message(self) -> str: """ @@ -220,10 +389,10 @@ class Project(ToStringMixin): msg = f"The project with name '{self.project_name}' at {self.project_root} is activated." languages_str = ", ".join([lang.value for lang in self.project_config.languages]) msg += f"\nProgramming languages: {languages_str}; file encoding: {self.project_config.encoding}" - memories = self.memories_manager.list_memories() - if memories: + project_memories = self.memories_manager.list_project_memories() + if project_memories: msg += ( - f"\nAvailable project memories: {json.dumps(memories)}\n" + f"\nAvailable project memories: {json.dumps(project_memories.to_dict())}\n" + "Use the `read_memory` tool to read these memories later if they are relevant to the task." ) if self.project_config.initial_prompt: @@ -464,39 +633,57 @@ class Project(ToStringMixin): source_file_path=relative_file_path, ) - def create_language_server_manager( - self, - log_level: int = logging.INFO, - ls_timeout: float | None = DEFAULT_TOOL_TIMEOUT - 5, - trace_lsp_communication: bool = False, - ls_specific_settings: dict[Language, Any] | None = None, - ) -> LanguageServerManager: + def create_language_server_manager(self) -> LanguageServerManager: """ Creates the language server manager for the project, starting one language server per configured programming language. - :param log_level: the log level for the language server - :param ls_timeout: the timeout for the language server - :param trace_lsp_communication: whether to trace LSP communication - :param ls_specific_settings: optional LS specific configuration of the language server, - see docstrings in the inits of subclasses of SolidLanguageServer to see what values may be passed. :return: the language server manager, which is also stored in the project instance """ - # if there is an existing instance, stop its language servers first - if self.language_server_manager is not None: - log.info("Stopping existing language server manager ...") - self.language_server_manager.stop_all() - self.language_server_manager = None + try: + # determine timeout to use for LS calls + tool_timeout = self.serena_config.tool_timeout + if tool_timeout is None or tool_timeout < 0: + ls_timeout = None + else: + if tool_timeout < 10: + raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds") + ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout - log.info(f"Creating language server manager for {self.project_root}") - factory = LanguageServerFactory( - project_root=self.project_root, - encoding=self.project_config.encoding, - ignored_patterns=self._ignored_patterns, - ls_timeout=ls_timeout, - ls_specific_settings=ls_specific_settings, - trace_lsp_communication=trace_lsp_communication, - ) - self.language_server_manager = LanguageServerManager.from_languages(self.project_config.languages, factory) + # if there is an existing instance, stop its language servers first + if self.language_server_manager is not None: + log.info("Stopping existing language server manager ...") + self.language_server_manager.stop_all() + self.language_server_manager = None + + log.info(f"Creating language server manager for {self.project_root}") + self._language_server_manager_init_error = None + ls_specific_settings = {**self.serena_config.ls_specific_settings, **self.project_config.ls_specific_settings} + factory = LanguageServerFactory( + project_root=self.project_root, + project_data_path=self._serena_data_folder, + encoding=self.project_config.encoding, + ignored_patterns=self._ignored_patterns, + ls_timeout=ls_timeout, + ls_specific_settings=ls_specific_settings, + trace_lsp_communication=self.serena_config.trace_lsp_communication, + ) + self.language_server_manager = LanguageServerManager.from_languages(self.project_config.languages, factory) + return self.language_server_manager + except Exception as e: + self._language_server_manager_init_error = e + raise + + def get_language_server_manager_or_raise(self) -> LanguageServerManager: + if self.language_server_manager is None: + msg = TextBuilder("The language server manager is not initialized, indicating a problem during project initialisation.") + if self._language_server_manager_init_error is not None: + msg.with_text(str(self._language_server_manager_init_error)) + if self._agent is not None: + msg.with_text("For details, please check the logs. " + self._agent.get_log_inspection_instructions()) + msg.with_text( + "IMPORTANT: Stop, do not attempt workarounds. Inform the user and wait for further instructions before you continue!" + ) + raise Exception(msg.build()) return self.language_server_manager def add_language(self, language: Language) -> None: diff --git a/src/serena/project_server.py b/src/serena/project_server.py new file mode 100644 index 00000000..ef88e883 --- /dev/null +++ b/src/serena/project_server.py @@ -0,0 +1,157 @@ +import json +import logging +from typing import TYPE_CHECKING + +import requests as requests_lib +from flask import Flask, request +from pydantic import BaseModel +from sensai.util.logging import LogTime + +from serena.config.serena_config import LanguageBackend, SerenaConfig +from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient + +if TYPE_CHECKING: + from serena.project import Project + +log = logging.getLogger(__name__) + +# disable Werkzeug's logging to avoid cluttering the output +logging.getLogger("werkzeug").setLevel(logging.WARNING) + + +class QueryProjectRequest(BaseModel): + """ + Request model for the /query_project endpoint, matching the interface of + :class:`~serena.tools.query_project_tools.QueryProjectTool`. + """ + + project_name: str + tool_name: str + tool_params_json: str + + +class ProjectServer: + """ + A lightweight Flask server that exposes a SerenaAgent's project querying + capabilities via HTTP, using the LSP language server backend for symbolic retrieval. + + Projects are loaded on demand when a query is made for them, and cached in memory for subsequent queries. + + The server instantiates a :class:`SerenaAgent` with default options and + provides a ``/query_project`` endpoint whose interface matches + :class:`~serena.tools.query_project_tools.QueryProjectTool`. + """ + + PORT = JetBrainsPluginClient.BASE_PORT - 1 + + def __init__(self) -> None: + from serena.agent import SerenaAgent + + serena_config = SerenaConfig.from_config_file() + serena_config.gui_log_window = False + serena_config.web_dashboard = False + serena_config.language_backend = LanguageBackend.LSP + + self._agent = SerenaAgent(serena_config=serena_config) + self._loaded_projects_by_name: dict[str, "Project"] = {} + + # create the Flask application + self._app = Flask(__name__) + self._setup_routes() + + def _setup_routes(self) -> None: + @self._app.route("/heartbeat", methods=["GET"]) + def heartbeat() -> dict[str, str]: + return {"status": "alive"} + + @self._app.route("/query_project", methods=["POST"]) + def query_project() -> str: + query_request = QueryProjectRequest.model_validate(request.get_json()) + return self._query_project(query_request) + + def _get_project(self, project_name: str) -> "Project": + """Gets the project with the given name, loading it if necessary.""" + if project_name in self._loaded_projects_by_name: + return self._loaded_projects_by_name[project_name] + else: + serena_config = self._agent.serena_config + registered_project = serena_config.get_registered_project(project_name) + if registered_project is None: + raise ValueError(f"Project '{project_name}' is not registered with Serena.") + with LogTime(f"Loading project '{project_name}'"): + project = registered_project.get_project_instance(serena_config) + project.create_language_server_manager() + self._loaded_projects_by_name[project_name] = project + return project + + def _query_project(self, req: QueryProjectRequest) -> str: + """Handle a /query_project request by invoking the agent on the specified project and tool.""" + project = self._get_project(req.project_name) + with self._agent.active_project_context(project): + tool = self._agent.get_tool_by_name(req.tool_name) + params = json.loads(req.tool_params_json) + return tool.apply_ex(**params) + + def run(self, host: str = "127.0.0.1", port: int = PORT) -> int: + """Run the server on the given host and port. + + :param host: the host address to listen on. + :param port: the port to listen on. + :return: the port number the server is running on. + """ + from flask import cli + + # suppress the default Flask startup banner + cli.show_server_banner = lambda *args, **kwargs: None + + self._app.run(host=host, port=port, debug=False, use_reloader=False, threaded=True) + return port + + +class ProjectServerClient: + """Client for interacting with a running :class:`ProjectServer`. + + Upon instantiation, the client verifies that the server is reachable + by sending a heartbeat request. If the server is not running, a + :class:`ConnectionError` is raised. + """ + + def __init__(self, host: str = "127.0.0.1", port: int = ProjectServer.PORT, timeout: int = 300) -> None: + """ + :param host: the host address of the project server. + :param port: the port of the project server. + :raises ConnectionError: if the project server is not reachable. + """ + self._base_url = f"http://{host}:{port}" + self._timeout = timeout + + # verify that the server is running + try: + response = requests_lib.get(f"{self._base_url}/heartbeat", timeout=5) + response.raise_for_status() + except requests_lib.ConnectionError: + raise ConnectionError(f"ProjectServer is not reachable at {self._base_url}. Make sure the server is running.") + except requests_lib.RequestException as e: + raise ConnectionError(f"ProjectServer health check failed: {e}") + + def query_project(self, project_name: str, tool_name: str, tool_params_json: str) -> str: + """ + Query a project by executing a Serena tool in its context. + + The interface matches :meth:`QueryProjectTool.apply + `. + + :param project_name: the name of the project to query. + :param tool_name: the name of the tool to execute. The tool must be read-only. + :param tool_params_json: the parameters to pass to the tool, encoded as a JSON string. + :return: the tool's result as a string. + """ + payload = QueryProjectRequest( + project_name=project_name, + tool_name=tool_name, + tool_params_json=tool_params_json, + ).model_dump() + + response = requests_lib.post(f"{self._base_url}/query_project", json=payload, timeout=self._timeout) + response.raise_for_status() + return response.text diff --git a/src/serena/resources/config/modes/no-memories.yml b/src/serena/resources/config/modes/no-memories.yml index 7a522875..08602141 100644 --- a/src/serena/resources/config/modes/no-memories.yml +++ b/src/serena/resources/config/modes/no-memories.yml @@ -6,6 +6,7 @@ excluded_tools: - read_memory - delete_memory - edit_memory + - rename_memory - list_memories - onboarding - check_onboarding_performed diff --git a/src/serena/resources/config/modes/query-projects.yml b/src/serena/resources/config/modes/query-projects.yml new file mode 100644 index 00000000..ed140de9 --- /dev/null +++ b/src/serena/resources/config/modes/query-projects.yml @@ -0,0 +1,8 @@ +description: Enables tools that allow inactive projects to be queried +prompt: | + You can use the 'query_project' tool to query Serena projects without activating them. + Use this when a project is related to the active project and you need to query it for information. +excluded_tools: [] +included_optional_tools: + - list_queryable_projects + - query_project diff --git a/src/serena/resources/config/prompt_templates/system_prompt.yml b/src/serena/resources/config/prompt_templates/system_prompt.yml index 13064c14..125a6818 100644 --- a/src/serena/resources/config/prompt_templates/system_prompt.yml +++ b/src/serena/resources/config/prompt_templates/system_prompt.yml @@ -35,10 +35,13 @@ prompts: You can understand relationships between symbols by using the `find_referencing_symbols` tool. {% endif %} - {% if 'read_memory' in available_tools %} + {% if 'read_memory' in available_tools -%} You generally have access to memories and it may be useful for you to read them. You infer whether memories are relevant based on their names. - {% endif %} + {% if global_memories_list -%} + The following global (not project-specific) memories are available to you: {{ global_memories_list }} + {%- endif -%} + {%- endif %} The context and modes of operation are described below. These determine how to interact with your user and which kinds of interactions are expected of you. diff --git a/src/serena/resources/dashboard/dashboard.css b/src/serena/resources/dashboard/dashboard.css index 662fed57..eb473e9a 100644 --- a/src/serena/resources/dashboard/dashboard.css +++ b/src/serena/resources/dashboard/dashboard.css @@ -836,12 +836,11 @@ body { border: 1px solid var(--border-color); border-radius: 5px; height: calc(100vh - var(--header-height) - 2 * var(--header-padding) - 3 * var(--header-gap-main)); - max-height: 700px; overflow-y: auto; overflow-x: auto; padding: 10px; white-space: pre-wrap; - font-size: 12px; + font-size: 14px; line-height: 1.4; color: var(--text-primary); transition: background-color 0.3s ease, border-color 0.3s ease, color 0.3s ease; @@ -1308,6 +1307,42 @@ body { gap: 8px; } +/* Memory Rename Styles */ +.memory-rename-btn { + display: inline-flex; + align-items: center; + justify-content: center; + width: 24px; + height: 24px; + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 4px; + cursor: pointer; + color: var(--text-muted); + opacity: 0.5; + transition: opacity 0.2s ease, background-color 0.2s ease; +} + +.memory-rename-btn:hover { + opacity: 1; + background-color: var(--border-color); +} + +.memory-rename-input { + font-size: inherit; + font-weight: inherit; + font-family: inherit; + color: var(--text-primary); + background: transparent; + border: none; + border-bottom: 1px solid var(--btn-primary); + outline: none; + padding: 0; + flex: 1; + min-width: 200px; + max-width: 80%; +} + /* Log Action Buttons (Save, Copy, Clear) */ .log-action-buttons { position: absolute; diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index f59dbb6a..5819cec3 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -307,6 +307,8 @@ class Dashboard { this.$modalCloseRemove = $('.modal-close-remove'); this.$editMemoryModal = $('#edit-memory-modal'); this.$editMemoryName = $('#edit-memory-name'); + this.$editMemoryRenameBtn = $('#edit-memory-rename-btn'); + this.$editMemoryRenameInput = $('#edit-memory-rename-input'); this.$editMemoryContent = $('#edit-memory-content'); this.$editMemorySaveBtn = $('#edit-memory-save-btn'); this.$editMemoryCancelBtn = $('#edit-memory-cancel-btn'); @@ -365,6 +367,19 @@ class Dashboard { this.$editMemoryCancelBtn.click(this.closeEditMemoryModal.bind(this)); this.$modalCloseEditMemory.click(this.closeEditMemoryModal.bind(this)); this.$editMemoryContent.on('input', this.trackMemoryChanges.bind(this)); + this.$editMemoryRenameBtn.click(this.startMemoryRename.bind(this)); + this.$editMemoryRenameInput.keydown(function (e) { + if (e.which === 13) { // Enter key + e.preventDefault(); + self.commitMemoryRename(); + } else if (e.which === 27) { // Escape key + e.preventDefault(); + self.cancelMemoryRename(); + } + }); + this.$editMemoryRenameInput.on('blur', function () { + self.cancelMemoryRename(); + }); this.$deleteMemoryOkBtn.click(this.confirmDeleteMemoryOk.bind(this)); this.$deleteMemoryCancelBtn.click(this.closeDeleteMemoryModal.bind(this)); this.$modalCloseDeleteMemory.click(this.closeDeleteMemoryModal.bind(this)); @@ -1887,6 +1902,62 @@ class Dashboard { }); } + startMemoryRename() { + this.$editMemoryName.hide(); + this.$editMemoryRenameBtn.hide(); + this.$editMemoryRenameInput.val(this.currentMemoryName).show().focus().select(); + } + + cancelMemoryRename() { + this.$editMemoryRenameInput.hide(); + this.$editMemoryName.show(); + this.$editMemoryRenameBtn.show(); + } + + commitMemoryRename() { + const newName = this.$editMemoryRenameInput.val().trim(); + const oldName = this.currentMemoryName; + + // If name unchanged, just cancel + if (!newName || newName === oldName) { + this.cancelMemoryRename(); + return; + } + + // Validate memory name (alphanumeric, underscores, and slashes for subdirectories) + if (!/^[a-zA-Z0-9_]+(?:\/[a-zA-Z0-9_]+)*$/.test(newName)) { + alert('Memory name can only contain letters, numbers, underscores, and "/" for subdirectories (e.g., "topic/memory_name")'); + this.$editMemoryRenameInput.focus(); + return; + } + + const self = this; + this.$editMemoryRenameInput.prop('disabled', true); + + $.ajax({ + url: '/rename_memory', type: 'POST', contentType: 'application/json', data: JSON.stringify({ + old_name: oldName, new_name: newName + }), success: function (response) { + if (response.status === 'success') { + self.currentMemoryName = newName; + self.$editMemoryName.text(newName); + self.cancelMemoryRename(); + // Reload config to reflect the rename in the memory list + self.loadConfigOverview(); + } else { + alert('Error: ' + response.message); + self.$editMemoryRenameInput.focus(); + } + }, error: function (xhr, status, error) { + console.error('Error renaming memory:', error); + alert('Error renaming memory: ' + (xhr.responseJSON ? xhr.responseJSON.message : error)); + self.$editMemoryRenameInput.focus(); + }, complete: function () { + self.$editMemoryRenameInput.prop('disabled', false); + } + }); + } + confirmDeleteMemory(memoryName) { // Set memory name to delete this.memoryToDelete = memoryName; @@ -1960,9 +2031,9 @@ class Dashboard { return; } - // Validate memory name (alphanumeric and underscores only) - if (!/^[a-zA-Z0-9_]+$/.test(memoryName)) { - alert('Memory name can only contain letters, numbers, and underscores'); + // Validate memory name (alphanumeric, underscores, and slashes for subdirectories) + if (!/^[a-zA-Z0-9_]+(?:\/[a-zA-Z0-9_]+)*$/.test(memoryName)) { + alert('Memory name can only contain letters, numbers, underscores, and "/" for subdirectories (e.g., "topic/memory_name")'); return; } diff --git a/src/serena/resources/dashboard/index.html b/src/serena/resources/dashboard/index.html index c8bf75d7..874aecf9 100644 --- a/src/serena/resources/dashboard/index.html +++ b/src/serena/resources/dashboard/index.html @@ -275,7 +275,17 @@