diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 1b036c3f..350288e1 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,6 +1,6 @@ name: Tests on Ubuntu -on: [push, pull_request] +on: [pull_request] jobs: cpu: @@ -16,8 +16,14 @@ jobs: with: python-version: "${{ matrix.python-version }}" - # Include the appropriate package manager steps based on the selection - + # Add Go bin directory to PATH for this workflow + # GITHUB_PATH is a special file that GitHub Actions uses to modify PATH + # Writing to this file adds the directory to the PATH for subsequent steps + - name: Install gopls + run: | + go install golang.org/x/tools/gopls@latest + echo "$HOME/go/bin" >> $GITHUB_PATH + - name: Install uv run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Create virtual environment diff --git a/.gitignore b/.gitignore index b04a9643..fc862162 100644 --- a/.gitignore +++ b/.gitignore @@ -138,7 +138,7 @@ pylint.html # Serena-specific /*.yml -!myproject.demo.yml +!*.template.yml /agent-ui # dynamic LS installations /src/multilspy/language_servers/*/static diff --git a/.serena/memories/language_servers.md b/.serena/memories/language_servers.md new file mode 100644 index 00000000..f6ea4d0b --- /dev/null +++ b/.serena/memories/language_servers.md @@ -0,0 +1,16 @@ +# Available Language Servers + +Serena supports multiple programming languages through language servers. The language servers currently available in the project are: + +## Directly Supported (Out of the Box) +- **Python**: Using Pyright Language Server (previously used Jedi) +- **Java**: Using Eclipse JDTLS (Note: startup is slow, especially initial startup) +- **TypeScript/JavaScript**: Using TypeScript Language Server + +## Indirectly Supported (May Require Manual Setup) +- **Ruby**: Using Solargraph +- **Go**: Using Gopls +- **C#**: Using OmniSharp +- **Rust**: Using Rust-Analyzer + +These language servers utilize the Language Server Protocol (LSP) to provide semantic code analysis capabilities. The system can be extended to support additional languages by providing adapters for new language server implementations. \ No newline at end of file diff --git a/.serena/memories/project_structure.md b/.serena/memories/project_structure.md new file mode 100644 index 00000000..c0e77127 --- /dev/null +++ b/.serena/memories/project_structure.md @@ -0,0 +1,16 @@ +# Project Structure + +The Serena codebase is organized as follows: + +- **src/serena/**: Main package with core functionality + - **llm/**: LLM integration + - **util/**: Utilities + - Key files: mcp.py, agno.py, agent.py +- **src/multilspy/**: Language server integration + - **language_servers/**: Implementations for different languages + - Python (pyright/jedi) + - Java (Eclipse JDTLS) + - TypeScript/JavaScript + - C#, Rust, Go, Ruby, C++ +- **test/**: Tests +- **scripts/**: Utility scripts \ No newline at end of file diff --git a/.serena/memories/suggested_commands.md b/.serena/memories/suggested_commands.md new file mode 100644 index 00000000..920b4d0c --- /dev/null +++ b/.serena/memories/suggested_commands.md @@ -0,0 +1,13 @@ +# Suggested Commands + +## Development Tasks (using uv and poe) + +The following tasks should generally be executed using `uv run poe `. + +- `lint`: This is the **only** allowed command for linting. Run as `uv run poe lint`. +- `format`: This is the **only** allowed command for formatting. Run as `uv run poe format`. +- `type-check`: This is the **only** allowed command for type checking. Run as `uv run poe type-check`. +- `test`: This is the preferred command for running tests (`uv run poe test [args]`). However, running tests directly with `uv run pytest [args]` is also permitted. + +For finishing a task, make sure format, type-check and test pass! Run them at the end of the task +and if needed fix any issues that come up and run them again until they pass. \ No newline at end of file diff --git a/.serena/project.yml b/.serena/project.yml new file mode 100644 index 00000000..e97251e5 --- /dev/null +++ b/.serena/project.yml @@ -0,0 +1,4 @@ +# language of the project (csharp, python, rust, java, typescript, javascript, go, or ruby) +language: python +# list of directories to ignore: either names (e.g. "temp") or relative paths (e.g. "build/foo") +ignore_all_files_in_gitignore: True diff --git a/.vscode/settings.json b/.vscode/settings.json index 3f32d63a..258e3b5c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,5 +10,7 @@ "pixi", "sensai", "vibing" - ] + ], + "basedpyright.disableLanguageServices": true, + "python.languageServer": "None" } \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd5f2ff4..5a3d7978 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,6 +45,13 @@ for you. Note: For the Windows subsystem for Linux (WSL), you may need to adjust the path for the volume. +## Running Tools Locally + +The Serena tools (and in fact all Serena code) can be executed without an LLM, and also without +any MCP specifics (though you can use the mcp inspector, if you want). + +An example script for running tools is provided in [scripts/demo_run_tools.py](scripts/demo_run_tools.py). + ## Adding a New Supported Language Serena interacts with code through language servers which are included in diff --git a/LICENSE b/LICENSE index d159169d..d27bba64 100644 --- a/LICENSE +++ b/LICENSE @@ -1,339 +1,21 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 +MIT License - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. +Copyright (c) 2025 Oraios AI - Preamble +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 378afbef..a81eaba7 100644 --- a/README.md +++ b/README.md @@ -150,9 +150,17 @@ Serena can be used in various ways, below you will find instructions for selecte 1. Install `uv` (instructions [here](https://docs.astral.sh/uv/getting-started/installation/)) 2. Clone the repository to `/path/to/serena`. 3. Copy `serena_config.template.yml` to `serena_config.yml` and adjust settings. -4. Copy `myproject.template.yml` to `myproject.yml` and adjust the settings specific to your project. - (Add one such file for each project you want Serena to work on.) -5. If you want Serena to dynamically switch between projects, add the list of all project files + ```shell + cp serena_config.template.yml serena_config.yml + ``` +4. Copy `project.template.yml` to `project.yml` and adjust the settings specific to your project + (add one such file for each project you want Serena to work on). We recommend that you copy + it to the `.serena` directory of your project, e.g., + ```shell + mkdir -p /myproject/.serena + cp project.template.yml /myproject/.serena/project.yml + ``` +6. If you want Serena to dynamically switch between projects, add the list of all project files created in the previous step to the `projects` list in `serena_config.yml`. > ⚠️ **Note:** Serena is under active development. We are continuously adding features, improving stability and the UX. @@ -188,7 +196,17 @@ want to use Serena. If you are using paths containing backslashes for paths on Windows (note that you can also just use forward slashes), be sure to escape them correctly (`\\`). -That's it! Save the config and then restart Claude Desktop. +That's it! Save the config and then restart Claude Desktop. + +#### Troubleshooting + +Some client/OS/setup configurations were reported to cause issues when using Serena with the standard `stdio` protocol, where the MCP server is started by the client application. +If you experience such problems, you can start Serena in `sse` mode by running, e.g., + +```shell +uv run --directory /path/to/serena serena-mcp-server --transport sse --port 9121 --project-file /path/to/project.yml +``` +(the `--project-file` option is optional). Then configure your client to connect to `http://localhost:9121`. Note: on Windows and macOS there are official Claude Desktop applications by Anthropic, for Linux there is an [open-source community version](https://github.com/aaddrick/claude-desktop-debian). @@ -211,6 +229,17 @@ therefore needs to be configured with a launch command. For more information on MCP servers with Claude Desktop, see [the official quick start guide](https://modelcontextprotocol.io/quickstart/user). +### Claude Code + +Serena is a great way to make Claude Code both cheaper and more powerful! We are collecting +several examples for that and have heard very positive feedback so far. Claude Code users can +add serena with + +```shell +claude mcp add serena -- /path/to/uv "run" --directory /path/to/serena serena-mcp-server --project-file /path/to/project.yml +``` + + ### Other MCP Clients - Cline, Roo-Code, Cursor, Windsurf etc. Being an MCP Server, Serena can be included in any MCP Client. The same config as above, @@ -348,14 +377,9 @@ this should not be a problem. However, if you have concerns, you can choose to disable certain commands in your project's .yml configuration file. If you only want to use Serena purely for analyzing code and suggesting implementations -without modifying the codebase, you can consider disabling the editing tools in the configuration, i.e. - - * `create_text_file` - * `insert_after_symbol` - * `insert_at_line` - * `insert_before_symbol` - * `replace_symbol_body` - * `delete_lines`. +without modifying the codebase, you can enable read-only mode by setting `read_only: true` in your project configuration file. +This will automatically disable all editing tools and prevent any modifications to your codebase while still +allowing all analysis and exploration capabilities. In general, be sure to back up your work and use a version control system in order to avoid losing any work. diff --git a/myproject.template.yml b/project.template.yml similarity index 87% rename from myproject.template.yml rename to project.template.yml index 51191a45..2722a095 100644 --- a/myproject.template.yml +++ b/project.template.yml @@ -2,7 +2,7 @@ # This is optional if this file is placed in the project directory under `.serena/project.yml`. project_root: /path/to/project -# language of the project (csharp, python, rust, java, typescript, javascript, go, or ruby) +# language of the project (csharp, python, rust, java, typescript, javascript, go, cpp, or ruby) # Special requirements: # * csharp: Requires the presence of a .sln file in the project folder. language: python @@ -10,13 +10,17 @@ language: python # whether to use the project's gitignore file to ignore files # Added on 2025-04-07 ignore_all_files_in_gitignore: true - # list of additional paths to ignore # same syntax as gitignore, so you can use * and ** # Was previously called `ignored_dirs`, please update your config if you are using that. # Added (renamed)on 2025-04-07 ignored_paths: [] +# whether the project is in read-only mode +# If set to true, all editing tools will be disabled and attempts to use them will result in an error +# Added on 2025-04-18 +read_only: false + # list of tool names to exclude. We recommend not excluding any tools, see the readme for more details. # Below is the complete list of tools for convenience. diff --git a/prompts/mcp_prompts.yml b/prompts/mcp_prompts.yml index 9a9f1cf6..641f08d2 100644 --- a/prompts/mcp_prompts.yml +++ b/prompts/mcp_prompts.yml @@ -19,7 +19,7 @@ prompts: * the util commands for the system, like `git`, `ls`, `cd`, `grep`, `find`, etc. Keep in mind that the system is {{ system }}, so the commands might be different than on a regular unix system. * whether there are particular guidelines, styles, design patterns, etc. that one should know about - + This list is not exhaustive, you can add more information if you think it is relevant. For doing that, you will need to acquire information about the project with the corresponding tools. @@ -51,9 +51,10 @@ prompts: think_about_whether_you_are_done: | Have you already performed all the steps required by the task? Is it appropriate to run tests and linting, and if so, - have you done that already? Should new tests be written to cover the changes? + have you done that already? Is it appropriate to adjust non-code files like documentation and config and have you done that already? + Should new tests be written to cover the changes? Note that a task that is just about exploring the codebase does not require running tests or linting. - Read the corresponding memory files to see what should be done when a task is completed. You might need to list them first. + Read the corresponding memory files to see what should be done when a task is completed. summarize_changes: | Summarize all the changes you have made to the codebase over the course of the conversation. @@ -74,6 +75,16 @@ prompts: system_prompt: | You are a professional coding agent who is concerned with one particular codebase. You have tools to explore the codebase iteratively, to edit files and to execute arbitrary commands. You heavily rely on these tools to solve the tasks given to you, - but you also operate in a frugal and intelligent manner, always - keeping in mind to not load content that is not needed for the task at hand. If you cannot find everything you need in the codebase - and generally when in doubt, you ask the user clarifying questions. + and you operate in a frugal and intelligent manner, always + keeping in mind to not load content that is not needed for the task at hand. + Whenever you want to read or edit code, you try using the symbolic tools first instead of just reading or editing files or lines. + For understanding symbol locations, you make sure to have read them using find symbol or + the overview tools. You only use read_file, replace_lines and so on if the + symbolic tools are not appropriate for some reason (e.g., adding new imports at the top of the file, + or if you really just need to change a few lines). + + If you cannot find everything you need in the codebase + and generally when in doubt, you ask the user clarifying questions instead of guessing. + + You never make assumptions about code that you failed to read or find, instead you stop + and ask the user for clarification. diff --git a/pyproject.toml b/pyproject.toml index 82aea490..e7f1cdbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "pathspec>=0.12.1", "psutil>=7.0.0", "agno>=1.2.15", + "docstring_parser>=0.16", ] [project.scripts] @@ -40,7 +41,7 @@ serena-mcp-server = "serena.mcp:start_mcp_server" serena-list-tools = "serena.agent:print_tool_overview" [project.license] -text = "GPL-2.0" +text = "MIT" [project.optional-dependencies] dev = [ @@ -70,6 +71,9 @@ google = [ [project.urls] Homepage = "https://github.com/oraios/serena" +[tool.hatch.build.targets.wheel] +packages = ["src/serena", "src/multilspy"] + [tool.black] line-length = 140 target-version = [ @@ -103,7 +107,7 @@ exclude = "^build/|^docs/" PYDEVD_DISABLE_FILE_VALIDATION = "1" [tool.poe.tasks] -test = "pytest test --cov=serena --cov-report=xml --cov-report=term-missing --durations=0 -v --color=yes" +test = "pytest test --color=yes" _black_check = "black --check --exclude src/multilspy/ src scripts test" _ruff_check = "ruff check --exclude .venv/ --exclude src/multilspy/ src scripts test" _black_format = "black --exclude .venv/|src/multilspy/ src scripts test" @@ -268,3 +272,12 @@ max-complexity = 20 "scripts/**" = [ "D103" ] + +[tool.pytest.ini_options] +markers = [ + "python: language server running for Python", + "go: language server running for Go", + "java: language server running for Java", + "rust: language server running for Rust", + "typescript: language server running for TypeScript", +] \ No newline at end of file diff --git a/scripts/demo_run_tools.py b/scripts/demo_run_tools.py new file mode 100644 index 00000000..4d639b48 --- /dev/null +++ b/scripts/demo_run_tools.py @@ -0,0 +1,26 @@ +""" +This script demonstrates how to use Serena's tools locally, useful +for testing or development. Here the config is pointing to the serena repo itself, +it assumes that you already have a serena_config.yml file setup. +""" + +import json +from pathlib import Path +from pprint import pprint + +from serena.agent import * + +project_root = Path(__file__).parent.parent + +if __name__ == "__main__": + agent = SerenaAgent(project_config=str(project_root / ".serena" / "project.yml")) + overview_tool = agent.get_tool(GetSymbolsOverviewTool) + find_symbol_tool = agent.get_tool(FindSymbolTool) + + print("Getting an overview of the util package\n") + pprint(json.loads(overview_tool.apply("src/serena/util"))) + + print("\n\n") + + print("Finding the symbol 'SerenaAgent'\n") + pprint(json.loads(find_symbol_tool.apply("SerenaAgent"))) diff --git a/scripts/mcp_server.py b/scripts/mcp_server.py index 299dd287..433d9b4e 100644 --- a/scripts/mcp_server.py +++ b/scripts/mcp_server.py @@ -1,6 +1,4 @@ -from serena.mcp import create_mcp_server - -mcp = create_mcp_server() +from serena.mcp import start_mcp_server if __name__ == "__main__": - mcp.run() + start_mcp_server() diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index ba636de7..35a3d591 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -14,32 +14,27 @@ import os import pathlib import pickle import re -from site import abs_paths import threading from collections import defaultdict from contextlib import asynccontextmanager, contextmanager from copy import copy -from typing import Dict, List, Optional, Tuple, Union -from fnmatch import fnmatch from pathlib import Path, PurePath from typing import AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union, cast import pathspec -from serena.text_utils import LineType, MatchedConsecutiveLines, TextLine, search_files, search_text +from serena.text_utils import LineType, MatchedConsecutiveLines, TextLine, search_files from . import multilspy_types from .lsp_protocol_handler import lsp_types as LSPTypes from .lsp_protocol_handler.lsp_constants import LSPConstants from .lsp_protocol_handler.lsp_types import SymbolKind from .lsp_protocol_handler.server import ( + Error, LanguageServerHandler, ProcessLaunchInfo, ) from .multilspy_config import Language, MultilspyConfig from .multilspy_exceptions import MultilspyException -from .multilspy_utils import PathUtils, FileUtils, TextUtils -from pathlib import PurePath -from typing import AsyncIterator, Iterator, List, Dict, Optional, Union, Tuple from .multilspy_logger import MultilspyLogger from .multilspy_utils import FileUtils, PathUtils, TextUtils from .type_helpers import ensure_all_methods_implemented @@ -584,6 +579,16 @@ class LanguageServer: return ret + # Some LS cause problems with this, so the call is isolated from the rest to allow overriding in subclasses + async def _send_references_request(self, relative_file_path: str, line: int, column: int): + return await self.server.send.references( + { + "textDocument": {"uri": PathUtils.path_to_uri(os.path.join(self.repository_root_path, relative_file_path))}, + "position": {"line": line, "character": column}, + "context": {"includeDeclaration": False}, + } + ) + async def request_references( self, relative_file_path: str, line: int, column: int ) -> List[multilspy_types.Location]: @@ -598,7 +603,7 @@ class LanguageServer: :return: A list of locations where the symbol is referenced (excluding ignored directories) """ - + if not self.server_started: self.logger.log( "request_references called before Language Server started", @@ -608,21 +613,23 @@ class LanguageServer: with self.open_file(relative_file_path): - # sending request to the language server and waiting for response - response = await self.server.send.references( - { - "context": {"includeDeclaration": False}, - "textDocument": { - "uri": pathlib.Path(os.path.join(self.repository_root_path, relative_file_path)).as_uri() - }, - "position": {"line": line, "character": column}, - } - ) + try: + response = await self._send_references_request(relative_file_path, line=line, column=column) + except Exception as e: + # Catch LSP internal error (-32603) and raise a more informative exception + if isinstance(e, Error) and getattr(e, 'code', None) == -32603: + raise RuntimeError( + f"LSP internal error (-32603) when requesting references for {relative_file_path}:{line}:{column}. " + "This often occurs when requesting references for a symbol not referenced in the expected way. " + ) from e + raise + if response is None: + return [] ret: List[multilspy_types.Location] = [] - assert isinstance(response, list), f"Unexpected response from Language Server: {response}" + assert isinstance(response, list), f"Unexpected response from Language Server (expected list, got {type(response)}): {response}" for item in response: - assert isinstance(item, dict) + assert isinstance(item, dict), f"Unexpected response from Language Server (expected dict, got {type(item)}): {item}" assert LSPConstants.URI in item assert LSPConstants.RANGE in item @@ -825,8 +832,10 @@ class LanguageServer: def turn_item_into_symbol_with_children(item: GenericDocumentSymbol): item = cast(multilspy_types.UnifiedSymbolInformation, item) + absolute_path = os.path.join(self.repository_root_path, relative_file_path) + + # handle missing entries in location if "location" not in item: - absolute_path = os.path.join(self.repository_root_path, relative_file_path) uri = pathlib.Path(absolute_path).as_uri() assert "range" in item tree_location = multilspy_types.Location( @@ -836,8 +845,19 @@ class LanguageServer: relativePath=relative_file_path, ) item['location'] = tree_location + location = item["location"] + if "absolutePath" not in location: + location["absolutePath"] = absolute_path + if "relativePath" not in location: + location["relativePath"] = relative_file_path if include_body: item['body'] = self.retrieve_symbol_body(item) + # handle missing selectionRange + if "selectionRange" not in item: + if "range" in item: + item["selectionRange"] = item["range"] + else: + item["selectionRange"] = item["location"]["range"] item[LSPConstants.CHILDREN] = item.get(LSPConstants.CHILDREN, []) flat_all_symbol_list: List[multilspy_types.UnifiedSymbolInformation] = [] @@ -956,14 +976,16 @@ class LanguageServer: # TODO: Not sure if this is actually still needed given recent changes to relative path handling def fix_relative_path(nodes: List[multilspy_types.UnifiedSymbolInformation]): for node in nodes: - path = Path(node["location"]["relativePath"]) - if path.is_absolute(): - try: - path = path.relative_to(self.repository_root_path) - node["location"]["relativePath"] = str(path) - except: - pass - fix_relative_path(node["children"]) + if "location" in node and "relativePath" in node["location"]: + path = Path(node["location"]["relativePath"]) + if path.is_absolute(): + try: + path = path.relative_to(self.repository_root_path) + node["location"]["relativePath"] = str(path) + except Exception: + pass + if "children" in node: + fix_relative_path(node["children"]) fix_relative_path(root_nodes) @@ -1046,15 +1068,20 @@ class LanguageServer: Returns the list of tuples (name, kind, line, column) of all top-level symbols in the file. """ _, document_roots = await self.request_document_symbols(relative_file_path) - return [ - ( - root["name"], - root["kind"], - root["selectionRange"]["start"]["line"], # type: ignore - root["selectionRange"]["start"]["character"] # type: ignore - ) - for root in document_roots - ] + result = [] + for root in document_roots: + try: + result.append( + ( root["name"], + root["kind"], + root["selectionRange"]["start"]["line"], + root["selectionRange"]["start"]["character"],) + ) + except KeyError as e: + raise KeyError( + f"Could not process symbol of name {root.get('name', 'unknown')} in {relative_file_path=}" + ) from e + return result async def request_overview(self, within_relative_path: str) -> dict[str, list[tuple[str, multilspy_types.SymbolKind, int, int]]]: """ @@ -1120,7 +1147,7 @@ class LanguageServer: assert "relativePath" in symbol["location"] symbol_file = self.retrieve_full_file_content(symbol["location"]["relativePath"]) symbol_lines = symbol_file.split("\n") - symbol_body = "\n".join(symbol_lines[symbol_start_line:symbol_end_line]) + symbol_body = "\n".join(symbol_lines[symbol_start_line:symbol_end_line+1]) # remove leading indentation symbol_start_column = symbol["location"]["range"]["start"]["character"] @@ -1704,11 +1731,21 @@ class SyncLanguageServer: :return List[multilspy_types.Location]: A list of locations where the symbol is referenced """ - result = asyncio.run_coroutine_threadsafe( - self.language_server.request_references(file_path, line, column), self.loop - ).result(timeout=self.timeout) + try: + result = asyncio.run_coroutine_threadsafe( + self.language_server.request_references(file_path, line, column), self.loop + ).result(timeout=self.timeout) + except Exception as e: + from multilspy.lsp_protocol_handler.server import Error + if isinstance(e, Error) and getattr(e, 'code', None) == -32603: + raise RuntimeError( + f"LSP internal error (-32603) when requesting references for {file_path}:{line}:{column}. " + "This often occurs when requesting references for a symbol not referenced in the expected way. " + ) from e + raise return result + def request_references_with_content( self, relative_file_path: str, line: int, column: int, context_lines_before: int = 0, context_lines_after: int = 0 ) -> List[MatchedConsecutiveLines]: diff --git a/src/multilspy/language_servers/clangd_language_server/clangd_language_server.py b/src/multilspy/language_servers/clangd_language_server/clangd_language_server.py index 17253c4a..8565825e 100644 --- a/src/multilspy/language_servers/clangd_language_server/clangd_language_server.py +++ b/src/multilspy/language_servers/clangd_language_server/clangd_language_server.py @@ -64,13 +64,19 @@ class ClangdLanguageServer(LanguageServer): clangd_ls_dir = os.path.join(os.path.dirname(__file__), "static/clangd") clangd_executable_path = os.path.join(clangd_ls_dir, "clangd_19.1.2", "bin", dependency["binaryName"]) - if not os.path.exists(clangd_ls_dir): - os.makedirs(clangd_ls_dir) + if not os.path.exists(clangd_executable_path): + clangd_url = dependency["url"] + logger.log(f"Clangd executable not found at {clangd_executable_path}. Downloading from {clangd_url}", logging.INFO) + os.makedirs(clangd_ls_dir, exist_ok=True) if dependency["archiveType"] == "zip": FileUtils.download_and_extract_archive( - logger, dependency["url"], clangd_ls_dir, dependency["archiveType"] + logger, clangd_url, clangd_ls_dir, dependency["archiveType"] ) - assert os.path.exists(clangd_executable_path) + if not os.path.exists(clangd_executable_path): + raise FileNotFoundError( + f"Clangd executable not found at {clangd_executable_path}.\n" + "Make sure you have installed clangd. See https://clangd.llvm.org/installation" + ) os.chmod(clangd_executable_path, stat.S_IEXEC) return clangd_executable_path diff --git a/src/multilspy/language_servers/dart_language_server/runtime_dependencies.json b/src/multilspy/language_servers/dart_language_server/runtime_dependencies.json index 8010b940..32e49299 100644 --- a/src/multilspy/language_servers/dart_language_server/runtime_dependencies.json +++ b/src/multilspy/language_servers/dart_language_server/runtime_dependencies.json @@ -8,6 +8,38 @@ "platformId": "linux-x64", "archiveType": "zip", "binaryName": "dart-sdk/bin/dart" + }, + { + "id": "DartLanguageServer", + "description": "Dart Language Server for Windows (x64)", + "url": "https://storage.googleapis.com/dart-archive/channels/stable/release/3.7.1/sdk/dartsdk-windows-x64-release.zip", + "platformId": "win-x64", + "archiveType": "zip", + "binaryName": "dart-sdk/bin/dart.exe" + }, + { + "id": "DartLanguageServer", + "description": "Dart Language Server for Windows (arm64)", + "url": "https://storage.googleapis.com/dart-archive/channels/stable/release/3.7.1/sdk/dartsdk-windows-arm64-release.zip", + "platformId": "win-arm64", + "archiveType": "zip", + "binaryName": "dart-sdk/bin/dart.exe" + }, + { + "id": "DartLanguageServer", + "description": "Dart Language Server for macOS (x64)", + "url": "https://storage.googleapis.com/dart-archive/channels/stable/release/3.7.1/sdk/dartsdk-macos-x64-release.zip", + "platformId": "osx-x64", + "archiveType": "zip", + "binaryName": "dart-sdk/bin/dart" + }, + { + "id": "DartLanguageServer", + "description": "Dart Language Server for macOS (arm64)", + "url": "https://storage.googleapis.com/dart-archive/channels/stable/release/3.7.1/sdk/dartsdk-macos-arm64-release.zip", + "platformId": "osx-arm64", + "archiveType": "zip", + "binaryName": "dart-sdk/bin/dart" } ] } \ No newline at end of file diff --git a/src/multilspy/language_servers/gopls/gopls.py b/src/multilspy/language_servers/gopls/gopls.py index 6dbe88cd..6269db82 100644 --- a/src/multilspy/language_servers/gopls/gopls.py +++ b/src/multilspy/language_servers/gopls/gopls.py @@ -5,13 +5,15 @@ import os import pathlib import subprocess from contextlib import asynccontextmanager -from typing import AsyncIterator +from typing import AsyncIterator, List from overrides import override +from multilspy import multilspy_types +from multilspy.multilspy_exceptions import MultilspyException from multilspy.multilspy_logger import MultilspyLogger from multilspy.language_server import LanguageServer -from multilspy.lsp_protocol_handler.server import ProcessLaunchInfo +from multilspy.lsp_protocol_handler.server import Error, ProcessLaunchInfo from multilspy.lsp_protocol_handler.lsp_types import InitializeParams from multilspy.multilspy_config import MultilspyConfig @@ -57,28 +59,21 @@ class Gopls(LanguageServer): Check if required Go runtime dependencies are available. Raises RuntimeError with helpful message if dependencies are missing. """ - missing_deps = [] - - # Check for Go installation go_version = cls._get_go_version() if not go_version: - missing_deps.append(("Go", "https://golang.org/doc/install")) + raise RuntimeError("Go is not installed. Please install Go from https://golang.org/doc/install and make sure it is added to your PATH.") - # Check for gopls gopls_version = cls._get_gopls_version() if not gopls_version: - missing_deps.append(("gopls", "https://pkg.go.dev/golang.org/x/tools/gopls#section-readme")) - - if missing_deps: - error_msg = "Missing required dependencies:\n" - for dep, install_url in missing_deps: - error_msg += f"- {dep}: Please install from {install_url}\n" - raise RuntimeError(error_msg) + raise RuntimeError( + "Found a Go version but gopls is not installed.\n" + "Please install gopls as described in https://pkg.go.dev/golang.org/x/tools/gopls#section-readme\n\n" + "After installation, make sure it is added to your PATH (it might be installed in a different location than Go)." + ) return True def __init__(self, config: MultilspyConfig, logger: MultilspyLogger, repository_root_path: str): - # Check runtime dependencies before initializing self.setup_runtime_dependency() super().__init__( @@ -159,3 +154,18 @@ class Gopls(LanguageServer): await self.server.shutdown() await self.server.stop() + + @override + async def _send_references_request(self, relative_file_path: str, line: int, column: int) -> List[multilspy_types.Location]: + NUM_COLS_TO_TRY = 10 + """selectionRange in Gopls always contains the wrong column, + the one at the beginning of the declaration instead of the symbol. We do a dirty hack here (loop over some columns). + See https://github.com/golang/go/issues/73521 + """ + actual_column = column + for actual_column in range(column, column+NUM_COLS_TO_TRY): + try: + return await super()._send_references_request(relative_file_path, line, actual_column) + except (Error, RuntimeError, MultilspyException) as e: + self.logger.log(f"Cannot find symbol at {line=}, column={actual_column}, trying to bump column by 1", logging.INFO) + raise MultilspyException(f"Failed to find references for symbol in\n{relative_file_path=} at {line=}, {column=}") diff --git a/src/multilspy/language_servers/typescript_language_server/typescript_language_server.py b/src/multilspy/language_servers/typescript_language_server/typescript_language_server.py index 14426c6f..6657ebba 100644 --- a/src/multilspy/language_servers/typescript_language_server/typescript_language_server.py +++ b/src/multilspy/language_servers/typescript_language_server/typescript_language_server.py @@ -4,22 +4,23 @@ Provides TypeScript specific instantiation of the LanguageServer class. Contains import asyncio import json -import shutil import logging import os -import subprocess import pathlib +import shutil +import subprocess +from collections.abc import AsyncIterator from contextlib import asynccontextmanager -from typing import AsyncIterator, Tuple +from time import sleep from overrides import override -from multilspy.multilspy_logger import MultilspyLogger from multilspy.language_server import LanguageServer -from multilspy.lsp_protocol_handler.server import ProcessLaunchInfo from multilspy.lsp_protocol_handler.lsp_types import InitializeParams +from multilspy.lsp_protocol_handler.server import ProcessLaunchInfo from multilspy.multilspy_config import MultilspyConfig -from multilspy.multilspy_utils import PlatformUtils, PlatformId +from multilspy.multilspy_logger import MultilspyLogger +from multilspy.multilspy_utils import PlatformId, PlatformUtils # Platform-specific imports if os.name != 'nt': # Unix-like systems @@ -82,7 +83,7 @@ class TypeScriptLanguageServer(LanguageServer): ] assert platform_id in valid_platforms, f"Platform {platform_id} is not supported for multilspy javascript/typescript at the moment" - with open(os.path.join(os.path.dirname(__file__), "runtime_dependencies.json"), "r") as f: + with open(os.path.join(os.path.dirname(__file__), "runtime_dependencies.json")) as f: d = json.load(f) del d["_description"] @@ -132,7 +133,7 @@ class TypeScriptLanguageServer(LanguageServer): """ Returns the initialize params for the TypeScript Language Server. """ - with open(os.path.join(os.path.dirname(__file__), "initialize_params.json"), "r") as f: + with open(os.path.join(os.path.dirname(__file__), "initialize_params.json")) as f: d = json.load(f) del d["_description"] @@ -222,3 +223,14 @@ class TypeScriptLanguageServer(LanguageServer): await self.server.shutdown() await self.server.stop() + + @override + # For some reason, the LS may need longer to process this, so we just retry + async def _send_references_request(self, relative_file_path: str, line: int, column: int): + # TODO: The LS doesn't return references contained in other files if it doesn't sleep. This is + # despite the LS having processed requests already. I don't know what causes this, but sleeping + # one second helps. It may be that sleeping only once is enough but that's hard to reliably test. + # It may be that even this 1sec is not enough in larger TS projects, at some point we should find what + # causes this and solve it. + sleep(1) + return await super()._send_references_request(relative_file_path, line, column) diff --git a/src/multilspy/lsp_protocol_handler/server.py b/src/multilspy/lsp_protocol_handler/server.py index b9dd0de8..81a9dc4a 100644 --- a/src/multilspy/lsp_protocol_handler/server.py +++ b/src/multilspy/lsp_protocol_handler/server.py @@ -38,6 +38,7 @@ from typing import Any, Dict, List, Optional, Union from .lsp_requests import LspNotification, LspRequest from .lsp_types import ErrorCodes +from ..multilspy_exceptions import MultilspyException StringDict = Dict[str, Any] PayloadLike = Union[List[StringDict], StringDict, None] @@ -481,7 +482,7 @@ class LanguageServerHandler: await self._send_payload(make_request(method, request_id, params)) await request.cv.wait() if isinstance(request.error, Error): - raise request.error + raise MultilspyException(f"Could not process request {method} with params:\n{params}.\n Language server error: {request.error}") from request.error return request.result def _send_payload_sync(self, payload: StringDict) -> None: diff --git a/src/multilspy/multilspy_utils.py b/src/multilspy/multilspy_utils.py index bdda6977..75f023a8 100644 --- a/src/multilspy/multilspy_utils.py +++ b/src/multilspy/multilspy_utils.py @@ -17,6 +17,8 @@ from enum import Enum from multilspy.multilspy_exceptions import MultilspyException from pathlib import PurePath, Path from multilspy.multilspy_logger import MultilspyLogger +from multilspy.multilspy_types import UnifiedSymbolInformation + class TextUtils: """ @@ -67,6 +69,7 @@ class TextUtils: c += len(text_to_be_inserted) return (l, c) + class PathUtils: """ Utilities for platform-agnostic path operations. @@ -88,7 +91,14 @@ class PathUtils: parsed = urlparse(uri) host = "{0}{0}{mnt}{0}".format(os.path.sep, mnt=parsed.netloc) return os.path.normpath(os.path.join(host, url2pathname(unquote(parsed.path)))) - + + @staticmethod + def path_to_uri(path: str) -> str: + """ + Converts a file path to a file URI (file:///...). + """ + return str(Path(path).absolute().as_uri()) + @staticmethod def is_glob_pattern(pattern: str) -> bool: """Check if a pattern contains glob-specific characters.""" @@ -104,6 +114,7 @@ class PathUtils: return str(PurePath(os.path.relpath(path, base_path))) return None + class FileUtils: """ Utility functions for file operations. @@ -114,7 +125,10 @@ class FileUtils: """ Reads the file at the given path and returns the contents as a string. """ - encodings = ["utf-8-sig", "utf-16"] + if not os.path.exists(file_path): + logger.log(f"File read '{file_path}' failed: File does not exist.", logging.ERROR) + raise MultilspyException(f"File read '{file_path}' failed: File does not exist.") + encodings = ["utf-8-sig", "utf-16", "utf-8", "latin-1"] try: for encoding in encodings: try: @@ -122,6 +136,9 @@ class FileUtils: return inp_file.read() except UnicodeError: continue + # Try system default encoding as a last resort + with open(file_path, "r") as inp_file: + return inp_file.read() except Exception as exc: logger.log(f"File read '{file_path}' failed: {exc}", logging.ERROR) raise MultilspyException("File read failed.") from None @@ -179,6 +196,7 @@ class FileUtils: if os.path.exists(tmp_file_name): Path.unlink(Path(tmp_file_name)) + class PlatformId(str, Enum): """ multilspy supported platforms @@ -195,6 +213,7 @@ class PlatformId(str, Enum): LINUX_MUSL_x64 = "linux-musl-x64" LINUX_MUSL_arm64 = "linux-musl-arm64" + class DotnetVersion(str, Enum): """ multilspy supported dotnet versions @@ -205,19 +224,22 @@ class DotnetVersion(str, Enum): V8 = "8" VMONO = "mono" + class PlatformUtils: """ This class provides utilities for platform detection and identification. """ - @staticmethod - def get_platform_id() -> PlatformId: + @classmethod + def get_platform_id(cls) -> PlatformId: """ Returns the platform id for the current system """ system = platform.system() machine = platform.machine() bitness = platform.architecture()[0] + if system == "Windows" and machine == "": + machine = cls._determine_windows_machine_type() system_map = {"Windows": "win", "Darwin": "osx", "Linux": "linux"} machine_map = {"AMD64": "x64", "x86_64": "x64", "i386": "x86", "i686": "x86", "aarch64": "arm64", "arm64": "arm64"} if system in system_map and machine in machine_map: @@ -230,6 +252,46 @@ class PlatformUtils: else: raise MultilspyException(f"Unknown platform: {system=}, {machine=}, {bitness=}") + @staticmethod + def _determine_windows_machine_type(): + import ctypes + from ctypes import wintypes + + class SYSTEM_INFO(ctypes.Structure): + class _U(ctypes.Union): + class _S(ctypes.Structure): + _fields_ = [("wProcessorArchitecture", wintypes.WORD), + ("wReserved", wintypes.WORD)] + _fields_ = [("dwOemId", wintypes.DWORD), + ("s", _S)] + _anonymous_ = ("s",) + + _fields_ = [("u", _U), + ("dwPageSize", wintypes.DWORD), + ("lpMinimumApplicationAddress", wintypes.LPVOID), + ("lpMaximumApplicationAddress", wintypes.LPVOID), + ("dwActiveProcessorMask", wintypes.LPVOID), + ("dwNumberOfProcessors", wintypes.DWORD), + ("dwProcessorType", wintypes.DWORD), + ("dwAllocationGranularity", wintypes.DWORD), + ("wProcessorLevel", wintypes.WORD), + ("wProcessorRevision", wintypes.WORD)] + _anonymous_ = ("u",) + + sys_info = SYSTEM_INFO() + ctypes.windll.kernel32.GetNativeSystemInfo(ctypes.byref(sys_info)) + + arch_map = { + 9: 'AMD64', + 5: 'ARM', + 12: 'arm64', + 6: 'Intel Itanium-based', + 0: 'i386', + } + + return arch_map.get(sys_info.wProcessorArchitecture, f'Unknown ({sys_info.wProcessorArchitecture})') + + @staticmethod def get_dotnet_version() -> DotnetVersion: """ @@ -261,3 +323,13 @@ class PlatformUtils: except (FileNotFoundError, subprocess.CalledProcessError): raise MultilspyException("dotnet or mono not found on the system") + +class SymbolUtils: + @staticmethod + def symbol_tree_contains_name(roots: list[UnifiedSymbolInformation], name: str) -> bool: + for symbol in roots: + if symbol["name"] == name: + return True + if SymbolUtils.symbol_tree_contains_name(symbol["children"], name): + return True + return False diff --git a/src/serena/agent.py b/src/serena/agent.py index addf3270..ec1e2d2b 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -11,6 +11,7 @@ import traceback from abc import ABC from collections import defaultdict from collections.abc import Callable, Generator, Iterable +from dataclasses import dataclass, field from logging import Logger from pathlib import Path from typing import TYPE_CHECKING, Any, Self, TypeVar, Union, cast @@ -28,7 +29,6 @@ from serena import serena_root_path, serena_version from serena.llm.prompt_factory import PromptFactory from serena.symbol import SymbolLocation, SymbolManager from serena.text_utils import search_files -from serena.util.class_decorators import singleton from serena.util.file_system import scan_directory from serena.util.inspection import iter_subclasses from serena.util.shell import execute_shell_command @@ -67,18 +67,40 @@ class SerenaConfigError(Exception): pass +@dataclass class ProjectConfig(ToStringMixin): + project_name: str + language: Language + project_root: str + ignored_paths: list[str] = field(default_factory=list) + excluded_tools: set[str] = field(default_factory=set) + read_only: bool = False + ignore_all_files_in_gitignore: bool = True + SERENA_MANAGED_DIR = ".serena" SERENA_DEFAULT_PROJECT_FILE = "project.yml" - def __init__(self, config_dict: dict[str, Any], project_name: str, project_root: Path | None = None): - self.project_name: str = project_name - self.language: Language = Language(config_dict["language"]) + @classmethod + def rel_path_to_project_yml(cls): + return os.path.join(cls.SERENA_MANAGED_DIR, cls.SERENA_DEFAULT_PROJECT_FILE) + + @classmethod + def from_config_dict(cls, config_dict: dict[str, Any], project_name: str, project_root: Path | None = None) -> Self: + """ + Create a ProjectConfig instance from a configuration dictionary + """ + try: + language = Language(config_dict["language"].lower()) + except ValueError as e: + raise ValueError(f"Invalid language: {config_dict['language']}.\nValid languages are: {[l.value for l in Language]}") from e + if project_root is None: project_root = Path(config_dict["project_root"]) - self.project_root: str = str(project_root.resolve()) - self.ignored_paths: list[str] = config_dict.get("ignored_paths", []) - self.excluded_tools: set[str] = set(config_dict.get("excluded_tools", [])) + project_root_str = str(project_root.resolve()) + + ignored_paths = config_dict.get("ignored_paths", []) + excluded_tools = set(config_dict.get("excluded_tools", [])) + read_only = config_dict.get("read_only", False) if "ignore_all_files_in_gitignore" not in config_dict: raise SerenaConfigError( @@ -86,7 +108,7 @@ class ProjectConfig(ToStringMixin): "Please update your `.yml` configuration file for this project. " "It is recommended to set this to `True`." ) - self.ignore_all_files_in_gitignore = config_dict["ignore_all_files_in_gitignore"] + ignore_all_files_in_gitignore = config_dict["ignore_all_files_in_gitignore"] # Raise errors for deprecated keys if "ignored_dirs" in config_dict: @@ -95,9 +117,24 @@ class ProjectConfig(ToStringMixin): "Note that you can also set `ignore_all_files_in_gitignore` to `True`, which will be enough for most cases." ) + return cls( + project_name=project_name, + language=language, + project_root=project_root_str, + ignored_paths=ignored_paths, + excluded_tools=excluded_tools, + read_only=read_only, + ignore_all_files_in_gitignore=ignore_all_files_in_gitignore, + ) + @classmethod - def from_yml(cls, yml_path: Path) -> Self: + def _from_yml(cls, yml_path: Path) -> Self: + """ + Create a ProjectConfig instance from a YAML file + """ log.info(f"Loading project configuration from {yml_path}") + if not yml_path.exists(): + raise FileNotFoundError(f"Project file not found: {yml_path}") try: with open(yml_path, encoding="utf-8") as f: config_dict = yaml.safe_load(f) @@ -107,26 +144,50 @@ class ProjectConfig(ToStringMixin): else: project_root = None project_name = yml_path.stem - return cls(config_dict, project_name=project_name, project_root=project_root) + return cls.from_config_dict(config_dict, project_name=project_name, project_root=project_root) except Exception as e: raise ValueError(f"Error loading project configuration from {yml_path}: {e}") from e + @classmethod + def from_path(cls, path: Path | str) -> Self: + """ + Create a ProjectConfig instance from a path. Can either be a path to a yaml file or a directory + containing `.serena/project.yml`. + """ + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"Path not found: {path}") + if path.is_dir(): + path = path / cls.SERENA_MANAGED_DIR / cls.SERENA_DEFAULT_PROJECT_FILE + return cls._from_yml(path) + def get_serena_managed_dir(self) -> str: return os.path.join(self.project_root, self.SERENA_MANAGED_DIR) -@singleton +@dataclass class SerenaConfig: """ Handles user-defined Serena configuration based on the configuration file """ + projects: dict[str, ProjectConfig] = field(default_factory=dict) + project_names: list[str] = field(default_factory=list) + gui_log_window_enabled: bool = False + gui_log_window_level: int = logging.INFO + enable_project_activation: bool = True + CONFIG_FILE = "serena_config.yml" - def __init__(self) -> None: - config_file = os.path.join(serena_root_path(), self.CONFIG_FILE) + @classmethod + def from_config_file(cls) -> "SerenaConfig": + """ + Static constructor to create SerenaConfig from the configuration file + """ + config_file = os.path.join(serena_root_path(), cls.CONFIG_FILE) if not os.path.exists(config_file): raise FileNotFoundError(f"Serena configuration file not found: {config_file}") + with open(config_file, encoding="utf-8") as f: try: log.info(f"Loading Serena configuration from {config_file}") @@ -134,10 +195,13 @@ class SerenaConfig: except Exception as e: raise ValueError(f"Error loading Serena configuration from {config_file}: {e}") from e + # Create instance + instance = cls() + # read projects - self.projects: dict[str, ProjectConfig] = {} if "projects" not in config_yaml: raise SerenaConfigError("`projects` key not found in Serena configuration. Please update your `serena_config.yml` file.") + projects = {} for project_config_path in config_yaml["projects"]: project_config_path = Path(project_config_path) if not project_config_path.is_absolute(): @@ -147,15 +211,18 @@ class SerenaConfig: if not project_config_path.is_file(): raise FileNotFoundError(f"Project file not found: {project_config_path}") log.info(f"Loading project configuration from {project_config_path}") - project_config = ProjectConfig.from_yml(project_config_path) - self.projects[project_config.project_name] = project_config - self.project_names = list(self.projects.keys()) + project_config = ProjectConfig.from_path(project_config_path) + projects[project_config.project_name] = project_config - self.gui_log_window_enabled = config_yaml.get("gui_log_window", False) - self.gui_log_window_level = config_yaml.get("gui_log_level", logging.INFO) - self.enable_project_activation = config_yaml.get("enable_project_activation", True) + instance.projects = projects + instance.project_names = list(projects.keys()) + instance.gui_log_window_enabled = config_yaml.get("gui_log_window", False) + instance.gui_log_window_level = config_yaml.get("gui_log_level", logging.INFO) + instance.enable_project_activation = config_yaml.get("enable_project_activation", True) - def get_project_configuration(self, project_name: str) -> ProjectConfig: + return instance + + def get_project_configuration(self, project_name: str) -> "ProjectConfig": if project_name not in self.projects: raise ValueError(f"Project '{project_name}' not found in Serena configuration; valid project names: {self.project_names}") return self.projects[project_name] @@ -178,15 +245,22 @@ class LinesRead: class SerenaAgent: - def __init__(self, project_file_path: str | None = None, project_activation_callback: Callable[[], None] | None = None): + def __init__( + self, + project_config: ProjectConfig | str | Path | None = None, + project_activation_callback: Callable[[], None] | None = None, + serena_config: SerenaConfig | None = None, + ) -> None: """ - :param project_file_path: the configuration file (.yml) of the project to load immediately; + :param project_config: the project to load immediately; may be a path to the project configuration file + or a configuration instance; if None, do not load any project (must use project selection tool to activate a project). If a project is provided, the corresponding language server will be started. :param project_activation_callback: a callback function to be called when a project is activated. + :param serena_config: the Serena configuration or None to read the configuration from the default location. """ # obtain serena configuration - self.serena_config = SerenaConfig() + self.serena_config = serena_config or SerenaConfig.from_config_file() # open GUI log window if enabled self._gui_log_handler: Union["GuiLogViewerHandler", None] = None # noqa @@ -236,20 +310,20 @@ class SerenaAgent: self._gui_log_handler.log_viewer.set_tool_names(tool_names) # activate a project configuration (if provided or if there is only a single project available) - project_config: ProjectConfig | None = None - if project_file_path is not None: - if not os.path.exists(project_file_path): - raise FileNotFoundError(f"Project file not found: {project_file_path}") - log.info(f"Loading project configuration from {project_file_path}") - project_config = ProjectConfig.from_yml(Path(project_file_path)) + project_config_to_load: ProjectConfig | None = None + if project_config is not None: + if isinstance(project_config, (str, Path)): + project_config_to_load = ProjectConfig.from_path(project_config) + else: + project_config_to_load = project_config else: match len(self.serena_config.projects): case 0: raise RuntimeError(f"No projects found in {SerenaConfig.CONFIG_FILE} and no project file specified.") case 1: - project_config = self.serena_config.get_project_configuration(self.serena_config.project_names[0]) - if project_config is not None: - self.activate_project(project_config) + project_config_to_load = self.serena_config.get_project_configuration(self.serena_config.project_names[0]) + if project_config_to_load is not None: + self.activate_project(project_config_to_load) else: if not self.serena_config.enable_project_activation: raise ValueError("Tool-based project activation is disabled in the configuration but no project file was provided.") @@ -260,9 +334,10 @@ class SerenaAgent: """ if self.serena_config.enable_project_activation: # With project activation, we must expose all tools and handle tool activation within Serena - # (because clients to not react to changed tools) + # (because clients do not react to changed tools) return list(self._all_tools.values()) else: + # When project activation is not enabled, we only expose the active tools return list(self._active_tools.values()) def activate_project(self, project_config: ProjectConfig) -> None: @@ -278,6 +353,13 @@ class SerenaAgent: else: self._active_tools = dict(self._all_tools) + # if read_only mode is enabled, exclude all editing tools + if self.project_config.read_only: + self._active_tools = {key: tool for key, tool in self._active_tools.items() if not key.can_edit()} + log.info( + f"Project is in read-only mode. Editing tools excluded. Active tools ({len(self._active_tools)}): {', '.join(self.get_active_tool_names())}" + ) + # start the language server self.reset_language_server() assert self.language_server is not None @@ -423,6 +505,16 @@ class Component(ABC): _DEFAULT_MAX_ANSWER_LENGTH = int(2e5) +class ToolMarkerCanEdit: + """ + Marker class for all tools that can perform editing operations on files. + """ + + +class ToolMarkerDoesNotRequireActiveProject: + pass + + class Tool(Component): # NOTE: each tool should implement the apply method, which is then used in # the central method of the Tool class `apply_ex`. @@ -449,6 +541,15 @@ class Tool(Component): raise RuntimeError(f"apply not defined in {self}. Did you forget to implement it?") return apply_fn + @classmethod + def can_edit(cls) -> bool: + """ + Returns whether this tool can perform editing operations on code. + + :return: True if the tool can edit code, False otherwise + """ + return issubclass(cls, ToolMarkerCanEdit) + @classmethod def get_tool_description(cls) -> str: docstring = cls.__doc__ @@ -512,6 +613,13 @@ class Tool(Component): f"active tools: {self.agent.get_active_tool_names()}" ) + # check if the project is in read-only mode and this is an editing tool + if self.agent.project_config is not None and self.agent.project_config.read_only and self.__class__.can_edit(): + return ( + f"Error: Tool '{self.get_name()}' cannot be used because the project '{self.project_config.project_name}' " + f"is in read-only mode. Editing operations are not allowed." + ) + # apply the actual tool result = apply_fn(**kwargs) @@ -528,10 +636,6 @@ class Tool(Component): return result -class ToolMarkerDoesNotRequireActiveProject: - pass - - class RestartLanguageServerTool(Tool): """Restarts the language server, may be necessary when edits not through Serena happen.""" @@ -579,7 +683,7 @@ class ReadFileTool(Tool): return self._limit_length(result, max_answer_chars) -class CreateTextFileTool(Tool): +class CreateTextFileTool(Tool, ToolMarkerCanEdit): """ Creates/overwrites a file in the project directory. """ @@ -613,6 +717,8 @@ class ListDirTool(Tool): def apply(self, relative_path: str, recursive: bool, max_answer_chars: int = _DEFAULT_MAX_ANSWER_LENGTH) -> str: """ + Lists files and directories in the given directory (optionally with recursion). + :param relative_path: the relative path to the directory to list; pass "." to scan the project root :param recursive: whether to scan subdirectories recursively :param max_answer_chars: if the output is longer than this number of characters, @@ -831,7 +937,7 @@ class FindReferencingCodeSnippetsTool(Tool): return self._limit_length(result_json_str, max_answer_chars) -class ReplaceSymbolBodyTool(Tool): +class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): """ Replaces the full definition of a symbol. """ @@ -860,7 +966,7 @@ class ReplaceSymbolBodyTool(Tool): return SUCCESS_RESULT -class InsertAfterSymbolTool(Tool): +class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit): """ Inserts content after the end of the definition of a given symbol. """ @@ -889,7 +995,7 @@ class InsertAfterSymbolTool(Tool): return SUCCESS_RESULT -class InsertBeforeSymbolTool(Tool): +class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit): """ Inserts content before the beginning of the definition of a given symbol. """ @@ -918,7 +1024,7 @@ class InsertBeforeSymbolTool(Tool): return SUCCESS_RESULT -class DeleteLinesTool(Tool): +class DeleteLinesTool(Tool, ToolMarkerCanEdit): """ Deletes a range of lines within a file. """ @@ -945,7 +1051,7 @@ class DeleteLinesTool(Tool): return SUCCESS_RESULT -class ReplaceLinesTool(Tool): +class ReplaceLinesTool(Tool, ToolMarkerCanEdit): """ Replaces a range of lines within a file with new content. """ @@ -976,7 +1082,7 @@ class ReplaceLinesTool(Tool): return SUCCESS_RESULT -class InsertAtLineTool(Tool): +class InsertAtLineTool(Tool, ToolMarkerCanEdit): """ Inserts content at a given line in a file. """ @@ -1011,7 +1117,10 @@ class CheckOnboardingPerformedTool(Tool): def apply(self) -> str: """ Checks whether project onboarding was already performed. - You should always call this tool before beginning to actually work on a project/after activating a project. + You should always call this tool before beginning to actually work on the project/after activating a project, + but after calling the initial instructions tool. + If onboarding was already performed, you will receive a list of available memories. + Don't read the memories immediately after if not needed, just remember that they exist and that you can read them later. """ list_memories_tool = self.agent.get_tool(ListMemoriesTool) memories = json.loads(list_memories_tool.apply()) @@ -1058,7 +1167,7 @@ class WriteMemoryTool(Tool): """ if len(content) > max_answer_chars: raise ValueError( - f"Content for {memory_file_name } is too long. Max length is {max_answer_chars} characters. " + f"Content for {memory_file_name} is too long. Max length is {max_answer_chars} characters. " + "Please make the content shorter." ) @@ -1143,9 +1252,7 @@ class ThinkAboutWhetherYouAreDoneTool(Tool): def apply(self) -> str: """ - Think about whether you are done with the task. - - This tool should ALWAYS be called after you have completed a task or a subtask. + Whenever you feel that you are done with what the user has asked for, it is important to call this tool. """ return self.prompt_factory.create_think_about_whether_you_are_done() @@ -1158,8 +1265,8 @@ class SummarizeChangesTool(Tool): def apply(self) -> str: """ Summarize the changes you have made to the codebase. - This tool should ALWAYS be called after you have fully completed any non-trivial coding task - (but after the think_about_whether_you_are_done call). + This tool should always be called after you have fully completed any non-trivial coding task, + but only after the think_about_whether_you_are_done call. """ return self.prompt_factory.create_summarize_changes() @@ -1247,7 +1354,7 @@ class SearchForPatternTool(Tool): return self._limit_length(result, max_answer_chars) -class ExecuteShellCommandTool(Tool): +class ExecuteShellCommandTool(Tool, ToolMarkerCanEdit): """ Executes a shell command. """ @@ -1321,8 +1428,32 @@ class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): return SUCCESS_RESULT -def iter_tool_classes() -> Generator[type[Tool], None, None]: - return iter_subclasses(Tool) +class InitialInstructionsTool(Tool): + """ + Gets the initial instructions for the current project. + Should only be used in settings where the system prompt cannot be set, + e.g. in clients you have no control over, like Claude Desktop. + """ + + def apply(self) -> str: + """ + Get the initial instructions for the current coding project. + You should always call this tool before starting to work (including using any other tool) on any programming task! + """ + return self.agent.prompt_factory.create_system_prompt() + + +def iter_tool_classes(same_module_only: bool = True) -> Generator[type[Tool], None, None]: + """ + Iterate over Tool subclasses. + + :param same_module_only: Whether to only iterate over tools defined in the same module as the Tool class + or over all subclasses of Tool. + """ + for tool_class in iter_subclasses(Tool): + if same_module_only and tool_class.__module__ != Tool.__module__: + continue + yield tool_class def print_tool_overview() -> None: diff --git a/src/serena/mcp.py b/src/serena/mcp.py index dc31e640..f52e5292 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -7,9 +7,10 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager from dataclasses import dataclass from logging import Formatter, Logger, StreamHandler -from typing import Literal +from typing import Any, Literal import click # Add click import +import docstring_parser from mcp.server.fastmcp import server from mcp.server.fastmcp.server import FastMCP, Settings from mcp.server.fastmcp.tools.base import Tool as MCPTool @@ -56,6 +57,28 @@ def make_tool( func_arg_metadata = func_metadata(apply_fn) parameters = func_arg_metadata.arg_model.model_json_schema() + docstring = docstring_parser.parse(func_doc) + + # Mount the tool description as a combination of the docstring description and + # the return value description, if it exists. + if docstring.description: + func_doc = f"{docstring.description.strip().strip('.')}." + else: + func_doc = "" + if (docstring.returns) and (docstring_returns := docstring.returns.description): + # Only add a space before "Returns" if func_doc is not empty + prefix = " " if func_doc else "" + func_doc = f"{func_doc}{prefix}Returns {docstring_returns.strip().strip('.')}." + + # Parse the parameter descriptions from the docstring and add pass its description + # to the parameters schema. + docstring_params = {param.arg_name: param for param in docstring.params} + parameters_properties: dict[str, dict[str, Any]] = parameters["properties"] + for parameter, properties in parameters_properties.items(): + if (param_doc := docstring_params.get(parameter)) and (param_doc.description): + param_desc = f"{param_doc.description.strip().strip('.') + '.'}" + properties["description"] = param_desc[0].upper() + param_desc[1:] + def execute_fn(**kwargs) -> str: # type: ignore return tool.apply_ex(log_call=True, catch_exceptions=True, **kwargs) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index 82280247..0fcc553f 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -297,7 +297,7 @@ class SymbolManager: yield root_path = self.lang_server.language_server.repository_root_path abs_path = os.path.join(root_path, relative_path) - with open(abs_path, "w") as f: + with open(abs_path, "w", encoding="utf-8") as f: f.write(file_buffer.contents) self.agent.mark_file_modified(relative_path) @@ -317,6 +317,9 @@ class SymbolManager: :param location: the location of the symbol to replace :param body: the new body """ + # make sure body always ends with at least one newline + if not body.endswith("\n"): + body += "\n" with self._edited_symbol_location(location) as symbol: assert location.relative_path is not None self.lang_server.delete_text_between_positions(location.relative_path, symbol.body_start_position, symbol.body_end_position) @@ -331,6 +334,9 @@ class SymbolManager: :param location: the location of the symbol after which to add new lines :param body: the body of the entity to append """ + # make sure body always ends with at least one newline + if not body.endswith("\n"): + body += "\n" with self._edited_symbol_location(location) as symbol: pos = symbol.body_end_position assert location.relative_path is not None @@ -343,6 +349,9 @@ class SymbolManager: :param location: the location of the symbol before which to add new lines :param body: the body of the entity to insert """ + # make sure body always ends with at least one newline + if not body.endswith("\n"): + body += "\n" with self._edited_symbol_location(location) as symbol: pos = copy(symbol.body_start_position) assert location.relative_path is not None diff --git a/test/resources/test_repo/test_repo/__init__.py b/test/__init__.py similarity index 100% rename from test/resources/test_repo/test_repo/__init__.py rename to test/__init__.py diff --git a/test/conftest.py b/test/conftest.py index 59cb70e0..4e403dfe 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -14,25 +14,72 @@ def resources_dir() -> Path: return current_dir / "resources" -@pytest.fixture(scope="session") -def repo_path() -> Path: - return Path(__file__).parent / "resources" / "test_repo" +class LanguageParamRequest: + param: Language -@pytest.fixture(scope="session") -def language_server(repo_path: Path): - """Create a SyncLanguageServer instance configured to use the test repository.""" - config = MultilspyConfig(code_language=Language.PYTHON) +def get_repo_path(language: Language) -> Path: + return Path(__file__).parent / "resources" / "repos" / language / "test_repo" + + +def create_default_ls(language: Language) -> SyncLanguageServer: + config = MultilspyConfig(code_language=language) + repo_path = str(get_repo_path(language)) logger = MultilspyLogger() + return SyncLanguageServer.create(config, logger, repo_path) - # Create a language server instance - server = SyncLanguageServer.create(config, logger, str(repo_path)) - # Start the server +@pytest.fixture(scope="session") +def repo_path(request: LanguageParamRequest) -> Path: + """Get the repository path for a specific language. + + This fixture requires a language parameter via pytest.mark.parametrize: + + Example: + ``` + @pytest.mark.parametrize("repo_path", [Language.PYTHON], indirect=True) + def test_python_repo(repo_path): + assert (repo_path / "src").exists() + ``` + + """ + if not hasattr(request, "param"): + raise ValueError("Language parameter must be provided via pytest.mark.parametrize") + + language = request.param + return get_repo_path(language) + + +@pytest.fixture(scope="session") +def language_server(request: LanguageParamRequest): + """Create a SyncLanguageServer instance configured for the specified language. + + This fixture requires a language parameter via pytest.mark.parametrize: + + Example: + ``` + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_python_server(language_server: SyncLanguageServer) -> None: + # Use the Python language server + pass + ``` + + You can also test multiple languages in a single test: + ``` + @pytest.mark.parametrize("language_server", [Language.PYTHON, Language.TYPESCRIPT], indirect=True) + def test_multiple_languages(language_server: SyncLanguageServer) -> None: + # This test will run once for each language + pass + ``` + + """ + if not hasattr(request, "param"): + raise ValueError("Language parameter must be provided via pytest.mark.parametrize") + + language = request.param + server = create_default_ls(language) server.start() - try: yield server finally: - # Ensure server is shut down server.stop() diff --git a/test/multilspy/go/test_go_basic.py b/test/multilspy/go/test_go_basic.py new file mode 100644 index 00000000..a87e9fe1 --- /dev/null +++ b/test/multilspy/go/test_go_basic.py @@ -0,0 +1,32 @@ +import os + +import pytest + +from multilspy import SyncLanguageServer +from multilspy.multilspy_config import Language +from multilspy.multilspy_utils import SymbolUtils + + +class TestGoLanguageServer: + @pytest.mark.parametrize("language_server", [Language.GO], indirect=True) + def test_find_symbol(self, language_server: SyncLanguageServer) -> None: + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "main"), "main function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Helper"), "Helper function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "DemoStruct"), "DemoStruct not found in symbol tree" + + @pytest.mark.parametrize("language_server", [Language.GO], indirect=True) + def test_find_referencing_symbols(self, language_server: SyncLanguageServer) -> None: + file_path = os.path.join("main.go") + symbols = language_server.request_document_symbols(file_path) + helper_symbol = None + for sym in symbols[0]: + if sym.get("name") == "Helper": + helper_symbol = sym + break + assert helper_symbol is not None, "Could not find 'Helper' function symbol in main.go" + sel_start = helper_symbol["selectionRange"]["start"] + refs = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert any( + "main.go" in ref.get("relativePath", "") for ref in refs + ), "main.go should reference Helper (tried all positions in selectionRange)" diff --git a/test/multilspy/java/test_java_basic.py b/test/multilspy/java/test_java_basic.py new file mode 100644 index 00000000..e7ff5a8c --- /dev/null +++ b/test/multilspy/java/test_java_basic.py @@ -0,0 +1,50 @@ +import os + +import pytest + +from multilspy import SyncLanguageServer +from multilspy.multilspy_config import Language +from multilspy.multilspy_utils import SymbolUtils + + +@pytest.mark.java +class TestJavaLanguageServer: + @pytest.mark.parametrize("language_server", [Language.JAVA], indirect=True) + def test_find_symbol(self, language_server: SyncLanguageServer) -> None: + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "Main"), "Main class not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Utils"), "Utils class not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Model"), "Model class not found in symbol tree" + + @pytest.mark.parametrize("language_server", [Language.JAVA], indirect=True) + def test_find_referencing_symbols(self, language_server: SyncLanguageServer) -> None: + # Use correct Maven/Java file paths + file_path = os.path.join("src", "main", "java", "test_repo", "Utils.java") + refs = language_server.request_references(file_path, 4, 20) + assert any("Main.java" in ref.get("relativePath", "") for ref in refs), "Main should reference Utils.printHello" + + # Dynamically determine the correct line/column for the 'Model' class name + file_path = os.path.join("src", "main", "java", "test_repo", "Model.java") + symbols = language_server.request_document_symbols(file_path) + model_symbol = None + for sym in symbols[0]: + if sym.get("name") == "Model" and sym.get("kind") == 5: # 5 = Class + model_symbol = sym + break + assert model_symbol is not None, "Could not find 'Model' class symbol in Model.java" + # Use selectionRange if present, otherwise fall back to range + if "selectionRange" in model_symbol: + sel_start = model_symbol["selectionRange"]["start"] + else: + sel_start = model_symbol["range"]["start"] + refs = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert any( + "Main.java" in ref.get("relativePath", "") for ref in refs + ), "Main should reference Model (tried all positions in selectionRange)" + + @pytest.mark.parametrize("language_server", [Language.JAVA], indirect=True) + def test_overview_methods(self, language_server: SyncLanguageServer) -> None: + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "Main"), "Main missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Utils"), "Utils missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Model"), "Model missing from overview" diff --git a/test/multilspy/test_basic.py b/test/multilspy/python/test_python_basic.py similarity index 78% rename from test/multilspy/test_basic.py rename to test/multilspy/python/test_python_basic.py index a22c0fb1..5af4f953 100644 --- a/test/multilspy/test_basic.py +++ b/test/multilspy/python/test_python_basic.py @@ -7,60 +7,79 @@ like request_references using the test repository. import os +import pytest + from multilspy.language_server import SyncLanguageServer +from multilspy.multilspy_config import Language from serena.text_utils import LineType +@pytest.mark.python class TestLanguageServerBasics: """Test basic functionality of the language server.""" - def test_request_references_user_class(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_references_user_class(self, language_server: SyncLanguageServer) -> None: """Test request_references on the User class.""" # Get references to the User class in models.py file_path = os.path.join("test_repo", "models.py") # Line 31 contains the User class definition - references = language_server.request_references(file_path, 31, 6) + # Use selectionRange only + symbols = language_server.request_document_symbols(file_path) + user_symbol = next((s for s in symbols[0] if s.get("name") == "User"), None) + if not user_symbol or "selectionRange" not in user_symbol: + raise AssertionError("User symbol or its selectionRange not found") + sel_start = user_symbol["selectionRange"]["start"] + references = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert len(references) > 1, "User class should be referenced in multiple files (using selectionRange if present)" - # User class should be referenced in multiple files - assert len(references) > 0 - - # At least two references should be found (one for the class definition itself) - assert len(references) > 1 - - def test_request_references_item_class(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_references_item_class(self, language_server: SyncLanguageServer) -> None: """Test request_references on the Item class.""" # Get references to the Item class in models.py file_path = os.path.join("test_repo", "models.py") # Line 56 contains the Item class definition - references = language_server.request_references(file_path, 56, 6) - - # Item class should be referenced in multiple places - assert len(references) > 0 - - # At least one reference should be in services.py (ItemService class) + # Use selectionRange only + symbols = language_server.request_document_symbols(file_path) + item_symbol = next((s for s in symbols[0] if s.get("name") == "Item"), None) + if not item_symbol or "selectionRange" not in item_symbol: + raise AssertionError("Item symbol or its selectionRange not found") + sel_start = item_symbol["selectionRange"]["start"] + references = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) services_references = [ref for ref in references if "services.py" in ref["uri"]] - assert len(services_references) > 0 + assert len(services_references) > 0, "At least one reference should be in services.py (using selectionRange if present)" - def test_request_references_function_parameter(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_references_function_parameter(self, language_server: SyncLanguageServer) -> None: """Test request_references on a function parameter.""" # Get references to the id parameter in get_user method file_path = os.path.join("test_repo", "services.py") # Line 24 contains the get_user method with id parameter - references = language_server.request_references(file_path, 24, 16) + # Use selectionRange only + symbols = language_server.request_document_symbols(file_path) + get_user_symbol = next((s for s in symbols[0] if s.get("name") == "get_user"), None) + if not get_user_symbol or "selectionRange" not in get_user_symbol: + raise AssertionError("get_user symbol or its selectionRange not found") + sel_start = get_user_symbol["selectionRange"]["start"] + references = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert len(references) > 0, "id parameter should be referenced within the method (using selectionRange if present)" - # id parameter should be referenced within the method - assert len(references) > 0 - - def test_request_references_create_user_method(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_references_create_user_method(self, language_server: SyncLanguageServer) -> None: # Get references to the create_user method in UserService file_path = os.path.join("test_repo", "services.py") # Line 15 contains the create_user method definition - references = language_server.request_references(file_path, 15, 9) + # Use selectionRange only + symbols = language_server.request_document_symbols(file_path) + create_user_symbol = next((s for s in symbols[0] if s.get("name") == "create_user"), None) + if not create_user_symbol or "selectionRange" not in create_user_symbol: + raise AssertionError("create_user symbol or its selectionRange not found") + sel_start = create_user_symbol["selectionRange"]["start"] + references = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert len(references) > 1, "Should get valid references for create_user (using selectionRange if present)" - # Verify that we get valid references - assert len(references) > 1 - - def test_retrieve_content_around_line(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_retrieve_content_around_line(self, language_server: SyncLanguageServer) -> None: """Test retrieve_content_around_line functionality with various scenarios.""" file_path = os.path.join("test_repo", "models.py") @@ -166,7 +185,8 @@ class TestLanguageServerBasics: else: assert line.match_type == LineType.AFTER_MATCH - def test_search_files_for_pattern(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_search_files_for_pattern(self, language_server: SyncLanguageServer) -> None: """Test search_files_for_pattern with various patterns and glob filters.""" # Test 1: Search for class definitions across all files class_pattern = r"class\s+\w+\s*(?:\([^{]*\)|:)" diff --git a/test/multilspy/test_retrieval_with_ignored_dirs.py b/test/multilspy/python/test_retrieval_with_ignored_dirs.py similarity index 84% rename from test/multilspy/test_retrieval_with_ignored_dirs.py rename to test/multilspy/python/test_retrieval_with_ignored_dirs.py index 09e4f538..5150d53a 100644 --- a/test/multilspy/test_retrieval_with_ignored_dirs.py +++ b/test/multilspy/python/test_retrieval_with_ignored_dirs.py @@ -6,10 +6,14 @@ import pytest from multilspy.language_server import SyncLanguageServer from multilspy.multilspy_config import Language, MultilspyConfig from multilspy.multilspy_logger import MultilspyLogger +from test.conftest import get_repo_path + +# This mark will be applied to all tests in this module +pytestmark = pytest.mark.python @pytest.fixture(scope="module") -def ls_with_ignored_dirs(repo_path: Path) -> Generator[SyncLanguageServer, None, None]: +def ls_with_ignored_dirs() -> Generator[SyncLanguageServer, None, None]: """Fixture to set up an LS for the python test repo with the 'scripts' directory ignored.""" config = MultilspyConfig( code_language=Language.PYTHON, @@ -17,6 +21,7 @@ def ls_with_ignored_dirs(repo_path: Path) -> Generator[SyncLanguageServer, None, ignored_paths=["scripts", "custom_test"], # Configure the relative path to be ignored ) logger = MultilspyLogger() + repo_path = get_repo_path(Language.PYTHON) ls = SyncLanguageServer.create(config, logger, str(repo_path)) ls.start() try: @@ -25,6 +30,7 @@ def ls_with_ignored_dirs(repo_path: Path) -> Generator[SyncLanguageServer, None, ls.stop() +@pytest.mark.parametrize("ls_with_ignored_dirs", [Language.PYTHON], indirect=True) def test_symbol_tree_ignores_dir(ls_with_ignored_dirs: SyncLanguageServer): """Tests that request_full_symbol_tree ignores the configured directory.""" root = ls_with_ignored_dirs.request_full_symbol_tree()[0] @@ -33,6 +39,7 @@ def test_symbol_tree_ignores_dir(ls_with_ignored_dirs: SyncLanguageServer): assert children_names == {"test_repo", "examples"} +@pytest.mark.parametrize("ls_with_ignored_dirs", [Language.PYTHON], indirect=True) def test_find_references_ignores_dir(ls_with_ignored_dirs: SyncLanguageServer): """Tests that find_references ignores the configured directory.""" # Location of Item, which is referenced in scripts @@ -46,6 +53,7 @@ def test_find_references_ignores_dir(ls_with_ignored_dirs: SyncLanguageServer): assert not any("scripts" in ref["relativePath"] for ref in references) +@pytest.mark.parametrize("repo_path", [Language.PYTHON], indirect=True) def test_refs_and_symbols_with_glob_patterns(repo_path: Path) -> None: """Tests that refs and symbols with glob patterns are ignored.""" config = MultilspyConfig( diff --git a/test/multilspy/test_symbol_retrieval.py b/test/multilspy/python/test_symbol_retrieval.py similarity index 84% rename from test/multilspy/test_symbol_retrieval.py rename to test/multilspy/python/test_symbol_retrieval.py index 14fe1fc7..61a83cea 100644 --- a/test/multilspy/test_symbol_retrieval.py +++ b/test/multilspy/python/test_symbol_retrieval.py @@ -8,14 +8,20 @@ These tests focus on the following methods: import os +import pytest + from multilspy.language_server import SyncLanguageServer +from multilspy.multilspy_config import Language from multilspy.multilspy_types import SymbolKind +pytestmark = pytest.mark.python + class TestLanguageServerSymbols: """Test the language server's symbol-related functionality.""" - def test_request_containing_symbol_function(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_containing_symbol_function(self, language_server: SyncLanguageServer) -> None: """Test request_containing_symbol for a function.""" # Test for a position inside the create_user method file_path = os.path.join("test_repo", "services.py") @@ -29,7 +35,8 @@ class TestLanguageServerSymbols: if "body" in containing_symbol: assert containing_symbol["body"].strip().startswith("def create_user(self") - def test_references_to_variables(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_references_to_variables(self, language_server: SyncLanguageServer) -> None: """Test request_referencing_symbols for a variable.""" file_path = os.path.join("test_repo", "variables.py") # Line 75 contains the field status that is later modified @@ -43,7 +50,8 @@ class TestLanguageServerSymbols: assert "dataclass_instance" in ref_names assert "second_dataclass" in ref_names - def test_request_containing_symbol_class(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_containing_symbol_class(self, language_server: SyncLanguageServer) -> None: """Test request_containing_symbol for a class.""" # Test for a position inside the UserService class but outside any method file_path = os.path.join("test_repo", "services.py") @@ -55,7 +63,8 @@ class TestLanguageServerSymbols: assert containing_symbol["name"] == "UserService" assert containing_symbol["kind"] == SymbolKind.Class - def test_request_containing_symbol_nested(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_containing_symbol_nested(self, language_server: SyncLanguageServer) -> None: """Test request_containing_symbol with nested scopes.""" # Test for a position inside a method which is inside a class file_path = os.path.join("test_repo", "services.py") @@ -80,7 +89,8 @@ class TestLanguageServerSymbols: assert parent_symbol["name"] == "UserService" assert parent_symbol["kind"] == SymbolKind.Class - def test_request_containing_symbol_none(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_containing_symbol_none(self, language_server: SyncLanguageServer) -> None: """Test request_containing_symbol for a position with no containing symbol.""" # Test for a position outside any function/class (e.g., in imports) file_path = os.path.join("test_repo", "services.py") @@ -90,15 +100,19 @@ class TestLanguageServerSymbols: # Should return None or an empty dictionary assert containing_symbol is None or containing_symbol == {} - def test_request_referencing_symbols_function(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_referencing_symbols_function(self, language_server: SyncLanguageServer) -> None: """Test request_referencing_symbols for a function.""" # Test referencing symbols for create_user function file_path = os.path.join("test_repo", "services.py") # Line 15 contains the create_user function definition - ref_symbols = language_server.request_referencing_symbols(file_path, 15, 9) - - # Verify we get referencing symbols - assert len(ref_symbols) > 0 + symbols = language_server.request_document_symbols(file_path) + create_user_symbol = next((s for s in symbols[0] if s.get("name") == "create_user"), None) + if not create_user_symbol or "selectionRange" not in create_user_symbol: + raise AssertionError("create_user symbol or its selectionRange not found") + sel_start = create_user_symbol["selectionRange"]["start"] + ref_symbols = language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"]) + assert len(ref_symbols) > 0, "No referencing symbols found for create_user (selectionRange)" # Verify the structure of referencing symbols for symbol in ref_symbols: @@ -108,45 +122,46 @@ class TestLanguageServerSymbols: assert "start" in symbol["location"]["range"] assert "end" in symbol["location"]["range"] - def test_request_referencing_symbols_class(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_referencing_symbols_class(self, language_server: SyncLanguageServer) -> None: """Test request_referencing_symbols for a class.""" # Test referencing symbols for User class file_path = os.path.join("test_repo", "models.py") # Line 31 contains the User class definition - ref_symbols = language_server.request_referencing_symbols(file_path, 31, 6) - - # Verify we get referencing symbols - assert len(ref_symbols) > 0 - - # At least one reference should be from services.py + symbols = language_server.request_document_symbols(file_path) + user_symbol = next((s for s in symbols[0] if s.get("name") == "User"), None) + if not user_symbol or "selectionRange" not in user_symbol: + raise AssertionError("User symbol or its selectionRange not found") + sel_start = user_symbol["selectionRange"]["start"] + ref_symbols = language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"]) services_references = [ symbol for symbol in ref_symbols if "location" in symbol and "uri" in symbol["location"] and "services.py" in symbol["location"]["uri"] ] + assert len(services_references) > 0, "No referencing symbols from services.py for User (selectionRange)" - assert len(services_references) > 0 - - def test_request_referencing_symbols_parameter(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_referencing_symbols_parameter(self, language_server: SyncLanguageServer) -> None: """Test request_referencing_symbols for a function parameter.""" # Test referencing symbols for id parameter in get_user file_path = os.path.join("test_repo", "services.py") # Line 24 contains the get_user method with id parameter - ref_symbols = language_server.request_referencing_symbols(file_path, 24, 16) - - # Verify we get referencing symbols - assert len(ref_symbols) > 0 - - # Verify the symbols include references within the method body + symbols = language_server.request_document_symbols(file_path) + get_user_symbol = next((s for s in symbols[0] if s.get("name") == "get_user"), None) + if not get_user_symbol or "selectionRange" not in get_user_symbol: + raise AssertionError("get_user symbol or its selectionRange not found") + sel_start = get_user_symbol["selectionRange"]["start"] + ref_symbols = language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"]) method_refs = [ symbol for symbol in ref_symbols - if "location" in symbol and "range" in symbol["location"] and symbol["location"]["range"]["start"]["line"] > 24 + if "location" in symbol and "range" in symbol["location"] and symbol["location"]["range"]["start"]["line"] > sel_start["line"] ] + assert len(method_refs) > 0, "No referencing symbols within method body for get_user (selectionRange)" - assert len(method_refs) > 0 - - def test_request_referencing_symbols_none(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_referencing_symbols_none(self, language_server: SyncLanguageServer) -> None: """Test request_referencing_symbols for a position with no symbol.""" # For positions with no symbol, the method might throw an error or return None/empty list # We'll modify our test to handle this by using a try-except block @@ -163,7 +178,8 @@ class TestLanguageServerSymbols: pass # Tests for request_defining_symbol - def test_request_defining_symbol_variable(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_defining_symbol_variable(self, language_server: SyncLanguageServer) -> None: """Test request_defining_symbol for a variable usage.""" # Test finding the definition of a symbol in the create_user method file_path = os.path.join("test_repo", "services.py") @@ -181,7 +197,8 @@ class TestLanguageServerSymbols: if "location" in defining_symbol and "uri" in defining_symbol["location"]: assert "services.py" in defining_symbol["location"]["uri"] - def test_request_defining_symbol_imported_class(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_defining_symbol_imported_class(self, language_server: SyncLanguageServer) -> None: """Test request_defining_symbol for an imported class.""" # Test finding the definition of the 'User' class used in the UserService.create_user method file_path = os.path.join("test_repo", "services.py") @@ -192,7 +209,8 @@ class TestLanguageServerSymbols: assert defining_symbol is not None assert defining_symbol.get("name") == "User" - def test_request_defining_symbol_method_call(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_defining_symbol_method_call(self, language_server: SyncLanguageServer) -> None: """Test request_defining_symbol for a method call.""" # Create an example file path for a file that calls UserService.create_user examples_file_path = os.path.join("examples", "user_management.py") @@ -216,7 +234,8 @@ class TestLanguageServerSymbols: warnings.warn("Could not verify method call definition - file structure may differ from expected") - def test_request_defining_symbol_none(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_defining_symbol_none(self, language_server: SyncLanguageServer) -> None: """Test request_defining_symbol for a position with no symbol.""" # Test for a position with no symbol (e.g., whitespace or comment) file_path = os.path.join("test_repo", "services.py") @@ -226,7 +245,8 @@ class TestLanguageServerSymbols: # Should return None for positions with no symbol assert defining_symbol is None - def test_request_containing_symbol_variable(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_containing_symbol_variable(self, language_server: SyncLanguageServer) -> None: """Test request_containing_symbol where the symbol is a variable.""" # Test for a position inside a variable definition file_path = os.path.join("test_repo", "services.py") @@ -238,7 +258,8 @@ class TestLanguageServerSymbols: assert containing_symbol["name"] == "user_var_str" assert containing_symbol["kind"] == SymbolKind.Variable - def test_request_defining_symbol_nested_function(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_defining_symbol_nested_function(self, language_server: SyncLanguageServer) -> None: """Test request_defining_symbol for a nested function or closure.""" # Use the existing nested.py file which contains nested classes and methods file_path = os.path.join("test_repo", "nested.py") @@ -288,7 +309,8 @@ class TestLanguageServerSymbols: assert defining_symbol.get("name") == "func_within_func" assert defining_symbol.get("kind") == SymbolKind.Function.value - def test_symbol_methods_integration(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_symbol_methods_integration(self, language_server: SyncLanguageServer) -> None: """Test the integration between different symbol-related methods.""" # This test demonstrates using the various symbol methods together # by finding a symbol and then checking its definition @@ -336,7 +358,8 @@ class TestLanguageServerSymbols: warnings.warn("Could not verify container hierarchy - implementation detail") - def test_symbol_tree_structure(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_symbol_tree_structure(self, language_server: SyncLanguageServer) -> None: """Test that the symbol tree structure is correctly built.""" # Get all symbols in the test file repo_structure = language_server.request_full_symbol_tree() @@ -363,7 +386,8 @@ class TestLanguageServerSymbols: _, user_management_roots = language_server.request_document_symbols(os.path.join("examples", "user_management.py")) assert user_management_roots == user_management_node["children"] - def test_symbol_tree_structure_subdir(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_symbol_tree_structure_subdir(self, language_server: SyncLanguageServer) -> None: """Test that the symbol tree structure is correctly built.""" # Get all symbols in the test file examples_package_roots = language_server.request_full_symbol_tree(within_relative_path="examples") @@ -383,7 +407,8 @@ class TestLanguageServerSymbols: _, user_management_roots = language_server.request_document_symbols(os.path.join("examples", "user_management.py")) assert user_management_roots == user_management_node["children"] - def test_request_dir_overview(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_dir_overview(self, language_server: SyncLanguageServer) -> None: """Test that request_dir_overview returns correct symbol information for files in a directory.""" # Get overview of the examples directory overview = language_server.request_dir_overview("test_repo") @@ -407,7 +432,8 @@ class TestLanguageServerSymbols: for symbol in expected_symbols: assert symbol in services_symbols - def test_request_document_overview(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_request_document_overview(self, language_server: SyncLanguageServer) -> None: """Test that request_document_overview returns correct symbol information for a file.""" # Get overview of the user_management.py file overview = language_server.request_document_overview(os.path.join("examples", "user_management.py")) @@ -416,7 +442,8 @@ class TestLanguageServerSymbols: symbol_names = {s_info[0] for s_info in overview} assert {"UserStats", "UserManager", "process_user_data", "main"}.issubset(symbol_names) - def test_containing_symbol_of_var_is_file(self, language_server: SyncLanguageServer): + @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) + def test_containing_symbol_of_var_is_file(self, language_server: SyncLanguageServer) -> None: """Test that the containing symbol of a variable is the file itself.""" # Get the containing symbol of a variable in a file file_path = os.path.join("test_repo", "services.py") diff --git a/test/multilspy/rust/test_rust_basic.py b/test/multilspy/rust/test_rust_basic.py new file mode 100644 index 00000000..02e9d502 --- /dev/null +++ b/test/multilspy/rust/test_rust_basic.py @@ -0,0 +1,57 @@ +import os + +import pytest + +from multilspy import SyncLanguageServer +from multilspy.multilspy_config import Language +from multilspy.multilspy_utils import SymbolUtils + + +@pytest.mark.rust +class TestRustLanguageServer: + @pytest.mark.parametrize("language_server", [Language.RUST], indirect=True) + def test_find_references_raw(self, language_server: SyncLanguageServer) -> None: + # Directly test the request_references method for the add function + file_path = os.path.join("src", "lib.rs") + symbols = language_server.request_document_symbols(file_path) + add_symbol = None + for sym in symbols[0]: + if sym.get("name") == "add": + add_symbol = sym + break + assert add_symbol is not None, "Could not find 'add' function symbol in lib.rs" + sel_start = add_symbol["selectionRange"]["start"] + refs = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert any( + "main.rs" in ref.get("relativePath", "") for ref in refs + ), "main.rs should reference add (raw, tried all positions in selectionRange)" + + @pytest.mark.parametrize("language_server", [Language.RUST], indirect=True) + def test_find_symbol(self, language_server: SyncLanguageServer) -> None: + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "main"), "main function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "add"), "add function not found in symbol tree" + # Add more as needed based on test_repo + + @pytest.mark.parametrize("language_server", [Language.RUST], indirect=True) + def test_find_referencing_symbols(self, language_server: SyncLanguageServer) -> None: + # Find references to 'add' defined in lib.rs, should be referenced from main.rs + file_path = os.path.join("src", "lib.rs") + symbols = language_server.request_document_symbols(file_path) + add_symbol = None + for sym in symbols[0]: + if sym.get("name") == "add": + add_symbol = sym + break + assert add_symbol is not None, "Could not find 'add' function symbol in lib.rs" + sel_start = add_symbol["selectionRange"]["start"] + refs = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert any( + "main.rs" in ref.get("relativePath", "") for ref in refs + ), "main.rs should reference add (tried all positions in selectionRange)" + + @pytest.mark.parametrize("language_server", [Language.RUST], indirect=True) + def test_overview_methods(self, language_server: SyncLanguageServer) -> None: + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "main"), "main missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "add"), "add missing from overview" diff --git a/test/multilspy/typescript/test_typescript_basic.py b/test/multilspy/typescript/test_typescript_basic.py new file mode 100644 index 00000000..442b5a1e --- /dev/null +++ b/test/multilspy/typescript/test_typescript_basic.py @@ -0,0 +1,33 @@ +import os + +import pytest + +from multilspy import SyncLanguageServer +from multilspy.multilspy_config import Language +from multilspy.multilspy_utils import SymbolUtils + + +@pytest.mark.typescript +class TestTypescriptLanguageServer: + @pytest.mark.parametrize("language_server", [Language.TYPESCRIPT], indirect=True) + def test_find_symbol(self, language_server: SyncLanguageServer) -> None: + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "DemoClass"), "DemoClass not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "helperFunction"), "helperFunction not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "printValue"), "printValue method not found in symbol tree" + + @pytest.mark.parametrize("language_server", [Language.TYPESCRIPT], indirect=True) + def test_find_referencing_symbols(self, language_server: SyncLanguageServer) -> None: + file_path = os.path.join("index.ts") + symbols = language_server.request_document_symbols(file_path) + helper_symbol = None + for sym in symbols[0]: + if sym.get("name") == "helperFunction": + helper_symbol = sym + break + assert helper_symbol is not None, "Could not find 'helperFunction' symbol in index.ts" + sel_start = helper_symbol["selectionRange"]["start"] + refs = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert any( + "index.ts" in ref.get("relativePath", "") for ref in refs + ), "index.ts should reference helperFunction (tried all positions in selectionRange)" diff --git a/test/resources/repos/go/test_repo/main.go b/test/resources/repos/go/test_repo/main.go new file mode 100644 index 00000000..b38c31ff --- /dev/null +++ b/test/resources/repos/go/test_repo/main.go @@ -0,0 +1,20 @@ +package main + +import "fmt" + +func main() { + fmt.Println("Hello, Go!") + Helper() +} + +func Helper() { + fmt.Println("Helper function called") +} + +type DemoStruct struct { + Field int +} + +func UsingHelper() { + Helper() +} diff --git a/test/resources/repos/java/test_repo/pom.xml b/test/resources/repos/java/test_repo/pom.xml new file mode 100644 index 00000000..7df64e11 --- /dev/null +++ b/test/resources/repos/java/test_repo/pom.xml @@ -0,0 +1,14 @@ + + 4.0.0 + org.example + test_repo + 1.0-SNAPSHOT + jar + Java Test Repo + + 17 + 17 + + diff --git a/test/resources/repos/java/test_repo/src/main/java/test_repo/Main.java b/test/resources/repos/java/test_repo/src/main/java/test_repo/Main.java new file mode 100644 index 00000000..ed79b9fe --- /dev/null +++ b/test/resources/repos/java/test_repo/src/main/java/test_repo/Main.java @@ -0,0 +1,13 @@ +package test_repo; + +public class Main { + public static void main(String[] args) { + Utils.printHello(); + Model model = new Model("Cascade"); + System.out.println(model.getName()); + acceptModel(model); + } + public static void acceptModel(Model m) { + // Do nothing, just for LSP reference + } +} diff --git a/test/resources/repos/java/test_repo/src/main/java/test_repo/Model.java b/test/resources/repos/java/test_repo/src/main/java/test_repo/Model.java new file mode 100644 index 00000000..a2c6459b --- /dev/null +++ b/test/resources/repos/java/test_repo/src/main/java/test_repo/Model.java @@ -0,0 +1,13 @@ +package test_repo; + +public class Model { + private String name; + + public Model(String name) { + this.name = name; + } + + public String getName() { + return name; + } +} diff --git a/test/resources/repos/java/test_repo/src/main/java/test_repo/ModelUser.java b/test/resources/repos/java/test_repo/src/main/java/test_repo/ModelUser.java new file mode 100644 index 00000000..9cffa9fd --- /dev/null +++ b/test/resources/repos/java/test_repo/src/main/java/test_repo/ModelUser.java @@ -0,0 +1,8 @@ +package test_repo; + +public class ModelUser { + public static void main(String[] args) { + Model model = new Model("Cascade"); + System.out.println(model.getName()); + } +} diff --git a/test/resources/repos/java/test_repo/src/main/java/test_repo/Utils.java b/test/resources/repos/java/test_repo/src/main/java/test_repo/Utils.java new file mode 100644 index 00000000..107f4c80 --- /dev/null +++ b/test/resources/repos/java/test_repo/src/main/java/test_repo/Utils.java @@ -0,0 +1,7 @@ +package test_repo; + +public class Utils { + public static void printHello() { + System.out.println("Hello from Utils!"); + } +} diff --git a/test/resources/test_repo/.gitignore b/test/resources/repos/python/test_repo/.gitignore similarity index 100% rename from test/resources/test_repo/.gitignore rename to test/resources/repos/python/test_repo/.gitignore diff --git a/test/resources/test_repo/custom_test/__init__.py b/test/resources/repos/python/test_repo/custom_test/__init__.py similarity index 100% rename from test/resources/test_repo/custom_test/__init__.py rename to test/resources/repos/python/test_repo/custom_test/__init__.py diff --git a/test/resources/test_repo/custom_test/advanced_features.py b/test/resources/repos/python/test_repo/custom_test/advanced_features.py similarity index 100% rename from test/resources/test_repo/custom_test/advanced_features.py rename to test/resources/repos/python/test_repo/custom_test/advanced_features.py diff --git a/test/resources/test_repo/examples/__init__.py b/test/resources/repos/python/test_repo/examples/__init__.py similarity index 100% rename from test/resources/test_repo/examples/__init__.py rename to test/resources/repos/python/test_repo/examples/__init__.py diff --git a/test/resources/test_repo/examples/user_management.py b/test/resources/repos/python/test_repo/examples/user_management.py similarity index 100% rename from test/resources/test_repo/examples/user_management.py rename to test/resources/repos/python/test_repo/examples/user_management.py diff --git a/test/resources/test_repo/scripts/__init__.py b/test/resources/repos/python/test_repo/scripts/__init__.py similarity index 100% rename from test/resources/test_repo/scripts/__init__.py rename to test/resources/repos/python/test_repo/scripts/__init__.py diff --git a/test/resources/test_repo/scripts/run_app.py b/test/resources/repos/python/test_repo/scripts/run_app.py similarity index 100% rename from test/resources/test_repo/scripts/run_app.py rename to test/resources/repos/python/test_repo/scripts/run_app.py diff --git a/test/resources/repos/python/test_repo/test_repo/__init__.py b/test/resources/repos/python/test_repo/test_repo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/resources/test_repo/test_repo/complex_types.py b/test/resources/repos/python/test_repo/test_repo/complex_types.py similarity index 100% rename from test/resources/test_repo/test_repo/complex_types.py rename to test/resources/repos/python/test_repo/test_repo/complex_types.py diff --git a/test/resources/test_repo/test_repo/models.py b/test/resources/repos/python/test_repo/test_repo/models.py similarity index 100% rename from test/resources/test_repo/test_repo/models.py rename to test/resources/repos/python/test_repo/test_repo/models.py diff --git a/test/resources/test_repo/test_repo/name_collisions.py b/test/resources/repos/python/test_repo/test_repo/name_collisions.py similarity index 100% rename from test/resources/test_repo/test_repo/name_collisions.py rename to test/resources/repos/python/test_repo/test_repo/name_collisions.py diff --git a/test/resources/test_repo/test_repo/nested.py b/test/resources/repos/python/test_repo/test_repo/nested.py similarity index 100% rename from test/resources/test_repo/test_repo/nested.py rename to test/resources/repos/python/test_repo/test_repo/nested.py diff --git a/test/resources/test_repo/test_repo/nested_base.py b/test/resources/repos/python/test_repo/test_repo/nested_base.py similarity index 100% rename from test/resources/test_repo/test_repo/nested_base.py rename to test/resources/repos/python/test_repo/test_repo/nested_base.py diff --git a/test/resources/test_repo/test_repo/overloaded.py b/test/resources/repos/python/test_repo/test_repo/overloaded.py similarity index 100% rename from test/resources/test_repo/test_repo/overloaded.py rename to test/resources/repos/python/test_repo/test_repo/overloaded.py diff --git a/test/resources/test_repo/test_repo/services.py b/test/resources/repos/python/test_repo/test_repo/services.py similarity index 100% rename from test/resources/test_repo/test_repo/services.py rename to test/resources/repos/python/test_repo/test_repo/services.py diff --git a/test/resources/test_repo/test_repo/utils.py b/test/resources/repos/python/test_repo/test_repo/utils.py similarity index 100% rename from test/resources/test_repo/test_repo/utils.py rename to test/resources/repos/python/test_repo/test_repo/utils.py diff --git a/test/resources/test_repo/test_repo/variables.py b/test/resources/repos/python/test_repo/test_repo/variables.py similarity index 100% rename from test/resources/test_repo/test_repo/variables.py rename to test/resources/repos/python/test_repo/test_repo/variables.py diff --git a/test/resources/repos/rust/test_repo/Cargo.lock b/test/resources/repos/rust/test_repo/Cargo.lock new file mode 100644 index 00000000..ebc459f1 --- /dev/null +++ b/test/resources/repos/rust/test_repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "rsandbox" +version = "0.1.0" diff --git a/test/resources/repos/rust/test_repo/Cargo.toml b/test/resources/repos/rust/test_repo/Cargo.toml new file mode 100644 index 00000000..43ffbf6e --- /dev/null +++ b/test/resources/repos/rust/test_repo/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "rsandbox" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/test/resources/repos/rust/test_repo/src/lib.rs b/test/resources/repos/rust/test_repo/src/lib.rs new file mode 100644 index 00000000..8ad43ead --- /dev/null +++ b/test/resources/repos/rust/test_repo/src/lib.rs @@ -0,0 +1,9 @@ +// This function returns the sum of 2 + 2 +pub fn add() -> i32 { + let res = 2 + 2; + res +} +pub fn multiply() -> i32 { + 2 * 3 +} + diff --git a/test/resources/repos/rust/test_repo/src/main.rs b/test/resources/repos/rust/test_repo/src/main.rs new file mode 100644 index 00000000..7385c9d9 --- /dev/null +++ b/test/resources/repos/rust/test_repo/src/main.rs @@ -0,0 +1,8 @@ +use rsandbox::add; + +fn main() { + println!("Hello, World!"); + println!("Good morning!"); + println!("add result: {}", add()); + println!("inserted line"); +} diff --git a/test/resources/repos/typescript/test_repo/index.ts b/test/resources/repos/typescript/test_repo/index.ts new file mode 100644 index 00000000..7b8bd1da --- /dev/null +++ b/test/resources/repos/typescript/test_repo/index.ts @@ -0,0 +1,16 @@ +export class DemoClass { + value: number; + constructor(value: number) { + this.value = value; + } + printValue() { + console.log(this.value); + } +} + +export function helperFunction() { + const demo = new DemoClass(42); + demo.printValue(); +} + +helperFunction(); diff --git a/test/resources/repos/typescript/test_repo/tsconfig.json b/test/resources/repos/typescript/test_repo/tsconfig.json new file mode 100644 index 00000000..ba0514f5 --- /dev/null +++ b/test/resources/repos/typescript/test_repo/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "ES2017", + "module": "commonjs", + "strict": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true + }, + "include": ["**/*.ts"] +} diff --git a/test/resources/repos/typescript/test_repo/use_helper.ts b/test/resources/repos/typescript/test_repo/use_helper.ts new file mode 100644 index 00000000..e9147d27 --- /dev/null +++ b/test/resources/repos/typescript/test_repo/use_helper.ts @@ -0,0 +1,8 @@ +import {helperFunction} from "./index"; + + +export function useHelper() { + helperFunction(); +} + +useHelper(); diff --git a/test/resources/test_repo/ignore_this_dir_with_postfix/ignored_module.py b/test/resources/test_repo/ignore_this_dir_with_postfix/ignored_module.py deleted file mode 100644 index 1e6da936..00000000 --- a/test/resources/test_repo/ignore_this_dir_with_postfix/ignored_module.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -Example demonstrating user management with the test_repo module. - -This example showcases: -- Creating and managing users -- Using various object types and relationships -- Type annotations and complex Python patterns -""" - -import logging -from dataclasses import dataclass -from typing import Any - -from test_repo.models import User, create_user_object -from test_repo.services import UserService - -# Set up logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -@dataclass -class UserStats: - """Statistics about user activity.""" - - user_id: str - login_count: int = 0 - last_active_days: int = 0 - engagement_score: float = 0.0 - - def is_active(self) -> bool: - """Check if the user is considered active.""" - return self.last_active_days < 30 - - -class UserManager: - """Example class demonstrating complex user management.""" - - def __init__(self, service: UserService): - self.service = service - self.active_users: dict[str, User] = {} - self.user_stats: dict[str, UserStats] = {} - - def register_user(self, name: str, email: str, roles: list[str] | None = None) -> User: - """Register a new user.""" - logger.info(f"Registering new user: {name} ({email})") - user = self.service.create_user(name=name, email=email, roles=roles) - self.active_users[user.id] = user - self.user_stats[user.id] = UserStats(user_id=user.id) - return user - - def get_user(self, user_id: str) -> User | None: - """Get a user by ID.""" - if user_id in self.active_users: - return self.active_users[user_id] - - # Try to fetch from service - user = self.service.get_user(user_id) - if user: - self.active_users[user.id] = user - return user - - def update_user_stats(self, user_id: str, login_count: int, days_since_active: int) -> None: - """Update statistics for a user.""" - if user_id not in self.user_stats: - self.user_stats[user_id] = UserStats(user_id=user_id) - - stats = self.user_stats[user_id] - stats.login_count = login_count - stats.last_active_days = days_since_active - - # Calculate engagement score based on activity - engagement = (100 - min(days_since_active, 100)) * 0.8 - engagement += min(login_count, 20) * 0.2 - stats.engagement_score = engagement - - def get_active_users(self) -> list[User]: - """Get all active users.""" - active_user_ids = [user_id for user_id, stats in self.user_stats.items() if stats.is_active()] - return [self.active_users[user_id] for user_id in active_user_ids if user_id in self.active_users] - - def get_user_by_email(self, email: str) -> User | None: - """Find a user by their email address.""" - for user in self.active_users.values(): - if user.email == email: - return user - return None - - -# Example function demonstrating type annotations -def process_user_data(users: list[User], include_inactive: bool = False, transform_func: callable | None = None) -> dict[str, Any]: - """Process user data with optional transformations.""" - result: dict[str, Any] = {"users": [], "total": 0, "admin_count": 0} - - for user in users: - if transform_func: - user_data = transform_func(user.to_dict()) - else: - user_data = user.to_dict() - - result["users"].append(user_data) - result["total"] += 1 - - if "admin" in user.roles: - result["admin_count"] += 1 - - return result - - -def main(): - """Main function demonstrating the usage of UserManager.""" - # Initialize service and manager - service = UserService() - manager = UserManager(service) - - # Register some users - admin = manager.register_user("Admin User", "admin@example.com", ["admin"]) - user1 = manager.register_user("Regular User", "user@example.com", ["user"]) - user2 = manager.register_user("Another User", "another@example.com", ["user"]) - - # Update some stats - manager.update_user_stats(admin.id, 100, 5) - manager.update_user_stats(user1.id, 50, 10) - manager.update_user_stats(user2.id, 10, 45) # Inactive user - - # Get active users - active_users = manager.get_active_users() - logger.info(f"Active users: {len(active_users)}") - - # Process user data - user_data = process_user_data(active_users, transform_func=lambda u: {**u, "full_name": u.get("name", "")}) - - logger.info(f"Processed {user_data['total']} users, {user_data['admin_count']} admins") - - # Example of calling create_user directly - external_user = create_user_object(id="ext123", name="External User", email="external@example.org", roles=["external"]) - logger.info(f"Created external user: {external_user.name}") - - -if __name__ == "__main__": - main() diff --git a/test/serena/test_edit_marker.py b/test/serena/test_edit_marker.py new file mode 100644 index 00000000..9a07f271 --- /dev/null +++ b/test/serena/test_edit_marker.py @@ -0,0 +1,13 @@ +from serena.agent import CreateTextFileTool, ReadFileTool, Tool + + +class TestEditMarker: + def test_tool_can_edit_method(self): + """Test that Tool.can_edit() method works correctly""" + # Non-editing tool should return False + assert issubclass(ReadFileTool, Tool) + assert not ReadFileTool.can_edit() + + # Editing tool should return True + assert issubclass(CreateTextFileTool, Tool) + assert CreateTextFileTool.can_edit() diff --git a/test/serena/test_mcp.py b/test/serena/test_mcp.py new file mode 100644 index 00000000..b1999174 --- /dev/null +++ b/test/serena/test_mcp.py @@ -0,0 +1,293 @@ +"""Tests for the mcp.py module in serena.""" + +import pytest +from mcp.server.fastmcp.tools.base import Tool as MCPTool + +from serena.agent import Tool, iter_tool_classes +from serena.mcp import make_tool + + +class BaseMockTool(Tool): + """A mock Tool class for testing.""" + + def __init__(self) -> None: + pass + + +class BasicTool(BaseMockTool): + """A mock Tool class for testing.""" + + def apply(self, name: str, age: int = 0) -> str: + """This is a test function. + + :param name: The person's name + :param age: The person's age + :return: A greeting message + """ + return f"Hello {name}, you are {age} years old!" + + def apply_ex( + self, + log_call: bool = True, + catch_exceptions: bool = True, + **kwargs, + ) -> str: + """Mock implementation of apply_ex.""" + return self.apply(**kwargs) + + +def test_make_tool_basic() -> None: + """Test that make_tool correctly creates an MCP tool from a Tool object.""" + mock_tool = BasicTool() + + mcp_tool = make_tool(mock_tool) + + # Test that the MCP tool has the correct properties + assert isinstance(mcp_tool, MCPTool) + assert mcp_tool.name == "basic" + assert "This is a test function. Returns A greeting message." in mcp_tool.description + + # Test that the parameters were correctly processed + parameters = mcp_tool.parameters + assert "properties" in parameters + assert "name" in parameters["properties"] + assert "age" in parameters["properties"] + assert parameters["properties"]["name"]["description"] == "The person's name." + assert parameters["properties"]["age"]["description"] == "The person's age." + + +def test_make_tool_execution() -> None: + """Test that the execution function created by make_tool works correctly.""" + mock_tool = BasicTool() + mcp_tool = make_tool(mock_tool) + + # Execute the MCP tool function + result = mcp_tool.fn(name="Alice", age=30) + + assert result == "Hello Alice, you are 30 years old!" + + +def test_make_tool_no_params() -> None: + """Test make_tool with a function that has no parameters.""" + + class NoParamsTool(BaseMockTool): + def apply(self) -> str: + """This is a test function with no parameters. + + :return: A simple result + """ + return "Simple result" + + def apply_ex(self, *args, **kwargs) -> str: + return self.apply() + + tool = NoParamsTool() + mcp_tool = make_tool(tool) + + assert mcp_tool.name == "no_params" + assert "This is a test function with no parameters. Returns A simple result." in mcp_tool.description + assert mcp_tool.parameters["properties"] == {} + + +def test_make_tool_no_return_description() -> None: + """Test make_tool with a function that has no return description.""" + + class NoReturnTool(BaseMockTool): + def apply(self, param: str) -> str: + """This is a test function. + + :param param: The parameter + """ + return f"Processed: {param}" + + def apply_ex(self, *args, **kwargs) -> str: + return self.apply(**kwargs) + + tool = NoReturnTool() + mcp_tool = make_tool(tool) + + assert mcp_tool.name == "no_return" + assert mcp_tool.description == "This is a test function." + assert mcp_tool.parameters["properties"]["param"]["description"] == "The parameter." + + +def test_make_tool_parameter_not_in_docstring() -> None: + """Test make_tool when a parameter in properties is not in the docstring.""" + + class MissingParamTool(BaseMockTool): + def apply(self, name: str, missing_param: str = "") -> str: + """This is a test function. + + :param name: The person's name + """ + return f"Hello {name}! Missing param: {missing_param}" + + def apply_ex(self, *args, **kwargs) -> str: + return self.apply(**kwargs) + + tool = MissingParamTool() + mcp_tool = make_tool(tool) + + assert "name" in mcp_tool.parameters["properties"] + assert "missing_param" in mcp_tool.parameters["properties"] + assert mcp_tool.parameters["properties"]["name"]["description"] == "The person's name." + assert "description" not in mcp_tool.parameters["properties"]["missing_param"] + + +def test_make_tool_multiline_docstring() -> None: + """Test make_tool with a complex multi-line docstring.""" + + class ComplexDocTool(BaseMockTool): + def apply(self, project_file_path: str, host: str, port: int) -> str: + """Create an MCP server. + + This function creates and configures a Model Context Protocol server + with the specified settings. + + :param project_file_path: The path to the project file, or None + :param host: The host to bind to + :param port: The port to bind to + :return: A configured FastMCP server instance + """ + return f"Server config: {project_file_path}, {host}:{port}" + + def apply_ex(self, *args, **kwargs) -> str: + return self.apply(**kwargs) + + tool = ComplexDocTool() + mcp_tool = make_tool(tool) + + assert "Create an MCP server" in mcp_tool.description + assert "Returns A configured FastMCP server instance" in mcp_tool.description + assert mcp_tool.parameters["properties"]["project_file_path"]["description"] == "The path to the project file, or None." + assert mcp_tool.parameters["properties"]["host"]["description"] == "The host to bind to." + assert mcp_tool.parameters["properties"]["port"]["description"] == "The port to bind to." + + +def test_make_tool_capitalization_and_periods() -> None: + """Test that make_tool properly handles capitalization and periods in descriptions.""" + + class FormatTool(BaseMockTool): + def apply(self, param1: str, param2: str, param3: str) -> str: + """Test function. + + :param param1: lowercase description + :param param2: description with period. + :param param3: description with Capitalized word. + """ + return f"Formatted: {param1}, {param2}, {param3}" + + def apply_ex(self, *args, **kwargs) -> str: + return self.apply(**kwargs) + + tool = FormatTool() + mcp_tool = make_tool(tool) + + assert mcp_tool.parameters["properties"]["param1"]["description"] == "Lowercase description." + assert mcp_tool.parameters["properties"]["param2"]["description"] == "Description with period." + assert mcp_tool.parameters["properties"]["param3"]["description"] == "Description with Capitalized word." + + +def test_make_tool_missing_apply() -> None: + """Test make_tool with a tool that doesn't have an apply method.""" + + class BadTool(BaseMockTool): + pass + + tool = BadTool() + + with pytest.raises(AttributeError): + make_tool(tool) + + +@pytest.mark.parametrize( + "docstring, expected_description", + [ + ( + """This is a test function. + + :param param: The parameter + :return: A result + """, + "This is a test function. Returns A result.", + ), + ( + """ + :param param: The parameter + :return: A result + """, + "Returns A result.", + ), + ( + """ + :param param: The parameter + """, + "", + ), + ("", ""), + ], +) +def test_make_tool_descriptions(docstring, expected_description) -> None: + """Test make_tool with various docstring formats.""" + + class TestTool(BaseMockTool): + def apply(self, param: str) -> str: + return f"Result: {param}" + + def apply_ex(self, *args, **kwargs) -> str: + return self.apply(**kwargs) + + # Dynamically set the docstring + TestTool.apply.__doc__ = docstring + + tool = TestTool() + mcp_tool = make_tool(tool) + + assert mcp_tool.name == "test" + assert mcp_tool.description == expected_description + + +def is_test_mock_class(tool_class: type) -> bool: + """Check if a class is a test mock class.""" + # Check if the class is defined in a test module + module_name = tool_class.__module__ + return ( + module_name.startswith(("test.", "tests.")) + or "test_" in module_name + or tool_class.__name__ + in [ + "BaseMockTool", + "BasicTool", + "BadTool", + "NoParamsTool", + "NoReturnTool", + "MissingParamTool", + "ComplexDocTool", + "FormatTool", + "NoDescriptionTool", + ] + ) + + +@pytest.mark.parametrize("tool_class", list(iter_tool_classes())) +def test_make_tool_all_tools(tool_class) -> None: + """Test that make_tool works for all tools in the codebase.""" + + # Create a mock agent for tool initialization + class MockAgent: + def __init__(self): + self.project_config = None + self.serena_config = None + + # Create an instance of the tool + tool_instance = tool_class(MockAgent()) + + # Try to create an MCP tool from it + mcp_tool = make_tool(tool_instance) + + # Basic validation + assert isinstance(mcp_tool, MCPTool) + assert mcp_tool.name == tool_class.get_name() + + # The description should be a string (either from docstring or default) + assert isinstance(mcp_tool.description, str) diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py new file mode 100644 index 00000000..7c3d76eb --- /dev/null +++ b/test/serena/test_serena_agent.py @@ -0,0 +1,75 @@ +import json +import os +import time + +import pytest + +from multilspy.multilspy_config import Language +from serena.agent import FindReferencingSymbolsTool, FindSymbolTool, ProjectConfig, SerenaAgent, SerenaConfig +from test.conftest import LanguageParamRequest, get_repo_path + + +@pytest.fixture +def serena_config(): + return SerenaConfig() + + +@pytest.fixture +def serena_agent(request: LanguageParamRequest, serena_config) -> SerenaAgent: + language = Language(request.param) + repo = get_repo_path(language) + project_config = ProjectConfig(project_name=os.path.basename(str(repo)), language=language, project_root=str(repo)) + return SerenaAgent(project_config=project_config, serena_config=serena_config) + + +class TestSerenaAgent: + @pytest.mark.parametrize( + "serena_agent,symbol_name,expected_kind,expected_file", + [ + pytest.param(Language.PYTHON, "User", "Class", "models.py", marks=pytest.mark.python), + pytest.param(Language.GO, "Helper", "Function", "main.go", marks=pytest.mark.go), + pytest.param(Language.JAVA, "Model", "Class", "Model.java", marks=pytest.mark.java), + pytest.param(Language.RUST, "add", "Function", "lib.rs", marks=pytest.mark.rust), + pytest.param(Language.TYPESCRIPT, "DemoClass", "Class", "index.ts", marks=pytest.mark.typescript), + ], + indirect=["serena_agent"], + ) + def test_find_symbol(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str): + agent = serena_agent + find_symbol_tool = agent.get_tool(FindSymbolTool) + result = find_symbol_tool.apply(symbol_name) + symbols = json.loads(result) + assert any( + symbol_name in s["name"] and expected_kind.lower() in s["kind"].lower() and expected_file in s["location"]["relative_path"] + for s in symbols + ), f"Expected to find {symbol_name} ({expected_kind}) in {expected_file} for {agent.project_config.language.name}" + + @pytest.mark.parametrize( + "serena_agent,symbol_name,def_file,ref_file", + [ + (Language.PYTHON, "User", "test_repo/models.py", "test_repo/services.py"), + (Language.GO, "Helper", "main.go", "main.go"), + (Language.JAVA, "Model", "src/main/java/test_repo/Model.java", "src/main/java/test_repo/Main.java"), + (Language.RUST, "add", "src/lib.rs", "src/main.rs"), + (Language.TYPESCRIPT, "helperFunction", "index.ts", "use_helper.ts"), + ], + indirect=["serena_agent"], + ) + def test_find_symbol_references(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: + agent = serena_agent + # Find the symbol location first + find_symbol_tool = agent.get_tool(FindSymbolTool) + result = find_symbol_tool.apply(symbol_name, within_relative_path=def_file) + time.sleep(1) + symbols = json.loads(result) + # Find the definition location + def_symbol = symbols[0] + loc = def_symbol["location"] + # sel_start = def_symbol["location"]["selectionRange"]["start"] + # Now find references + find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) + result = find_refs_tool.apply(def_file, loc["line"], loc["column"]) + refs = json.loads(result) + assert any( + ref["location"]["relative_path"] == ref_file for ref in refs + ), f"Expected to find reference to {symbol_name} in {ref_file} for {agent.project_config.language.name}. refs={refs}" diff --git a/uv.lock b/uv.lock index 456931c3..d8185ded 100644 --- a/uv.lock +++ b/uv.lock @@ -971,6 +971,7 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "agno" }, + { name = "docstring-parser" }, { name = "dotenv" }, { name = "fastmcp" }, { name = "jinja2" }, @@ -1017,6 +1018,7 @@ requires-dist = [ { name = "agno", marker = "extra == 'agno'", specifier = ">=1.2.6" }, { name = "anthropic", marker = "extra == 'anthropic'", specifier = ">=0.49.0" }, { name = "black", extras = ["jupyter"], marker = "extra == 'dev'", specifier = ">=23.7.0" }, + { name = "docstring-parser", specifier = ">=0.16" }, { name = "dotenv", specifier = ">=0.9.9" }, { name = "fastapi", marker = "extra == 'agno'", specifier = ">=0.115.12" }, { name = "fastmcp", specifier = ">=0.4.1" },