Merge pull request #1 from oraios/feature/onboarding

Feature/onboarding

.gitignore

@@ -135,3 +135,8 @@ dmypy.json
 # reports
 pylint.html
 .pylint.d
+
+# Serena-specific
+/*.yml
+!myproject.demo.yml
+

README.md

@@ -17,13 +17,13 @@ notebooks. Just run `poe` to see the available commands.
 
 ### Python (uv) setup
 
-You can install the dependencies with
+You can install a virtual environment with the required as follows
 
-```shell
-uv venv
-uv pip install -e ".[dev]"
-source .venv/bin/activate
-```
+1. Create a new virtual environment: `uv venv`
+2. Activate the environment:
+    * On Unix or MacOS: `source .venv/bin/activate`
+    * On Windows: `.venv\Scripts\activate.bat`
+3. Install the required packages: `uv pip install -e ".[dev]"`
 
 ### Docker setup
 
@@ -45,10 +45,42 @@ for you.
 Note: for the WSL subsystem on Windows you might need to adjust the path for the
 volume.
 
-### Codespaces
+## Model Context Protocol (MCP) Server
 
-The fastest way to get started is to use a GitHub Codespace. Just click on the
-button in the repository's main page.
+Serena's functionality is intended to be used via the model context protocol (MCP),
+which allows for easy integration into applications like Claude Desktop and IDEs.
+
+### Project Configuration File
+
+The first step is to create a `.yml` configuration file for the project you want
+Serena to work on:
+
+Copy `myproject.demo.yml` to `myproject.yml` and adjust the settings to your project.
+
+### MCP Server Configuration
+
+Tools like Claude Desktop need to be informed about the MCP server.
+They typically start the server themselves and only need to be informed how to start
+the server. 
+This is typically done by providing a configuration file in JSON format as follows,
+
+```json
+{
+    "mcpServers": {
+        "serena": {
+            "command": "uv",
+            "args": ["run", "--directory", "/path/to/serena", "python", "scripts/mcp_server.py", "myproject.yml"]
+        }
+    }
+}
+```
+
+where you must adjust `/path/to/serena` to the actual path of the Serena repository as well as the `.yml` file
+for your project configuration.
+On Windows, you can specify the path in the format `C:/path/to/serena`.
+
+**Claude Desktop**: The JSON configuration file can be found via File / Settings / Developer / Edit Config. 
+    Then edit the file named `claude_desktop_config.json`.
 
 ## Contributing
 

code_templates/prompt_factory_template.py

@@ -0,0 +1,21 @@
+from .multilang_prompt import MultiLangContainer, MultiLangPromptTemplateCollection, PromptList
+
+
+class PromptFactory:
+    # NOTE: This class is auto-generated by gen_prompt_factory.py
+
+    def __init__(self, lang_shortcode: str = "en", fallback_mode=MultiLangContainer.FallbackMode.EXCEPTION):
+        self.lang_shortcode = lang_shortcode
+        self.collection = MultiLangPromptTemplateCollection()
+        self.fallback_mode = fallback_mode
+
+    def _format_prompt(self, prompt_name: str, kwargs) -> str:
+        del kwargs["self"]
+        mpt = self.collection.get_multilang_prompt_template(prompt_name)
+        return mpt.get_item(self.lang_shortcode, self.fallback_mode).instantiate(**kwargs)
+
+    def _get_list(self, prompt_name: str) -> PromptList:
+        mpl = self.collection.get_multilang_prompt_list(prompt_name)
+        return mpl.get_item(self.lang_shortcode, self.fallback_mode)
+
+    # methods

myproject.demo.yml

@@ -0,0 +1,6 @@
+# absolute path to the project
+project_root: /path/to/project
+# language of the project (csharp, python, rust, java, typescript, javascript, go, or ruby)
+language: python
+# list of directories to ignore: either names (e.g. "temp") or relative paths (e.g. "build/foo")
+ignored_dirs: [".git", "temp"]

prompts/mcp_prompts.yml

@@ -0,0 +1,12 @@
+lang: en
+prompts:
+  onboarding_prompt: |
+    You are to assist in generating overview documentation for onboarding developers of the project.
+    Specifically, you are to create the file `{{ onboarding_file }}` in the project root directory.
+    The file should contain 
+      * a brief description of the project, including the project's purpose, 
+      * an overview of the structure of the project,
+      * instructions on how to add and execute new automated tests.
+    Use functions for reading the project's directory structure and individual files to accomplish this.
+    Read only the necessary files and directories to generate the documentation.
+    You are to provide the content for the file in markdown format.

pyproject.toml

@@ -24,6 +24,9 @@ dependencies = [
   "pyright>=1.1.396,<2",
   "overrides>=7.7.0,<8",
   "python-dotenv>=1.0.0, <2",
+  "mcp>=1.5.0",
+  "fastmcp>=0.4.1",
+  "sensai-utils>=1.4.0",
   "pydantic>=2.10.6",
 ]
 
@@ -227,7 +230,12 @@ ignore = [
   "F403",
   "F405",
   "C401",
-  "C901"
+  "C901",
+  "ASYNC230",
+  "ISC003",
+  "B024",
+  "B007",
+  "SIM102",
 ]
 unfixable = [
   "F841",

scripts/gen_prompt_factory.py

@@ -0,0 +1,43 @@
+from sensai.util import logging
+
+from serena.llm.multilang_prompt import MultiLangPromptTemplateCollection
+
+log = logging.getLogger(__name__)
+
+
+def main():
+    coll = MultiLangPromptTemplateCollection()
+
+    package = "src/serena"
+
+    # collect methods to generate
+    indent = "    "
+    methods = []
+    for mpt in coll.prompt_templates.values():
+        prompt_name = mpt.name
+        params = mpt.get_parameters()
+        if len(params) == 0:
+            params_str = ""
+        else:
+            params_str = ", *, " + ", ".join(params)
+        methods.append(
+            f"def create_{prompt_name}(self{params_str}) -> str:"
+            + f"\n{indent}{indent}return self._format_prompt('{prompt_name}', locals())\n\n{indent}"
+        )
+    for mpl in coll.prompt_lists.values():
+        prompt_name = mpl.name
+        methods.append(
+            f"def get_list_{prompt_name}(self) -> PromptList:" + f"\n{indent}{indent}return self._get_list('{prompt_name}')\n\n{indent}"
+        )
+
+    # write prompt factory with added methods
+    with open("code_templates/prompt_factory_template.py") as f:
+        code = f.read()
+    methods_str = "".join(methods)
+    code = code.replace("# methods", methods_str)
+    with open(f"{package}/llm/prompt_factory.py", "w") as f:
+        f.write(code)
+
+
+if __name__ == "__main__":
+    logging.run_main(main)

scripts/mcp_server.py

@@ -0,0 +1,200 @@
+"""
+The Serena Model Context Protocol (MCP) Server
+"""
+
+import json
+import os
+import sys
+from abc import ABC, abstractmethod
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from typing import Any, cast
+
+import yaml
+from mcp.server.fastmcp.prompts.base import Message, UserMessage
+from mcp.server.fastmcp.server import Context, FastMCP, Settings
+from sensai.util import logging
+
+from multilspy import SyncLanguageServer
+from multilspy.multilspy_config import Language, MultilspyConfig
+from multilspy.multilspy_logger import MultilspyLogger
+from serena.llm.prompt_factory import PromptFactory
+from serena.util.file_system import scan_directory
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class SerenaMCPRequestContext:
+    language_server: SyncLanguageServer
+    project_root: str
+    project_config: dict[str, Any]
+    prompt_factory: PromptFactory
+
+
+@asynccontextmanager
+async def server_lifespan(mcp_server: FastMCP) -> AsyncIterator[SerenaMCPRequestContext]:
+    """Manage server startup and shutdown lifecycle."""
+    argv = sys.argv[1:]
+    if len(argv) != 1:
+        print("\nUsage: mcp_server <.yml project file>", file=sys.stderr)
+        sys.exit(1)
+
+    project_file = argv[0]
+    if not os.path.exists(project_file):
+        print(f"Project file not found: {project_file}", file=sys.stderr)
+        sys.exit(1)
+
+    # read project configuration
+    with open(project_file, encoding="utf-8") as f:
+        project_config = yaml.safe_load(f)
+    language = Language(project_config["language"])
+    project_root = project_config["project_root"]
+
+    # create and start the language server instance
+    config = MultilspyConfig(code_language=language)
+    logger = MultilspyLogger()
+    language_server = SyncLanguageServer.create(config, logger, project_root)
+    language_server.start()
+    try:
+        yield SerenaMCPRequestContext(
+            language_server=language_server, project_root=project_root, project_config=project_config, prompt_factory=PromptFactory()
+        )
+    finally:
+        language_server.stop()
+
+
+mcp_settings = Settings(lifespan=server_lifespan)
+mcp = FastMCP(**mcp_settings.model_dump())
+
+
+class Component(ABC):
+    def __init__(self, ctx: Context):
+        lifespan_context = cast(SerenaMCPRequestContext, ctx.request_context.lifespan_context)
+        self.langsrv = lifespan_context.language_server
+        self.project_root = lifespan_context.project_root
+        self.project_config = lifespan_context.project_config
+        self.prompt_factory = lifespan_context.prompt_factory
+
+
+class Tool(Component):
+    def execute(self) -> str:
+        try:
+            return self._execute()
+        except Exception as e:
+            log.error(f"Error executing tool: {e}")
+            return f"Error executing tool: {e}"
+
+    @abstractmethod
+    def _execute(self) -> str:
+        pass
+
+
+class SimplePrompt(Component):
+    def __init__(self, ctx: Context):
+        super().__init__(ctx)
+        self.messages = []
+
+    def create(self) -> str:
+        return self._create_prompt()
+
+    @abstractmethod
+    def _create_prompt(self) -> str:
+        pass
+
+
+class SequentialPrompt(Component):
+    def __init__(self, ctx: Context):
+        super().__init__(ctx)
+        self.messages = []
+
+    def create(self) -> list[Message]:
+        self._add_messages()
+        return self.messages
+
+    @abstractmethod
+    def _add_messages(self) -> None:
+        pass
+
+    def _add_user_message(self, msg: str) -> None:
+        self.messages.append(UserMessage(content=msg))
+
+
+@mcp.tool()
+def read_file(ctx: Context, relative_path: str) -> str:
+    """
+    :param ctx: the context object, which will be created and provided automatically
+    :param relative_path: the relative path to the file to read
+    :return: the full text of the file at the given relative path
+    """
+    log.info(f"read_file: {relative_path=}")
+
+    class ReadFileTool(Tool):
+        def _execute(self):
+            with self.langsrv.open_file(relative_path):
+                return self.langsrv.get_open_file_text(relative_path)
+
+    return ReadFileTool(ctx).execute()
+
+
+@mcp.tool()
+def create_text_file(ctx: Context, relative_path: str, content: str) -> str:
+    """
+    :param ctx: the context object, which will be created and provided automatically
+    :param relative_path: the relative path to the file to create
+    :param content: the (utf-8-encoded) content to write to the file
+    :return: a message indicating success or failure
+    """
+    log.info(f"create_file: {relative_path=}")
+
+    class CreateFileTool(Tool):
+        def _execute(self):
+            absolute_path = os.path.join(self.project_root, relative_path)
+            with open(absolute_path, "w", encoding="utf-8") as f:
+                f.write(content)
+            return f"File created: {relative_path}"
+
+    return CreateFileTool(ctx).execute()
+
+
+@mcp.tool()
+def list_dir(ctx: Context, relative_path: str, recursive: bool) -> str:
+    """
+    :param ctx: the context object, which will be created and provided automatically
+    :param relative_path: the relative path to the directory to list; pass "." to scan the project root
+    :param recursive: whether to scan subdirectories recursively
+    :return: a JSON object with the names of directories and files within the given directory
+    """
+    log.info(f"list_dir: {relative_path=}")
+
+    class ListDirTool(Tool):
+        def _execute(self):
+            dirs, files = scan_directory(
+                os.path.join(self.project_root, relative_path), recursive=recursive, ignored_dirs=self.project_config["ignored_dirs"]
+            )
+            return json.dumps({"dirs": dirs, "files": files})
+
+    return ListDirTool(ctx).execute()
+
+
+@mcp.tool()
+def onboarding(ctx: Context) -> str:
+    """
+    :param ctx: the context object, which will be created and provided automatically
+    :return: instructions on how to create the onboarding information
+    """
+    onboarding_file = "serena_onboarding.md"
+
+    class OnboardingPrompt(SimplePrompt):
+        def _create_prompt(self):
+            return self.prompt_factory.create_onboarding_prompt(onboarding_file=onboarding_file)
+
+    return OnboardingPrompt(ctx).create()
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.DEBUG, stream=sys.stderr, format="%(levelname)-5s %(asctime)-15s %(name)s:%(funcName)s:%(lineno)d - %(message)s"
+    )
+    mcp.run()

src/serena/llm/jinja_template.py

@@ -0,0 +1,30 @@
+import jinja2
+import jinja2.meta
+import jinja2.nodes
+import jinja2.visitor
+
+from serena.util.class_decorators import singleton
+
+
+@singleton
+class JinjaEnvProvider:
+    def __init__(self):
+        self._env = None
+
+    def get_env(self):
+        if self._env is None:
+            self._env = jinja2.Environment()
+        return self._env
+
+
+class JinjaTemplate:
+    def __init__(self, template_string: str):
+        self._template_string = template_string
+        self._template = JinjaEnvProvider().get_env().from_string(self._template_string)
+
+    def render(self, **kwargs) -> str:
+        return self._template.render(**kwargs)
+
+    def get_parameters(self) -> set[str]:
+        parsed_content = self._template.environment.parse(self._template_string)
+        return jinja2.meta.find_undeclared_variables(parsed_content)

src/serena/llm/multilang_prompt.py

@@ -0,0 +1,203 @@
+import os
+from collections.abc import Callable
+from enum import Enum
+from typing import Any, Generic, TypeVar
+
+import yaml
+from sensai.util.string import ToStringMixin
+
+from .jinja_template import JinjaTemplate
+
+LANG_CODES = ["en", "de"]
+
+
+class PromptTemplate(ToStringMixin):
+    def __init__(self, name: str, jinja_template_string: str):
+        self.name = name
+        self.jinja_template = JinjaTemplate(jinja_template_string.strip())
+        self.parameters = self.jinja_template.get_parameters()
+
+    def _tostring_excludes(self) -> list[str]:
+        return ["jinja_template"]
+
+    def instantiate(self, **kwargs) -> str:
+        return self.jinja_template.render(**kwargs)
+
+
+class PromptList:
+    def __init__(self, items: list[str]):
+        self.items = [x.strip() for x in items]
+
+    def to_string(self):
+        bullet = " * "
+        indent = " " * len(bullet)
+        items = [x.replace("\n", "\n" + indent) for x in self.items]
+        return "\n * ".join(items)
+
+
+T = TypeVar("T")
+
+
+class MultiLangContainer(Generic[T], ToStringMixin):
+    """
+    Represents a container of items which are associated with different languages
+    """
+
+    def __init__(self, name: str):
+        self.name = name
+        self.lang2item: dict[str, T] = {}
+
+    def _tostring_excludes(self) -> list[str]:
+        return ["lang2item"]
+
+    def _tostring_additional_entries(self) -> dict[str, Any]:
+        return dict(languages=list(self.lang2item.keys()))
+
+    class FallbackMode(Enum):
+        ANY = "any"
+        """
+        If the requested language is not found, return the item for any language
+        """
+        EXCEPTION = "exception"
+        """
+        If the requested language is not found, raise an exception
+        """
+
+    def add_item(self, item: T, lang: str = ""):
+        self.lang2item[lang] = item
+
+    def get_item(self, lang: str, fallback_mode: FallbackMode = FallbackMode.EXCEPTION) -> T:
+        """
+        Gets the item for the given language.
+
+        :param lang: the language shortcode for which to obtain the prompt template
+        :param fallback_mode: defines what to do if there is no item for the given language
+        :return: the item
+        """
+        item = self.lang2item.get(lang)
+        if item is not None:
+            return item
+        else:
+            if fallback_mode == self.FallbackMode.EXCEPTION:
+                raise KeyError(f"Item for language '{lang}' not found for name '{self.name}'")
+            if fallback_mode == self.FallbackMode.ANY:
+                return next(iter(self.lang2item.values()))
+            else:
+                raise ValueError(f"Invalid fallback mode '{fallback_mode}'")
+
+
+class MultiLangPromptTemplate(MultiLangContainer[PromptTemplate]):
+    """
+    Represents a prompt template which potentially supports multiple languages.
+    """
+
+    def get_parameters(self) -> list[str]:
+        """
+        Gets the parameters of the prompt template (which must be the same for all languages).
+
+        :return: the list of parameters required to instantiate the template
+        """
+        prev_params = None
+        for template in self.lang2item.values():
+            params = set(template.parameters)
+            if prev_params is not None:
+                assert (
+                    params == prev_params
+                ), f"Parameters of MLPT '{self.name}' are inconsistent: {sorted(params)} vs {sorted(prev_params)}"
+            prev_params = params
+        return sorted(prev_params)
+
+
+class MultiLangPromptList(MultiLangContainer[PromptList]):
+    pass
+
+
+class MultiLangPromptTemplateCollection:
+    """
+    Represents a collection of multi-language prompts, which are read from the prompts/ directory.
+    The language of a prompt is determined by suffix in the prompt name, e.g. "_de" for German.
+    For yaml files, the language can also be set globally for all prompts within it.
+
+    The prompts/ folder may contain:
+        - .txt files with a single prompt template (filename defines prompt name, content is the prompt template)
+        - .yml files with multiple prompt templates under the top-level key 'prompts'.
+          Each key under 'prompts' defines a prompt name and the corresponding value is the prompt template.
+          The value can also be a list of prompts instead of a single template.
+          The language of all can be set by specifying the key 'lang' in addition to 'prompts'.
+    """
+
+    def __init__(self):
+        self.prompt_templates: dict[str, MultiLangPromptTemplate] = {}
+        self.prompt_lists: dict[str, MultiLangPromptList] = {}
+        prompts_dir = self._prompt_template_folder()
+        self._read_prompt_templates(prompts_dir)
+
+    @classmethod
+    def _prompt_template_folder(cls) -> str:
+        dir_path = os.path.dirname(os.path.realpath(__file__))
+        prompts_dir = None
+        for i in range(5):
+            dir_path = os.path.join(dir_path, "..")
+            prompts_dir = os.path.join(dir_path, "prompts")
+            if os.path.isdir(prompts_dir):
+                break
+        if not os.path.isdir(prompts_dir):
+            raise FileNotFoundError("Could not find the 'prompts' directory")
+        return prompts_dir
+
+    @staticmethod
+    def _container_lang(prompt_name: str, collection: dict[str, T], container_factory: Callable[[str], T]) -> tuple[T, str]:
+        lang = ""
+        if prompt_name[-2:] in LANG_CODES and prompt_name[-3] == "_":
+            lang = prompt_name[-2:]
+            prompt_name = prompt_name[:-3]
+
+        container = collection.get(prompt_name)
+        if container is None:
+            container = container_factory(prompt_name)
+            collection[prompt_name] = container
+
+        return container, lang
+
+    def _add_prompt_template(self, prompt_name: str, jinja_prompt_template: str):
+        """
+        :param prompt_name: a prompt name, which may have a language shortcode suffix (e.g. "_de")
+        :param jinja_prompt_template: the actual prompt string which may contain placeholders/parameters (e.g. "{name}")
+        """
+        multilang_prompt_template, lang = self._container_lang(prompt_name, self.prompt_templates, MultiLangPromptTemplate)
+        multilang_prompt_template.add_item(PromptTemplate(prompt_name, jinja_prompt_template), lang=lang)
+
+    def _add_prompt_list(self, prompt_name: str, prompt_list: list[str]):
+        """
+        :param prompt_name: a prompt name, which may have a language shortcode suffix (e.g. "_de")
+        :param prompt_list: a list of prompts
+        """
+        multilang_prompt_list, lang = self._container_lang(prompt_name, self.prompt_lists, MultiLangPromptList)
+        multilang_prompt_list.add_item(PromptList(prompt_list), lang=lang)
+
+    def _read_prompt_templates(self, prompts_dir: str):
+        for fn in os.listdir(prompts_dir):
+            path = os.path.join(prompts_dir, fn)
+            if fn.endswith(".txt"):
+                with open(path) as file:
+                    prompt_template_string = file.read()
+                self._add_prompt_template(fn[:-4], prompt_template_string)
+            elif fn.endswith(".yml"):
+                with open(path, encoding="utf-8") as f:
+                    data = yaml.safe_load(f)
+                    assert "prompts" in data, f"YAML file {fn} must contain key 'prompts'"
+                    lang_suffix = ("_" + data["lang"]) if "lang" in data else ""
+                    for prompt_name, prompt in data["prompts"].items():
+                        prompt_name += lang_suffix
+                        if isinstance(prompt, list):
+                            self._add_prompt_list(prompt_name, prompt)
+                        elif isinstance(prompt, str):
+                            self._add_prompt_template(prompt_name, prompt)
+                        else:
+                            raise ValueError(f"Invalid prompt type: {prompt}")
+
+    def get_multilang_prompt_template(self, prompt_name: str) -> MultiLangPromptTemplate:
+        return self.prompt_templates[prompt_name]
+
+    def get_multilang_prompt_list(self, prompt_name: str) -> MultiLangPromptList:
+        return self.prompt_lists[prompt_name]

src/serena/llm/prompt_factory.py

@@ -0,0 +1,22 @@
+from .multilang_prompt import MultiLangContainer, MultiLangPromptTemplateCollection, PromptList
+
+
+class PromptFactory:
+    # NOTE: This class is auto-generated by gen_prompt_factory.py
+
+    def __init__(self, lang_shortcode: str = "en", fallback_mode=MultiLangContainer.FallbackMode.EXCEPTION):
+        self.lang_shortcode = lang_shortcode
+        self.collection = MultiLangPromptTemplateCollection()
+        self.fallback_mode = fallback_mode
+
+    def _format_prompt(self, prompt_name: str, kwargs) -> str:
+        del kwargs["self"]
+        mpt = self.collection.get_multilang_prompt_template(prompt_name)
+        return mpt.get_item(self.lang_shortcode, self.fallback_mode).instantiate(**kwargs)
+
+    def _get_list(self, prompt_name: str) -> PromptList:
+        mpl = self.collection.get_multilang_prompt_list(prompt_name)
+        return mpl.get_item(self.lang_shortcode, self.fallback_mode)
+
+    def create_onboarding_prompt(self, *, onboarding_file) -> str:
+        return self._format_prompt("onboarding_prompt", locals())

src/serena/util/class_decorators.py

@@ -0,0 +1,10 @@
+def singleton(cls):
+    instance = None
+
+    def get_instance(*args, **kwargs):
+        nonlocal instance
+        if instance is None:
+            instance = cls(*args, **kwargs)
+        return instance
+
+    return get_instance

src/serena/util/file_system.py

@@ -0,0 +1,66 @@
+import os
+from collections.abc import Sequence
+
+
+def scan_directory(
+    path: str,
+    recursive=False,
+    relative_to: str | None = None,
+    ignored_dirs: Sequence[str] = (),
+    ignored_files: Sequence[str] = (),
+) -> tuple[list[str], list[str]]:
+    """
+    :param path: the path to scan
+    :param recursive: whether to recursively scan subdirectories
+    :param relative_to: the path to which the results should be relative to; if None, provide absolute paths
+    :param ignored_dirs: a list of directory names or relative paths to ignore
+    :param ignored_files: a list of file names or relative paths to ignore
+    :return: the list of directories and files
+    """
+    files = []
+    directories = []
+
+    abs_path = os.path.abspath(path)
+    rel_base = os.path.abspath(relative_to) if relative_to else None
+
+    # Helper function to check if an item should be ignored
+    def is_ignored(entry_path, ignored_items):
+        entry_name = os.path.basename(entry_path)
+
+        # Check if name is directly in ignored list
+        if entry_name in ignored_items:
+            return True
+
+        # Check if relative path matches any ignored path
+        if rel_base:
+            rel_path = os.path.relpath(entry_path, rel_base)
+            for item in ignored_items:
+                if rel_path == item or rel_path.startswith(f"{item}/"):
+                    return True
+
+        return False
+
+    with os.scandir(abs_path) as entries:
+        for entry in entries:
+            entry_path = entry.path
+
+            if rel_base:
+                result_path = os.path.relpath(entry_path, rel_base)
+            else:
+                result_path = entry_path
+
+            if entry.is_file():
+                if not is_ignored(entry_path, ignored_files):
+                    files.append(result_path)
+            elif entry.is_dir():
+                if not is_ignored(entry_path, ignored_dirs):
+                    directories.append(result_path)
+                    if recursive:
+                        sub_dirs, sub_files = scan_directory(
+                            entry_path, recursive=True, relative_to=relative_to, ignored_dirs=ignored_dirs, ignored_files=ignored_files
+                        )
+                        files.extend(sub_files)
+                        directories.extend(sub_dirs)
+
+    # Note: I've swapped the return order to match your function signature
+    return directories, files

test/serena/test_text_utils.py

@@ -1,6 +1,7 @@
 import re
 
 import pytest
+
 from serena.text_utils import LineType, search_text
 
 

uv.lock

@@ -489,6 +489,23 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/90/2b/0817a2b257fe88725c25589d89aec060581aabf668707a8d03b2e9e0cb2a/fastjsonschema-2.21.1-py3-none-any.whl", hash = "sha256:c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667", size = 23924 },
 ]
 
+[[package]]
+name = "fastmcp"
+version = "0.4.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "httpx" },
+    { name = "mcp" },
+    { name = "pydantic" },
+    { name = "pydantic-settings" },
+    { name = "python-dotenv" },
+    { name = "typer" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/6f/84/17b549133263d7ee77141970769bbc401525526bf1af043ea6842bce1a55/fastmcp-0.4.1.tar.gz", hash = "sha256:713ad3b8e4e04841c9e2f3ca022b053adb89a286ceffad0d69ae7b56f31cbe64", size = 785575 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/79/0b/008a340435fe8f0879e9d608f48af2737ad48440e09bd33b83b3fd03798b/fastmcp-0.4.1-py3-none-any.whl", hash = "sha256:664b42c376fb89ec90a50c9433f5a1f4d24f36696d6c41b024b427ae545f9619", size = 35282 },
+]
+
 [[package]]
 name = "filelock"
 version = "3.18.0"
@@ -574,6 +591,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 },
 ]
 
+[[package]]
+name = "httpx-sse"
+version = "0.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/4c/60/8f4281fa9bbf3c8034fd54c0e7412e66edbab6bc74c4996bd616f8d0406e/httpx-sse-0.4.0.tar.gz", hash = "sha256:1e81a3a3070ce322add1d3529ed42eb5f70817f45ed6ec915ab753f961139721", size = 12624 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e1/9b/a181f281f65d776426002f330c31849b86b31fc9d848db62e16f03ff739f/httpx_sse-0.4.0-py3-none-any.whl", hash = "sha256:f329af6eae57eaa2bdfd962b42524764af68075ea87370a2de920af5341e318f", size = 7819 },
+]
+
 [[package]]
 name = "identify"
 version = "2.6.9"
@@ -1100,6 +1126,25 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899 },
 ]
 
+[[package]]
+name = "mcp"
+version = "1.5.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "httpx" },
+    { name = "httpx-sse" },
+    { name = "pydantic" },
+    { name = "pydantic-settings" },
+    { name = "sse-starlette" },
+    { name = "starlette" },
+    { name = "uvicorn" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/6d/c9/c55764824e893fdebe777ac7223200986a275c3191dba9169f8eb6d7c978/mcp-1.5.0.tar.gz", hash = "sha256:5b2766c05e68e01a2034875e250139839498c61792163a7b221fc170c12f5aa9", size = 159128 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c1/d1/3ff566ecf322077d861f1a68a1ff025cad337417bd66ad22a7c6f7dfcfaf/mcp-1.5.0-py3-none-any.whl", hash = "sha256:51c3f35ce93cb702f7513c12406bbea9665ef75a08db909200b07da9db641527", size = 73734 },
+]
+
 [[package]]
 name = "mdit-py-plugins"
 version = "0.4.2"
@@ -1226,11 +1271,11 @@ wheels = [
 
 [[package]]
 name = "narwhals"
-version = "1.31.0"
+version = "1.32.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/36/fa/c2b6a4d5dbc4af15aa58c86920d5275a9c65142318179b246685069f57da/narwhals-1.31.0.tar.gz", hash = "sha256:333472e2562343dfdd27407ec9b5114a07c81d0416794e4ac6b703dd925c6a1a", size = 253463 }
+sdist = { url = "https://files.pythonhosted.org/packages/c1/e5/aa97891440bf6bc4239e9b918d89fea58eb87dff1d8d14a69b45ef677e66/narwhals-1.32.0.tar.gz", hash = "sha256:bd0aa41434737adb4b26f8593f3559abc7d938730ece010fe727b58bc363580d", size = 258915 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f9/c0/fb39bd876ea2fd9509343d643690cd2f9715e6a77271e7c7b26f1eea70c1/narwhals-1.31.0-py3-none-any.whl", hash = "sha256:2a7b79bb5f511055c4c0142121fc0d4171ea171458e12d44dbd9c8fc6488e997", size = 313124 },
+    { url = "https://files.pythonhosted.org/packages/e9/96/79a6168dc7a5098066e097c01a45d01608c8df6552dfb92a2676ce623186/narwhals-1.32.0-py3-none-any.whl", hash = "sha256:8bdbf3f76155887412eea04b0b06303856ac1aa3d9e8bda5b5e54612855fa560", size = 320073 },
 ]
 
 [[package]]
@@ -1673,6 +1718,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/10/1c/16b3a3e3398fd29dca77cea0a1d998d6bde3902fa2706985191e2313cc76/pydantic_core-2.27.2-cp311-cp311-win_arm64.whl", hash = "sha256:26f0d68d4b235a2bae0c3fc585c585b4ecc51382db0e3ba402a22cbc440915e4", size = 1867102 },
 ]
 
+[[package]]
+name = "pydantic-settings"
+version = "2.8.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pydantic" },
+    { name = "python-dotenv" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/88/82/c79424d7d8c29b994fb01d277da57b0a9b09cc03c3ff875f9bd8a86b2145/pydantic_settings-2.8.1.tar.gz", hash = "sha256:d5c663dfbe9db9d5e1c646b2e161da12f0d734d422ee56f567d0ea2cee4e8585", size = 83550 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0b/53/a64f03044927dc47aafe029c42a5b7aabc38dfb813475e0e1bf71c4a59d0/pydantic_settings-2.8.1-py3-none-any.whl", hash = "sha256:81942d5ac3d905f7f3ee1a70df5dfb62d5569c12f51a5a647defc1c3d9ee2e9c", size = 30839 },
+]
+
 [[package]]
 name = "pydata-sphinx-theme"
 version = "0.15.4"
@@ -1905,6 +1963,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9", size = 4242 },
 ]
 
+[[package]]
+name = "rich"
+version = "13.9.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "markdown-it-py" },
+    { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ab/3a/0316b28d0761c6734d6bc14e770d85506c986c85ffb239e688eeaab2c2bc/rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098", size = 223149 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", size = 242424 },
+]
+
 [[package]]
 name = "rpds-py"
 version = "0.23.1"
@@ -1989,11 +2060,25 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/40/b0/4562db6223154aa4e22f939003cb92514c79f3d4dccca3444253fd17f902/Send2Trash-1.8.3-py3-none-any.whl", hash = "sha256:0c31227e0bd08961c7665474a3d1ef7193929fedda4233843689baa056be46c9", size = 18072 },
 ]
 
+[[package]]
+name = "sensai-utils"
+version = "1.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e9/b1/9dba08e2d8ea739b58d1ac393c066169f1ca6506c86f0322d6e56e600ae2/sensai_utils-1.4.0.tar.gz", hash = "sha256:2d32bdcc91fd1428c5cae0181e98623142d2d5f7e115e23d585a842dd9dc59ba", size = 56317 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c6/df/49cc942b653427df0bbd3b9a2fb1e85652df3d4c1d9f6dcbfce1b6155716/sensai_utils-1.4.0-py3-none-any.whl", hash = "sha256:ed6fc57552620e43b33cf364ea0bc0fd7df39391069dd7b621b113ef55547507", size = 63214 },
+]
+
 [[package]]
 name = "serena"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [
+    { name = "fastmcp" },
+    { name = "mcp" },
     { name = "overrides" },
     { name = "pandas" },
     { name = "plotly" },
@@ -2001,6 +2086,7 @@ dependencies = [
     { name = "pyright" },
     { name = "python-dotenv" },
     { name = "requests" },
+    { name = "sensai-utils" },
 ]
 
 [package.optional-dependencies]
@@ -2034,9 +2120,11 @@ dev = [
 requires-dist = [
     { name = "black", extras = ["jupyter"], marker = "extra == 'dev'", specifier = ">=23.7.0" },
     { name = "docutils", marker = "extra == 'dev'", specifier = ">=0.20.1" },
+    { name = "fastmcp", specifier = ">=0.4.1" },
     { name = "jinja2", marker = "extra == 'dev'" },
     { name = "jupyter", marker = "extra == 'dev'", specifier = ">=1.0.0" },
     { name = "jupyter-book", marker = "extra == 'dev'", specifier = "==1.0.0" },
+    { name = "mcp", specifier = ">=1.5.0" },
     { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.4.1" },
     { name = "nbqa", marker = "extra == 'dev'", specifier = ">=1.7.1" },
     { name = "nbstripout", marker = "extra == 'dev'", specifier = ">=0.6.1" },
@@ -2052,6 +2140,7 @@ requires-dist = [
     { name = "python-dotenv", specifier = ">=1.0.0,<2" },
     { name = "requests", specifier = ">=2.32.3,<3" },
     { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.0.285" },
+    { name = "sensai-utils", specifier = ">=1.4.0" },
     { name = "sphinx", marker = "extra == 'dev'", specifier = ">=7" },
     { name = "sphinx-book-theme", marker = "extra == 'dev'", specifier = ">=1.0.1" },
     { name = "sphinx-comments", marker = "extra == 'dev'", specifier = ">=0.0.3" },
@@ -2067,11 +2156,20 @@ provides-extras = ["dev"]
 
 [[package]]
 name = "setuptools"
-version = "77.0.3"
+version = "78.0.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/81/ed/7101d53811fd359333583330ff976e5177c5e871ca8b909d1d6c30553aa3/setuptools-77.0.3.tar.gz", hash = "sha256:583b361c8da8de57403743e756609670de6fb2345920e36dc5c2d914c319c945", size = 1367236 }
+sdist = { url = "https://files.pythonhosted.org/packages/47/42/55a8f24bd1287676b23e56a6d94e416be390ca6e0ee30fa46a782d038f80/setuptools-78.0.1.tar.gz", hash = "sha256:4321d2dc2157b976dee03e1037c9f2bc5fea503c0c47d3c9458e0e8e49e659ce", size = 1367415 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a9/07/99f2cefae815c66eb23148f15d79ec055429c38fa8986edcc712ab5f3223/setuptools-77.0.3-py3-none-any.whl", hash = "sha256:67122e78221da5cf550ddd04cf8742c8fe12094483749a792d56cd669d6cf58c", size = 1255678 },
+    { url = "https://files.pythonhosted.org/packages/42/c8/3faed884acdb2c1f2eb353cbacdd1ee4943de89a199d1f622ebefb6170e5/setuptools-78.0.1-py3-none-any.whl", hash = "sha256:1cc9b32ee94f93224d6c80193cbb768004667aa2f2732a473d6949b0236c1d4e", size = 1255630 },
+]
+
+[[package]]
+name = "shellingham"
+version = "1.5.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 },
 ]
 
 [[package]]
@@ -2437,6 +2535,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7b/0f/d69904cb7d17e65c65713303a244ec91fd3c96677baf1d6331457fd47e16/sqlalchemy-2.0.39-py3-none-any.whl", hash = "sha256:a1c6b0a5e3e326a466d809b651c63f278b1256146a377a528b6938a279da334f", size = 1898621 },
 ]
 
+[[package]]
+name = "sse-starlette"
+version = "2.2.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "starlette" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/71/a4/80d2a11af59fe75b48230846989e93979c892d3a20016b42bb44edb9e398/sse_starlette-2.2.1.tar.gz", hash = "sha256:54470d5f19274aeed6b2d473430b08b4b379ea851d953b11d7f1c4a2c118b419", size = 17376 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d9/e0/5b8bd393f27f4a62461c5cf2479c75a2cc2ffa330976f9f00f5f6e4f50eb/sse_starlette-2.2.1-py3-none-any.whl", hash = "sha256:6410a3d3ba0c89e7675d4c273a301d64649c03a5ef1ca101f10b47f895fd0e99", size = 10120 },
+]
+
 [[package]]
 name = "stack-data"
 version = "0.6.3"
@@ -2451,6 +2562,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521 },
 ]
 
+[[package]]
+name = "starlette"
+version = "0.46.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/04/1b/52b27f2e13ceedc79a908e29eac426a63465a1a01248e5f24aa36a62aeb3/starlette-0.46.1.tar.gz", hash = "sha256:3c88d58ee4bd1bb807c0d1acb381838afc7752f9ddaec81bbe4383611d833230", size = 2580102 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a0/4b/528ccf7a982216885a1ff4908e886b8fb5f19862d1962f56a3fce2435a70/starlette-0.46.1-py3-none-any.whl", hash = "sha256:77c74ed9d2720138b25875133f3a2dae6d854af2ec37dceb56aef370c1d8a227", size = 71995 },
+]
+
 [[package]]
 name = "tabulate"
 version = "0.9.0"
@@ -2562,6 +2685,21 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359 },
 ]
 
+[[package]]
+name = "typer"
+version = "0.15.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "rich" },
+    { name = "shellingham" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/6f/3991f0f1c7fcb2df31aef28e0594d8d54b05393a0e4e34c65e475c2a5d41/typer-0.15.2.tar.gz", hash = "sha256:ab2fab47533a813c49fe1f16b1a370fd5819099c00b119e0633df65f22144ba5", size = 100711 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7f/fc/5b29fea8cee020515ca82cc68e3b8e1e34bb19a3535ad854cac9257b414c/typer-0.15.2-py3-none-any.whl", hash = "sha256:46a499c6107d645a9c13f7ee46c5d5096cae6f5fc57dd11eccbbb9ae3e44ddfc", size = 45061 },
+]
+
 [[package]]
 name = "types-python-dateutil"
 version = "2.9.0.20241206"
@@ -2616,6 +2754,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", size = 128369 },
 ]
 
+[[package]]
+name = "uvicorn"
+version = "0.34.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/4b/4d/938bd85e5bf2edeec766267a5015ad969730bb91e31b44021dfe8b22df6c/uvicorn-0.34.0.tar.gz", hash = "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9", size = 76568 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/61/14/33a3a1352cfa71812a3a21e8c9bfb83f60b0011f5e36f2b1399d51928209/uvicorn-0.34.0-py3-none-any.whl", hash = "sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4", size = 62315 },
+]
+
 [[package]]
 name = "virtualenv"
 version = "20.29.3"