From 437bddd7a47390021d9985644249443c2bf1691c Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Tue, 12 May 2026 17:11:43 +0200 Subject: [PATCH 01/20] Feature: handle renaming of references to memories --- src/serena/project.py | 50 ++++++++- src/serena/tools/memory_tools.py | 48 +++++---- test/serena/test_memories_manager.py | 156 +++++++++++++++++++++++++++ 3 files changed, 231 insertions(+), 23 deletions(-) create mode 100644 test/serena/test_memories_manager.py diff --git a/src/serena/project.py b/src/serena/project.py index ae062420..54c621d3 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -33,6 +33,7 @@ log = logging.getLogger(__name__) class MemoriesManager: GLOBAL_TOPIC = "global" _global_memory_dir = SerenaPaths().global_memories_path + _MEMORY_REF_PREFIX = "mem:" def __init__( self, @@ -77,9 +78,48 @@ class MemoriesManager: def _is_global(self, name: str) -> bool: return name == self.GLOBAL_TOPIC or name.startswith(self.GLOBAL_TOPIC + "/") + @classmethod + def _prepare_name(cls, name: str) -> str: + """Corrects the name for common mistakes made by LLMs (``mem:`` prefix, ``.md`` suffix, OS-specific separators).""" + name = name.removeprefix(cls._MEMORY_REF_PREFIX) + if name.endswith(".md"): + name = name[:-3] + return name.replace(os.sep, "/") + + @classmethod + def _add_reference_prefix(cls, name: str) -> str: + name = cls._prepare_name(name) + return cls._MEMORY_REF_PREFIX + name + + def rename_references_to_memory(self, content: str, old_name: str, new_name: str) -> tuple[str, int]: + r""" + Replaces all occurrences of a memory reference (e.g. ``mem:foo``) in ``content`` with + the reference to ``new_name``. + + Matches only references whose name is exactly ``old_name``: the match must not be + embedded in a longer memory name. A memory name consists of the character class + ``[A-Za-z0-9_\\-/]`` (alphanumerics, underscore, hyphen, slash for topic separation), + which determines the boundary of the match. The surrounding delimiters (backticks, + quotes, parentheses, whitespace, etc.) are intentionally unconstrained. + + :param content: the text to search through + :param old_name: the memory name being renamed away from (without the ``mem:`` prefix) + :param new_name: the memory name being renamed to (without the ``mem:`` prefix) + :return: a tuple of (updated content, number of replacements made) + """ + # define the character class that constitutes a memory name; matches inside such a run are not real references + name_char = r"[A-Za-z0-9_\-/]" + ref_old = self._add_reference_prefix(old_name) + ref_new = self._add_reference_prefix(new_name) + + # build a pattern that anchors the reference on both sides so it cannot be embedded inside a longer name + pattern = rf"(? Path: - # Strip .md extension if present - name = name.replace(".md", "").replace(os.sep, "/") + name = self._prepare_name(name) parts = name.split("/") if ".." in parts: raise ValueError(f"Memory name cannot contain '..' segments for security reasons. Got: {name}") @@ -118,6 +158,7 @@ class MemoriesManager: raise PermissionError(f"Attempted to write to read_only memory: '{name}')") def load_memory(self, name: str) -> str: + name = self._prepare_name(name) self._check_not_ignored(name) memory_file_path = self.get_memory_file_path(name) if not memory_file_path.exists(): @@ -126,6 +167,7 @@ class MemoriesManager: return f.read() def save_memory(self, name: str, content: str, is_tool_context: bool) -> str: + name = self._prepare_name(name) self._check_not_ignored(name) self._check_write_access(name, is_tool_context) memory_file_path = self.get_memory_file_path(name) @@ -208,6 +250,7 @@ class MemoriesManager: return memories def delete_memory(self, name: str, is_tool_context: bool) -> str: + name = self._prepare_name(name) self._check_not_ignored(name) self._check_write_access(name, is_tool_context) memory_file_path = self.get_memory_file_path(name) @@ -221,6 +264,8 @@ class MemoriesManager: Rename or move a memory file. Moving between global and project scope (e.g. "global/foo" -> "bar") is supported. """ + old_name = self._prepare_name(old_name) + new_name = self._prepare_name(new_name) self._check_not_ignored(old_name) self._check_not_ignored(new_name) self._check_write_access(new_name, is_tool_context) @@ -250,6 +295,7 @@ class MemoriesManager: :param mode: "literal" or "regex" :param allow_multiple_occurrences: """ + name = self._prepare_name(name) self._check_not_ignored(name) self._check_write_access(name, is_tool_context) memory_file_path = self.get_memory_file_path(name) diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index 9ecb0c0c..0fd98ec4 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -1,7 +1,10 @@ +import logging from typing import Literal from serena.tools import Tool, ToolMarkerCanEdit +log = logging.getLogger(__name__) + class WriteMemoryTool(Tool, ToolMarkerCanEdit): """ @@ -11,13 +14,15 @@ class WriteMemoryTool(Tool, ToolMarkerCanEdit): def apply(self, memory_name: str, content: str, max_chars: int = -1) -> str: """ - Write information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format. - The memory name should be meaningful and can include "/" to organize into topics (e.g., "auth/login/logic"). - If explicitly instructed, use the "global/" prefix for writing a memory that is shared across projects - (e.g., "global/java/style_guide") + Write information about this project that can be useful for future tasks in md format. + The name should be meaningful and can include "/" to organize into topics. + If explicitly instructed, use the "global/" prefix for writing a memory that is shared across projects. + References to other memories should be inside backticks and prefixed with mem:, + e.g., `mem:auth`. - :param max_chars: the maximum number of characters to write. By default, determined by the config, - change only if instructed to do so. + :param memory_name: memory name + :param content: memory content, utf8-encoded + :param max_chars: see other tools """ # NOTE: utf-8 encoding is configured in the MemoriesManager if max_chars == -1: @@ -32,23 +37,19 @@ class WriteMemoryTool(Tool, ToolMarkerCanEdit): class ReadMemoryTool(Tool): """ - Read the content of a memory file. This tool should only be used if the information - is relevant to the current task. You can infer whether the information - is relevant from the memory file name. - You should not read the same memory file multiple times in the same conversation. + Reads the content of a memory file. """ def apply(self, memory_name: str) -> str: """ - Reads the contents of a memory. Should only be used if the information - is likely to be relevant to the current task, inferring relevance from the memory name. + Use to read a memory that is likely to be relevant to the current task, inferring relevance e.g. from the name. """ return self.memories_manager.load_memory(memory_name) class ListMemoriesTool(Tool): """ - List available memories. Any memory can be read using the `read_memory` tool. + Lists available memories. """ def apply(self, topic: str = "") -> str: @@ -60,9 +61,7 @@ class ListMemoriesTool(Tool): class DeleteMemoryTool(Tool, ToolMarkerCanEdit): """ - Delete a memory file. Should only happen if a user asks for it explicitly, - for example by saying that the information retrieved from a memory file is no longer correct - or no longer relevant for the project. + Delete a memory file. """ def apply(self, memory_name: str) -> str: @@ -74,16 +73,23 @@ class DeleteMemoryTool(Tool, ToolMarkerCanEdit): class RenameMemoryTool(Tool, ToolMarkerCanEdit): """ - Renames or moves a memory. Moving between project and global scope is supported - (e.g., renaming "global/foo" to "bar" moves it from global to project scope). + Renames or moves a memory, updating references that are marked with the `mem:` prefix. """ def apply(self, old_name: str, new_name: str) -> str: """ Rename or move a memory, use "/" in the name to organize into topics. The "global" topic should only be used if explicitly instructed. + References to other memories that are marked with the `mem:` prefix will be updated accordingly. """ - return self.memories_manager.move_memory(old_name, new_name, is_tool_context=True) + renaming_message = self.memories_manager.move_memory(old_name, new_name, is_tool_context=True) + for memory in self.memories_manager.list_memories().get_full_list(): + memory_content = self.memories_manager.load_memory(memory) + updated_content, n_replacements = self.memories_manager.rename_references_to_memory(memory_content, old_name, new_name) + if n_replacements > 0: + log.info(f"Updated {n_replacements} references to memory {old_name} to {new_name}") + self.memories_manager.save_memory(memory, updated_content, is_tool_context=True) + return renaming_message class EditMemoryTool(Tool, ToolMarkerCanEdit): @@ -100,10 +106,10 @@ class EditMemoryTool(Tool, ToolMarkerCanEdit): allow_multiple_occurrences: bool = False, ) -> str: r""" - Replaces content matching a regular expression in a memory. + Replace content matching a regular expression in a memory. :param memory_name: the name of the memory - :param needle: the string or regex pattern to search for. + :param needle: the string or regex pattern to search for. In regex mode, be careful to not replace too much! If `mode` is "literal", this string will be matched exactly. If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module, with flags DOTALL and MULTILINE enabled). diff --git a/test/serena/test_memories_manager.py b/test/serena/test_memories_manager.py new file mode 100644 index 00000000..a862846c --- /dev/null +++ b/test/serena/test_memories_manager.py @@ -0,0 +1,156 @@ +""" +Unit tests for :class:`serena.project.MemoriesManager` helpers that do not +require touching the filesystem (in particular, :meth:`rename_references_to_memory` +and :meth:`_prepare_name`). +""" + +import os + +import pytest + +from serena.project import MemoriesManager + + +@pytest.fixture +def manager() -> MemoriesManager: + # serena_data_folder=None: the tested helpers do not access the filesystem + return MemoriesManager(serena_data_folder=None) + + +class TestPrepareName: + @pytest.mark.parametrize( + "name", + [ + "memory", + "emergency", + "meeting_notes", + "em-dash", + "ml_notes", + "embedding", + "e", + "m", + "mem", + ":colon_prefixed_unlikely", + ], + ) + def test_does_not_strip_partial_prefix_chars_from_real_names(self, manager: MemoriesManager, name: str) -> None: + """A regression guard: previously used ``str.lstrip("mem:")`` which strips any leading m/e/:.""" + assert manager._prepare_name(name) == name + + @pytest.mark.parametrize( + ("input_name", "expected"), + [ + ("mem:foo", "foo"), + ("mem:foo/bar", "foo/bar"), + ("mem:memory", "memory"), # prefix is removed only once, not greedily + ("mem:", ""), + ], + ) + def test_strips_mem_prefix_only_when_actually_a_prefix(self, manager: MemoriesManager, input_name: str, expected: str) -> None: + assert manager._prepare_name(input_name) == expected + + @pytest.mark.parametrize( + ("input_name", "expected"), + [ + ("foo.md", "foo"), + ("topic/foo.md", "topic/foo"), + ("a.md.notes", "a.md.notes"), # ".md" must only be stripped as trailing extension + ("foo.markdown", "foo.markdown"), + (".md", ""), + ], + ) + def test_strips_md_only_as_suffix(self, manager: MemoriesManager, input_name: str, expected: str) -> None: + assert manager._prepare_name(input_name) == expected + + def test_normalizes_os_separator(self, manager: MemoriesManager) -> None: + assert manager._prepare_name(f"topic{os.sep}sub{os.sep}name") == "topic/sub/name" + + def test_combined_normalizations(self, manager: MemoriesManager) -> None: + assert manager._prepare_name(f"mem:topic{os.sep}foo.md") == "topic/foo" + + +class TestRenameReferencesToMemory: + @pytest.mark.parametrize( + ("content", "old_name", "new_name", "expected_content", "expected_count"), + [ + # simple cases + pytest.param("see mem:foo for details", "foo", "bar", "see mem:bar for details", 1, id="simple"), + pytest.param("nothing to see here", "foo", "bar", "nothing to see here", 0, id="no_match"), + pytest.param( + "mem:foo and mem:foo and `mem:foo`", + "foo", + "bar", + "mem:bar and mem:bar and `mem:bar`", + 3, + id="multiple_occurrences", + ), + # boundaries + pytest.param("mem:foo rest", "foo", "bar", "mem:bar rest", 1, id="start_of_string"), + pytest.param("prefix mem:foo", "foo", "bar", "prefix mem:bar", 1, id="end_of_string"), + pytest.param("mem:foo", "foo", "bar", "mem:bar", 1, id="entire_string"), + # must NOT match: embedded in longer name + pytest.param("`mem:foobar`", "foo", "bar", "`mem:foobar`", 0, id="longer_suffix"), + pytest.param("`mem:foo/sub`", "foo", "bar", "`mem:foo/sub`", 0, id="subtopic"), + pytest.param("mem:foo-extra", "foo", "bar", "mem:foo-extra", 0, id="hyphen_suffix"), + pytest.param("mem:foo_extra", "foo", "bar", "mem:foo_extra", 0, id="underscore_suffix"), + pytest.param("xmem:foo", "foo", "bar", "xmem:foo", 0, id="preceded_by_name_char"), + # boundaries on non-name characters + pytest.param("see `mem:foo.md`", "foo", "bar", "see `mem:bar.md`", 1, id="md_extension_suffix"), + # topic renames + pytest.param("`mem:foo/bar`", "foo/bar", "baz", "`mem:baz`", 1, id="topic_to_flat"), + pytest.param("`mem:foo`", "foo/bar", "baz", "`mem:foo`", 0, id="parent_does_not_match_topic"), + pytest.param("`mem:foo`", "foo", "topic/bar", "`mem:topic/bar`", 1, id="flat_to_topic"), + # regex-meta safety + pytest.param("`mem:a.b`", "a.b", "c", "`mem:c`", 1, id="dot_in_old_name_literal"), + pytest.param("`mem:axb`", "a.b", "c", "`mem:axb`", 0, id="dot_in_old_name_not_wildcard"), + # mixed + pytest.param( + "valid `mem:foo`, embedded mem:foobar, subtopic mem:foo/x, and another `mem:foo`.", + "foo", + "bar", + "valid `mem:bar`, embedded mem:foobar, subtopic mem:foo/x, and another `mem:bar`.", + 2, + id="mixed_matching_and_nonmatching", + ), + # input names that previously broke due to lstrip("mem:") in _add_reference_prefix + pytest.param("see `mem:memory`", "memory", "diary", "see `mem:diary`", 1, id="old_name_starts_with_m"), + pytest.param("see `mem:emergency`", "emergency", "incident", "see `mem:incident`", 1, id="old_name_starts_with_e"), + pytest.param("see `mem:foo`", "foo", "memory", "see `mem:memory`", 1, id="new_name_starts_with_m"), + ], + ) + def test_rename( + self, + manager: MemoriesManager, + content: str, + old_name: str, + new_name: str, + expected_content: str, + expected_count: int, + ) -> None: + result, n = manager.rename_references_to_memory(content, old_name, new_name) + assert result == expected_content + assert n == expected_count + + @pytest.mark.parametrize( + "delimiter_pair", + [ + ("`", "`"), + ('"', '"'), + ("'", "'"), + ("(", ")"), + ("[", "]"), + ("{", "}"), + (" ", " "), + ("\n", "\n"), + ("\t", "\t"), + (",", ","), + (";", ";"), + ("", ""), # raw, no delimiters + ], + ) + def test_replacement_inside_various_delimiters(self, manager: MemoriesManager, delimiter_pair: tuple[str, str]) -> None: + left, right = delimiter_pair + content = f"{left}mem:foo{right}" + result, n = manager.rename_references_to_memory(content, "foo", "bar") + assert result == f"{left}mem:bar{right}" + assert n == 1 From 18874913daa73c8cc9fdabaa1f3462051af4c8a0 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 13 May 2026 14:29:13 +0200 Subject: [PATCH 02/20] Feature: added memory referential integrity check --- src/serena/cli.py | 146 ++++++- src/serena/project.py | 628 ++++++++++++++++++++++++++- test/serena/test_memories_manager.py | 347 +++++++++++++++ 3 files changed, 1119 insertions(+), 2 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index ee0bf173..86fc4d5f 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -1113,6 +1113,149 @@ class ToolCommands(AutoRegisteringGroup): click.echo(mcp_tool.description) +class MemoryCommands(AutoRegisteringGroup): + """Group for 'memories' subcommands; manage and inspect a project's memory files.""" + + def __init__(self) -> None: + super().__init__( + name="memories", + help="Inspect, read, write, and validate Serena memories. " + "You can run `serena memories --help` for more info on each command.", + ) + + @staticmethod + def _load_memories_manager(project: str): + """Load the active project and return its memories manager.""" + from serena.project import Project + + serena_config = SerenaConfig.from_config_file() + proj = Project.load(os.path.abspath(project), serena_config=serena_config) + return proj.memories_manager + + @staticmethod + @click.command( + "list", + help="List memories of the active project (and global memories).", + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + @click.option("--topic", "-t", type=str, default="", help="Restrict the listing to a single topic (e.g. 'auth' or 'global/style').") + def list(project: str, topic: str) -> None: + manager = MemoryCommands._load_memories_manager(project) + memories = manager.list_memories(topic=topic) + if not len(memories): + click.echo("(no memories found)") + return + if memories.memories: + click.echo("Project memories:") + for name in sorted(memories.memories): + click.echo(f" - {name}") + if memories.read_only_memories: + click.echo("Read-only memories:") + for name in sorted(memories.read_only_memories): + click.echo(f" - {name}") + + @staticmethod + @click.command("read", help="Print the content of a memory to stdout.", context_settings={"max_content_width": _MAX_CONTENT_WIDTH}) + @click.argument("memory_name", type=str) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + def read(memory_name: str, project: str) -> None: + manager = MemoryCommands._load_memories_manager(project) + click.echo(manager.load_memory(memory_name)) + + @staticmethod + @click.command( + "write", + help="Write a memory file. Reads the content from --content, --file, or stdin (in that order of precedence).", + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.argument("memory_name", type=str) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + @click.option("--content", type=str, default=None, help="Memory content provided directly on the command line.") + @click.option( + "--file", "file_path", type=click.Path(exists=True, dir_okay=False), default=None, help="Read memory content from the given file." + ) + def write(memory_name: str, project: str, content: str | None, file_path: str | None) -> None: + if content is None and file_path is not None: + with open(file_path, encoding="utf-8") as f: + content = f.read() + if content is None: + content = sys.stdin.read() + manager = MemoryCommands._load_memories_manager(project) + click.echo(manager.save_memory(memory_name, content, is_tool_context=False)) + + @staticmethod + @click.command( + "check", + help=( + "Check referential integrity across all memories of the project (and global memories). " + "Reports stale `mem:` references and unmarked-reference warnings split into high- and " + "low-confidence groups. Read-only and never writes. Always exits 0." + ), + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + def check(project: str) -> None: + manager = MemoryCommands._load_memories_manager(project) + report = manager.validate_referential_integrity() + click.echo(report.format()) + + @staticmethod + @click.command( + "auto-prefix-references", + help=( + "Rewrite exact bare occurrences of existing memory names by adding the `mem:` prefix. " + "This is a heuristic, file-mutating operation: a word that happens to coincide with a memory name " + "will be rewritten as a reference even if it was intended as ordinary prose. " + "Use --dry-run to preview the rewrites without modifying any files. " + "\n\n" + "Scope is narrower than what `serena memories check` reports. Only EXACT bare occurrences are rewritten " + "(the body text must equal an existing memory name verbatim); fuzzy near-miss findings surfaced by `check` " + "are NOT autofixable here, since rewriting them would require substring substitution rather than a prefix " + "addition — they are reported for manual review. " + "By default the rewrite is further restricted to memory names containing `/` or longer than the configured " + "threshold, and skips global and read-only memories; use the --include-* flags below to widen the scope." + ), + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + @click.option( + "--dry-run", + is_flag=True, + default=False, + help="Preview the rewrites this command would apply; do not modify any files.", + ) + @click.option( + "--include-flat-names", + is_flag=True, + default=False, + help="Also rewrite short, flat memory names (no `/`, below the length threshold). Raises false-positive risk significantly.", + ) + @click.option( + "--include-read-only", + is_flag=True, + default=False, + help="Also rewrite occurrences inside read-only memories.", + ) + @click.option( + "--include-global", + is_flag=True, + default=False, + help="Also rewrite occurrences inside global memories (affects every project consuming them).", + ) + def auto_prefix_references( + project: str, dry_run: bool, include_flat_names: bool, include_read_only: bool, include_global: bool + ) -> None: + manager = MemoryCommands._load_memories_manager(project) + report = manager.auto_prefix_bare_references( + include_flat_names=include_flat_names, + include_read_only=include_read_only, + include_global=include_global, + dry_run=dry_run, + ) + click.echo(report.format()) + + class PromptCommands(AutoRegisteringGroup): def __init__(self) -> None: super().__init__(name="prompts", help="Commands related to Serena's prompts that are outside of contexts and modes.") @@ -1244,10 +1387,11 @@ _project = ProjectCommands() _config = SerenaConfigCommands() _tools = ToolCommands() _prompts = PromptCommands() +_memories = MemoryCommands() # Expose so we can use this as an entrypoint top_level = TopLevelCommands() # needed for the help script to work - register all subcommands to the top-level group -for subgroup in (_mode, _context, _project, _config, _tools, _prompts): +for subgroup in (_mode, _context, _project, _config, _tools, _prompts, _memories): top_level.add_command(subgroup) diff --git a/src/serena/project.py b/src/serena/project.py index 54c621d3..dceb65a7 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -3,7 +3,9 @@ import os import re import shutil import threading -from collections.abc import Sequence +from collections.abc import Iterator, Sequence +from dataclasses import dataclass, field +from difflib import SequenceMatcher from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, Optional @@ -118,6 +120,192 @@ class MemoriesManager: # use a callable replacement to avoid backreference interpretation of characters in ref_new return re.subn(pattern, lambda _m: ref_new, content) + NAME_SIMILARITY_THRESHOLD: float = 0.55 + _HIGH_CONFIDENCE_NAME_LENGTH: int = 10 + _SHORT_NAME_FLOOR: int = 3 + # For fuzzy bare-text matching (validate_referential_integrity), an additional safeguard: + # the bare token and the candidate existing name must share at least this fraction of their + # tokenized name parts. Containment / SequenceMatcher signals alone over-match — e.g. a + # generic prose word like "repository" is a substring of "serena_repository_structure" + # but their token Jaccard is only 1/3, so we reject it at this floor. + _FUZZY_BARE_TOKEN_JACCARD_FLOOR: float = 0.5 + _VERSION_SUFFIX_PATTERN = re.compile(r"_(?:v\d+|old|new|legacy|bak)$", re.IGNORECASE) + + @classmethod + def _normalize_for_similarity(cls, name: str) -> str: + """ + :param name: a memory name + :return: a lowercased copy of ``name`` with version/legacy-style trailing suffixes stripped + (e.g. ``"auth_v2"`` -> ``"auth"``); used as the canonical form for similarity scoring. + """ + return cls._VERSION_SUFFIX_PATTERN.sub("", name.lower()) + + @classmethod + def _tokenize_name(cls, name: str) -> set[str]: + """ + :param name: a memory name + :return: the set of lowercase tokens extracted by splitting ``name`` on ``/``, ``_``, ``-`` + and at camelCase boundaries; empty tokens are dropped. + """ + # split on path/word separators and at camelCase boundaries + parts = re.split(r"[/_\-]|(?<=[a-z])(?=[A-Z])", name) + return {p.lower() for p in parts if p} + + @classmethod + def compute_name_similarity(cls, a: str, b: str) -> float: + """ + Computes a similarity score in ``[0, 1]`` between two memory names. Examples + (with default threshold ``0.55``):: + + auth/login == auth/login -> 1.00 (exact) + auth_v1 ~ auth_v2 -> 1.00 (version suffix normalized) + login ~ auth/login -> 1.00 (flat <-> topic move) + auth/login ~ auth/v2/login -> 0.75 (shared prefix token + basename) + auth/login ~ security/login -> 0.50 (basename only; disjoint topics) + auth ~ authentication -> 0.64 (substring containment) + foo ~ for -> 0.00 (short-name floor) + + :param a: first memory name + :param b: second memory name + :return: the similarity score + """ + # normalize: lowercase + strip version/legacy suffixes + norm_a, norm_b = cls._normalize_for_similarity(a), cls._normalize_for_similarity(b) + if norm_a == norm_b: + return 1.0 + + # split into prefix / basename + basename_a = norm_a.rsplit("/", 1)[-1] + basename_b = norm_b.rsplit("/", 1)[-1] + prefix_a = norm_a.rsplit("/", 1)[0] if "/" in norm_a else "" + prefix_b = norm_b.rsplit("/", 1)[0] if "/" in norm_b else "" + + # case 1: basenames match -- score is determined entirely by prefix relationship + if basename_a == basename_b: + if not prefix_a or not prefix_b: + # one side is flat -> canonical topic move, strongly confident + return 1.0 + # both sides carry a topic prefix -> blend basename match (0.5 floor) with prefix token similarity + prefix_tokens_a, prefix_tokens_b = cls._tokenize_name(prefix_a), cls._tokenize_name(prefix_b) + prefix_jaccard = ( + len(prefix_tokens_a & prefix_tokens_b) / len(prefix_tokens_a | prefix_tokens_b) + if (prefix_tokens_a | prefix_tokens_b) + else 0.0 + ) + return 0.5 + 0.5 * prefix_jaccard + + # case 2: basenames differ -- combine generic similarity signals via max + tokens_a, tokens_b = cls._tokenize_name(norm_a), cls._tokenize_name(norm_b) + jaccard = len(tokens_a & tokens_b) / len(tokens_a | tokens_b) if (tokens_a | tokens_b) else 0.0 + seq = SequenceMatcher(None, norm_a, norm_b).ratio() + # containment is a meaningful rename signal on its own (e.g. auth -> authentication), + # so we lift it above the threshold floor when one name fully contains the other, + # with the length-ratio gating how confident we are + contained = 0.0 + if norm_a and norm_b and (norm_a in norm_b or norm_b in norm_a): + length_ratio = min(len(norm_a), len(norm_b)) / max(len(norm_a), len(norm_b)) + contained = 0.5 + 0.5 * length_ratio + + # short-name floor: for very short names, only accept candidates with a strong token signal + if min(len(norm_a), len(norm_b)) <= cls._SHORT_NAME_FLOOR and jaccard < 1.0: + return 0.0 + return max(jaccard, seq, contained) + + @classmethod + def _find_stale_reference_candidates(cls, missing_name: str, existing_names: list[str], threshold: float | None = None) -> list[str]: + """ + :param missing_name: the unresolved reference target (without the ``mem:`` prefix). + :param existing_names: the names of all currently existing memories. + :param threshold: optional override for the minimum similarity required; defaults to + :attr:`NAME_SIMILARITY_THRESHOLD`. + :return: existing memory names whose similarity to ``missing_name`` meets or exceeds + ``threshold``, sorted in descending order of similarity (ties broken alphabetically). + """ + cutoff = cls.NAME_SIMILARITY_THRESHOLD if threshold is None else threshold + scored: list[tuple[float, str]] = [] + for existing in existing_names: + score = cls.compute_name_similarity(missing_name, existing) + if score >= cutoff: + scored.append((score, existing)) + scored.sort(key=lambda pair: (-pair[0], pair[1])) + return [name for _, name in scored] + + # character class delimiting a memory name; used to anchor regex matches so we never + # consume a partial name (kept in sync with the boundary rule in rename_references_to_memory) + _NAME_CHAR_CLASS: str = r"[A-Za-z0-9_\-/]" + + @classmethod + def _iter_referenced_names_in_content(cls, content: str) -> Iterator[str]: + """ + :param content: arbitrary memory content + :return: an iterator over the memory names appearing as ``mem:NAME`` references in + ``content``; duplicate occurrences are yielded once each. + """ + # use the same boundary rule as rename_references_to_memory so the two stay consistent + pattern = rf"(? int: + """ + :param content: arbitrary memory content + :param name: a memory name to look for + :return: the count of bare (i.e. not already ``mem:``-prefixed) word-boundary-anchored + occurrences of ``name`` in ``content``. The match must not be preceded by ``mem:`` + and must not be embedded within a longer memory-name-like run of characters. + """ + # negative lookbehind for the mem: prefix prevents double-counting valid references; + # the additional name-char lookbehind/lookahead prevents matching inside longer names + pattern = ( + rf"(? tuple[str, int]: + """ + :param content: arbitrary memory content + :param name: the memory name whose bare occurrences should be rewritten + :return: ``(new_content, n_replacements)`` after prefixing each bare occurrence of + ``name`` with ``mem:`` (using the same boundary rule as + :meth:`_find_bare_occurrences`). + """ + pattern = ( + rf"(? Iterator[tuple[str, int]]: + """ + :param content: arbitrary memory content + :return: an iterator over ``(token, count)`` pairs for each distinct name-shaped + token in ``content`` whose length meets :attr:`_HIGH_CONFIDENCE_NAME_LENGTH` + and which is not preceded by ``mem:`` (those are already valid references). + The boundaries follow :attr:`_NAME_CHAR_CLASS`, so embedded substrings of + longer name-character runs are not matched. + """ + # require minimum length so we only scan tokens that are unlikely to be coincidental prose + pattern = ( + rf"(? Path: name = self._prepare_name(name) parts = name.split("/") @@ -204,6 +392,212 @@ class MemoriesManager: def get_full_list(self) -> list[str]: return sorted(self.memories + self.read_only_memories) + @dataclass(frozen=True) + class StaleReference: + """ + A ``mem:NAME`` reference whose target memory does not exist. + + :ivar source_memory: the name of the memory whose content contains the broken reference. + :ivar referenced_name: the name following ``mem:`` that did not resolve to an existing memory. + :ivar candidates: existing memory names proposed as likely intended targets, ranked by + decreasing similarity. May be empty if no candidate exceeded the similarity threshold. + :ivar source_is_read_only: whether the source memory is read-only. + """ + + source_memory: str + referenced_name: str + candidates: list[str] + source_is_read_only: bool + + @dataclass(frozen=True) + class UnmarkedReferenceWarning: + """ + A bare occurrence in a memory's content that looks like a forgotten reference to an + existing memory. + + Two flavours of finding share this class: + + * **exact match** — the bare text equals an existing memory name verbatim + (``actual_token`` either equals ``suspected_name`` or is left empty). + * **fuzzy near-miss** — the bare text does not equal any existing memory name, but + a long, distinctive token in the body similarity-matches a high-confidence existing + memory name (``actual_token`` is the bare text actually found, distinct from + ``suspected_name``). Such findings are reported but **not** rewritten by + :meth:`auto_prefix_bare_references`, since they would require substring + substitution rather than a prefix addition. + + :ivar source_memory: the name of the memory whose content contains the bare occurrence. + :ivar suspected_name: the existing memory name proposed as the intended target. + :ivar occurrences: the number of occurrences of ``actual_token`` in the source memory's content. + :ivar is_high_confidence: True when ``suspected_name`` contains a ``/`` separator or + exceeds the configured length threshold; such names are unlikely to coincide with + ordinary prose. False otherwise (a low-confidence warning). + :ivar source_is_read_only: whether the source memory is read-only. + :ivar actual_token: the bare text actually found in the source memory's content. + Defaults to ``""``, which is taken to mean ``suspected_name`` (the exact-match case). + When non-empty and different from ``suspected_name``, this is a fuzzy near-miss. + """ + + source_memory: str + suspected_name: str + occurrences: int + is_high_confidence: bool + source_is_read_only: bool + actual_token: str = "" + + @property + def is_exact_match(self) -> bool: + """:return: True iff the bare text in the body equals ``suspected_name`` (i.e. not a fuzzy near-miss).""" + return self.actual_token == "" or self.actual_token == self.suspected_name + + @dataclass(frozen=True) + class AutofixedReference: + """ + A bare occurrence rewritten to include the ``mem:`` prefix. + + :ivar source_memory: the name of the memory whose content was modified. + :ivar referenced_name: the memory name whose bare occurrences were prefixed. + :ivar n_replacements: the number of bare occurrences replaced in the source memory. + """ + + source_memory: str + referenced_name: str + n_replacements: int + + @dataclass + class ReferentialIntegrityReport: + """ + Outcome of :meth:`MemoriesManager.validate_referential_integrity`. + + :ivar stale_references: ``mem:NAME`` references whose target memory does not exist. + :ivar high_confidence_warnings: bare references whose suspected target name is unlikely + to be coincidental prose (topic-path or sufficiently long). + :ivar low_confidence_warnings: bare references whose suspected target name could plausibly + appear in ordinary prose (short, flat names). + """ + + stale_references: list["MemoriesManager.StaleReference"] = field(default_factory=list) + high_confidence_warnings: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) + low_confidence_warnings: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) + + def is_clean(self) -> bool: + """:return: True iff no stale references and no warnings of any confidence level were found.""" + return not (self.stale_references or self.high_confidence_warnings or self.low_confidence_warnings) + + def format(self) -> str: + """:return: a human-readable rendering suitable for CLI display.""" + tb = TextBuilder() + if self.is_clean(): + tb.with_line("✓ No referential integrity issues found.") + return tb.build() + + # stale references — grouped by writable / read-only for clarity + if self.stale_references: + tb.with_line(f"Stale references ({len(self.stale_references)}):") + for ref in self.stale_references: + ro_tag = " [read-only source]" if ref.source_is_read_only else "" + tb.with_line(f" - `mem:{ref.referenced_name}` in `{ref.source_memory}`{ro_tag}") + if ref.candidates: + candidates_str = ", ".join(f"`mem:{c}`" for c in ref.candidates) + tb.with_line(f" candidates: {candidates_str}") + else: + tb.with_line(" candidates: (none above similarity threshold)") + tb.with_line("") + + # unmarked-reference warnings, split by confidence + for label, warnings in ( + ("High-confidence unmarked references", self.high_confidence_warnings), + ("Low-confidence unmarked references", self.low_confidence_warnings), + ): + if not warnings: + continue + tb.with_line(f"{label} ({len(warnings)}):") + for w in warnings: + ro_tag = " [read-only source]" if w.source_is_read_only else "" + occ = "occurrence" if w.occurrences == 1 else "occurrences" + if w.is_exact_match: + tb.with_line( + f" - {w.occurrences} {occ} of `{w.suspected_name}` in `{w.source_memory}`{ro_tag}" + f" (suggested: `mem:{w.suspected_name}`)" + ) + else: + # fuzzy near-miss: the bare text differs from the proposed target + tb.with_line( + f" - {w.occurrences} {occ} of `{w.actual_token}` in `{w.source_memory}`{ro_tag}" + f" — near-miss (suggested: `mem:{w.suspected_name}`)" + ) + tb.with_line("") + return tb.build() + + @dataclass + class AutofixReport: + """ + Outcome of :meth:`MemoriesManager.auto_prefix_bare_references`. + + :ivar autofixed: per-(source, target) records of bare references that were rewritten. + When the call was a dry run, these records describe what *would* have been + written; the files themselves are unchanged. + :ivar dry_run: True if the run was a preview that did not write any files. + :ivar skipped_read_only: warnings whose source memory was read-only and therefore not + modified (only populated when ``include_read_only`` is False). + :ivar skipped_flat: warnings skipped because their suspected target had no ``/`` + separator and was not long enough to be high-confidence (only populated when + ``include_flat_names`` is False). + :ivar skipped_global: warnings skipped because their source memory was global and + ``include_global`` was False. + :ivar skipped_fuzzy: warnings whose bare text in the source memory differs from the + suspected target (fuzzy near-misses). These require substring substitution rather + than a prefix addition and are never autofixed; surface them to the user for + manual review instead. + """ + + autofixed: list["MemoriesManager.AutofixedReference"] = field(default_factory=list) + dry_run: bool = False + skipped_read_only: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) + skipped_flat: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) + skipped_global: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) + skipped_fuzzy: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) + + @property + def total_replacements(self) -> int: + return sum(a.n_replacements for a in self.autofixed) + + def format(self) -> str: + """:return: a human-readable rendering suitable for CLI display.""" + tb = TextBuilder() + verb = "Would apply" if self.dry_run else "Applied" + if self.autofixed: + tb.with_line( + f"{verb} {self.total_replacements} replacement(s) across " + f"{len(self.autofixed)} memory/target pair(s)" + (" (dry run; no files were modified)" if self.dry_run else "") + ":" + ) + for a in self.autofixed: + tb.with_line(f" - {a.n_replacements} x `{a.referenced_name}` -> `mem:{a.referenced_name}` in `{a.source_memory}`") + tb.with_line("") + else: + tb.with_line("No replacements" + (" would be" if self.dry_run else "") + " applied.") + tb.with_line("") + + for label, items in ( + ("Skipped (read-only source; pass --include-read-only to override)", self.skipped_read_only), + ("Skipped (flat name; pass --include-flat-names to include)", self.skipped_flat), + ("Skipped (global memory; pass --include-global to include)", self.skipped_global), + ( + "Skipped (fuzzy near-miss; bare text differs from target name — not safe to auto-rewrite, review manually)", + self.skipped_fuzzy, + ), + ): + if not items: + continue + tb.with_line(f"{label}:") + for w in items: + if w.is_exact_match: + tb.with_line(f" - `{w.suspected_name}` in `{w.source_memory}`") + else: + tb.with_line(f" - `{w.actual_token}` in `{w.source_memory}` (suggested: `mem:{w.suspected_name}`)") + tb.with_line("") + return tb.build() + def _list_memories(self, search_dir: Path, base_dir: Path, prefix: str = "") -> MemoriesList: result = self.MemoriesList() if not search_dir.exists(): @@ -309,6 +703,238 @@ class MemoriesManager: f.write(updated_content) return f"Memory {name} edited successfully." + def _is_self_reference(self, source_memory: str, suspected_name: str) -> bool: + """ + :return: True iff ``suspected_name`` would refer to ``source_memory`` itself + (including the case where ``suspected_name`` equals the basename of a topic-path + source memory). + """ + if suspected_name == source_memory: + return True + source_basename = source_memory.rsplit("/", 1)[-1] + return suspected_name == source_basename + + def validate_referential_integrity(self) -> "MemoriesManager.ReferentialIntegrityReport": + """ + Scans every (non-ignored) memory's content for referential integrity issues. + + The scan covers both project-local and global memories. Three kinds of finding are + produced: + + * **stale references** — occurrences of ``mem:NAME`` where ``NAME`` does not resolve + to an existing memory. For each, a list of similarly-named existing memories is + proposed as candidate intended targets (see :meth:`compute_name_similarity`). + * **exact unmarked-reference warnings** — bare occurrences of an existing memory + name that appear without the ``mem:`` prefix. Split into high-confidence (the + suspected name contains a ``/`` or exceeds :attr:`_HIGH_CONFIDENCE_NAME_LENGTH`) + and low-confidence groups. + * **fuzzy near-miss warnings** — long, distinctive bare tokens in a memory body + that *do not* match an existing name exactly but similarity-match a + high-confidence existing name AND share at least :attr:`_FUZZY_BARE_TOKEN_JACCARD_FLOOR` + of their tokenized name parts (e.g. ``adding_new_language_support`` in prose when + the memory is named ``adding_new_language_support_guide``). Always reported as + high-confidence; the actual bare text is preserved on + :attr:`UnmarkedReferenceWarning.actual_token`. + + Self-references (a memory's content mentioning its own name or basename) and empty + memories are skipped silently. + + This method has no side effects. + + :return: a :class:`ReferentialIntegrityReport` summarizing all findings. + """ + report = MemoriesManager.ReferentialIntegrityReport() + + # gather all existing memory names (project + global) with their read-only status + memories_list = self.list_memories() + all_names = sorted(set(memories_list.memories) | set(memories_list.read_only_memories)) + read_only_names = set(memories_list.read_only_memories) + existing_set = set(all_names) + # only existing names that are themselves "memory-name-like" qualify as fuzzy candidates, + # otherwise short flat names would match arbitrary long prose words via containment + high_confidence_names = [n for n in all_names if "/" in n or len(n) >= self._HIGH_CONFIDENCE_NAME_LENGTH] + + for source_name in all_names: + content = self.load_memory(source_name) + if not content: + # skip empty memories silently — no references can exist in empty content + continue + source_is_read_only = source_name in read_only_names + + # stale references: every mem:NAME whose NAME is not an existing memory + stale_seen: set[str] = set() + for referenced in self._iter_referenced_names_in_content(content): + if referenced in existing_set or referenced in stale_seen: + continue + stale_seen.add(referenced) + candidates = self._find_stale_reference_candidates(referenced, all_names) + report.stale_references.append( + MemoriesManager.StaleReference( + source_memory=source_name, + referenced_name=referenced, + candidates=candidates, + source_is_read_only=source_is_read_only, + ) + ) + + # exact unmarked references: bare occurrences of any existing memory name (except self) + for candidate_name in all_names: + if self._is_self_reference(source_name, candidate_name): + continue + n = self._find_bare_occurrences(content, candidate_name) + if n == 0: + continue + is_high_conf = "/" in candidate_name or len(candidate_name) >= self._HIGH_CONFIDENCE_NAME_LENGTH + warning = MemoriesManager.UnmarkedReferenceWarning( + source_memory=source_name, + suspected_name=candidate_name, + occurrences=n, + is_high_confidence=is_high_conf, + source_is_read_only=source_is_read_only, + actual_token=candidate_name, + ) + if is_high_conf: + report.high_confidence_warnings.append(warning) + else: + report.low_confidence_warnings.append(warning) + + # fuzzy near-misses: long bare tokens that similarity-match a high-confidence existing name + for token, n in self._iter_long_bare_tokens(content): + if token in existing_set: + # exact-match path already handled this token (or self-referenced and was skipped) + continue + token_tokens = self._tokenize_name(token) + best_name: str | None = None + best_score = 0.0 + for existing in high_confidence_names: + if self._is_self_reference(source_name, existing): + continue + score = self.compute_name_similarity(token, existing) + if score < self.NAME_SIMILARITY_THRESHOLD: + continue + # safeguard against substring-only matches: require meaningful token overlap so + # generic prose words don't get flagged just because they happen to be substrings + # of an existing memory name + existing_tokens = self._tokenize_name(existing) + union = token_tokens | existing_tokens + if not union: + continue + jaccard = len(token_tokens & existing_tokens) / len(union) + if jaccard < self._FUZZY_BARE_TOKEN_JACCARD_FLOOR: + continue + if score > best_score: + best_score = score + best_name = existing + if best_name is None: + continue + report.high_confidence_warnings.append( + MemoriesManager.UnmarkedReferenceWarning( + source_memory=source_name, + suspected_name=best_name, + occurrences=n, + is_high_confidence=True, + source_is_read_only=source_is_read_only, + actual_token=token, + ) + ) + + return report + + def auto_prefix_bare_references( + self, + include_flat_names: bool = False, + include_read_only: bool = False, + include_global: bool = False, + dry_run: bool = False, + ) -> "MemoriesManager.AutofixReport": + """ + Rewrites *exact* bare occurrences of existing memory names by adding the ``mem:`` prefix. + + .. warning:: + **This is a heuristic, file-mutating operation** (unless ``dry_run`` is True). A + bare word that happens to coincide with a memory name will be rewritten as a + reference, even if it was intended as ordinary prose. Pass ``dry_run=True`` to + preview the rewrites before applying them. + + Scope is intentionally narrower than what :meth:`validate_referential_integrity` + reports: + + * Only **exact** bare occurrences are rewritten — i.e. the bare text in the source + body must equal an existing memory name verbatim. Fuzzy near-miss findings + (where the actual token differs from the suspected target) require substring + substitution rather than a prefix addition and are routed into + :attr:`AutofixReport.skipped_fuzzy` for manual review. + * By default the rewrite is restricted to *high-confidence* findings only — those + whose suspected target name contains a ``/`` separator or exceeds the + configured length threshold — and skips global memories and read-only memories. + The defaults intentionally err toward false negatives over false positives. + + :param include_flat_names: if True, also rewrite low-confidence findings (flat, + short memory names). Increases recall but markedly raises false-positive risk. + :param include_read_only: if True, also rewrite occurrences inside read-only + memories. Use with care, as read-only memories are typically considered + authoritative. + :param include_global: if True, also rewrite occurrences inside global memories. + Modifying a global memory affects every project that consumes it. + :param dry_run: if True, the report describes the rewrites that *would* be applied + but no files are modified. + :return: an :class:`AutofixReport` describing every replacement (made or planned) + and every warning that was deliberately skipped. + """ + report = MemoriesManager.AutofixReport(dry_run=dry_run) + + # build the integrity report first so autofix decisions are made on the same data + validation = self.validate_referential_integrity() + + # combine the warning groups according to the include_flat_names policy + warnings: list[MemoriesManager.UnmarkedReferenceWarning] = list(validation.high_confidence_warnings) + if include_flat_names: + warnings.extend(validation.low_confidence_warnings) + else: + report.skipped_flat.extend(validation.low_confidence_warnings) + + # apply replacements per source memory, grouping warnings to avoid re-reading content + warnings_by_source: dict[str, list[MemoriesManager.UnmarkedReferenceWarning]] = {} + for w in warnings: + # fuzzy near-misses are never auto-rewritten — the bare text differs from the target + if not w.is_exact_match: + report.skipped_fuzzy.append(w) + continue + # filter remaining warnings according to scope flags + if w.source_is_read_only and not include_read_only: + report.skipped_read_only.append(w) + continue + if self._is_global(w.source_memory) and not include_global: + report.skipped_global.append(w) + continue + warnings_by_source.setdefault(w.source_memory, []).append(w) + + for source_memory, source_warnings in warnings_by_source.items(): + content = self.load_memory(source_memory) + total_n = 0 + per_target_records: list[MemoriesManager.AutofixedReference] = [] + + # apply replacements sequentially; each pass uses the (potentially updated) content + for w in source_warnings: + content, n = self._add_bare_occurrences_prefix(content, w.suspected_name) + if n > 0: + per_target_records.append( + MemoriesManager.AutofixedReference( + source_memory=source_memory, + referenced_name=w.suspected_name, + n_replacements=n, + ) + ) + total_n += n + + if total_n > 0: + if not dry_run: + # use is_tool_context=False so we don't trip read-only protection when --include-read-only is set + self.save_memory(source_memory, content, is_tool_context=False) + report.autofixed.extend(per_target_records) + + return report + class Project(ToStringMixin): def __init__( diff --git a/test/serena/test_memories_manager.py b/test/serena/test_memories_manager.py index a862846c..991feb98 100644 --- a/test/serena/test_memories_manager.py +++ b/test/serena/test_memories_manager.py @@ -154,3 +154,350 @@ class TestRenameReferencesToMemory: result, n = manager.rename_references_to_memory(content, "foo", "bar") assert result == f"{left}mem:bar{right}" assert n == 1 + + +class TestNameSimilarity: + @pytest.mark.parametrize( + ("a", "b", "expected"), + [ + # exact match (and case-insensitive) -> 1.0 + pytest.param("foo", "foo", 1.0, id="identical"), + pytest.param("Foo", "foo", 1.0, id="case_insensitive"), + # version-suffix renames are normalized away -> 1.0 + pytest.param("auth_v1", "auth_v2", 1.0, id="version_suffix_rename"), + pytest.param("auth_old", "auth", 1.0, id="old_suffix_normalized"), + pytest.param("auth_legacy", "auth", 1.0, id="legacy_suffix_normalized"), + # flat <-> topic basename move: canonical rename, fully confident + pytest.param("a/b/c/login", "login", 1.0, id="topic_move_to_flat"), + pytest.param("login", "auth/login", 1.0, id="flat_to_topic_move"), + # both sides share a prefix token AND basename: high confidence + pytest.param("auth/login", "auth/v2/login", 0.75, id="basename_match_shared_prefix"), + ], + ) + def test_high_similarity_cases(self, manager: MemoriesManager, a: str, b: str, expected: float) -> None: + assert manager.compute_name_similarity(a, b) == pytest.approx(expected) + + @pytest.mark.parametrize( + ("a", "b"), + [ + # completely unrelated names should be below threshold + pytest.param("authentication", "database", id="unrelated_long"), + pytest.param("api/users", "logging/handlers", id="unrelated_topic_paths"), + # short names without strong token signals must hit the short-name floor + pytest.param("foo", "bar", id="short_names_unrelated"), + pytest.param("foo", "for", id="short_names_typo_distance"), + # IMPORTANT: same basename under unrelated topic prefixes is NOT a rename. + # `auth/login` and `security/login` are two different login flows in different + # subsystems; `topic1/topic2/core` and `topic3/topic5/core` are unrelated. + pytest.param("auth/login", "security/login", id="same_basename_unrelated_prefixes"), + pytest.param("topic1/topic2/core", "topic3/topic5/core", id="same_basename_disjoint_prefix_tokens"), + ], + ) + def test_below_threshold_cases(self, manager: MemoriesManager, a: str, b: str) -> None: + assert manager.compute_name_similarity(a, b) < MemoriesManager.NAME_SIMILARITY_THRESHOLD + + @pytest.mark.parametrize( + ("a", "b"), + [ + # prefix/suffix extensions: caught by sequence ratio / containment, even though Jaccard is 0 + pytest.param("auth", "authentication", id="prefix_extension"), + pytest.param("login", "user_login", id="token_extension"), + # typos on longer names: caught by sequence ratio + pytest.param("authentication", "autentication", id="typo_in_long_name"), + # token reorder / partial overlap + pytest.param("user_login_flow", "login_flow", id="partial_token_overlap"), + ], + ) + def test_above_threshold_cases(self, manager: MemoriesManager, a: str, b: str) -> None: + assert manager.compute_name_similarity(a, b) >= MemoriesManager.NAME_SIMILARITY_THRESHOLD + + def test_symmetry(self, manager: MemoriesManager) -> None: + # similarity is order-independent + assert manager.compute_name_similarity("auth/login", "security/login") == manager.compute_name_similarity( + "security/login", "auth/login" + ) + + def test_find_candidates_ranks_by_descending_similarity(self, manager: MemoriesManager) -> None: + existing = ["auth/login", "security/login", "database", "authentication"] + candidates = manager._find_stale_reference_candidates("login", existing) + # both auth/login and security/login share basename `login` -> tied at 1.0; expect them first, alphabetical tiebreak + assert candidates[:2] == ["auth/login", "security/login"] + # `database` is unrelated and must not appear + assert "database" not in candidates + + def test_find_candidates_returns_empty_below_threshold(self, manager: MemoriesManager) -> None: + existing = ["completely_unrelated", "another_thing"] + assert manager._find_stale_reference_candidates("foo", existing) == [] + + def test_find_candidates_custom_threshold(self, manager: MemoriesManager) -> None: + # raising the threshold filters out borderline matches + existing = ["authentication"] + loose = manager._find_stale_reference_candidates("auth", existing, threshold=0.2) + strict = manager._find_stale_reference_candidates("auth", existing, threshold=0.95) + assert loose == ["authentication"] + assert strict == [] + + +@pytest.fixture +def fs_manager(tmp_path) -> MemoriesManager: + """A MemoriesManager backed by tmp_path with no read-only patterns.""" + return MemoriesManager(serena_data_folder=tmp_path) + + +def _write(manager: MemoriesManager, name: str, content: str) -> None: + manager.save_memory(name, content, is_tool_context=False) + + +class TestValidateReferentialIntegrity: + def test_clean_report_when_all_references_resolve(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth", "# auth notes") + _write(fs_manager, "login_flow", "see `mem:auth` for details") + report = fs_manager.validate_referential_integrity() + assert report.is_clean() + assert "No referential integrity issues" in report.format() + + def test_stale_reference_with_topic_move_candidate(self, fs_manager: MemoriesManager) -> None: + # original memory `login` was moved to `auth/login`; a reference still points to bare `login` + _write(fs_manager, "auth/login", "# moved here") + _write(fs_manager, "docs", "old reference: `mem:login`") + report = fs_manager.validate_referential_integrity() + assert len(report.stale_references) == 1 + stale = report.stale_references[0] + assert stale.source_memory == "docs" + assert stale.referenced_name == "login" + assert "auth/login" in stale.candidates + + def test_stale_reference_with_no_candidate(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "completely_unrelated", "# content") + _write(fs_manager, "docs", "broken: `mem:xyz_unknown_thing`") + report = fs_manager.validate_referential_integrity() + assert len(report.stale_references) == 1 + assert report.stale_references[0].candidates == [] + + def test_high_confidence_unmarked_warning_for_topic_path(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth/login", "# notes") + _write(fs_manager, "docs", "the auth/login process is documented elsewhere") + report = fs_manager.validate_referential_integrity() + assert len(report.high_confidence_warnings) == 1 + warning = report.high_confidence_warnings[0] + assert warning.suspected_name == "auth/login" + assert warning.is_high_confidence is True + assert warning.occurrences == 1 + assert report.low_confidence_warnings == [] + + def test_low_confidence_unmarked_warning_for_short_flat_name(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth", "# notes") + _write(fs_manager, "docs", "you must use auth here") + report = fs_manager.validate_referential_integrity() + assert report.high_confidence_warnings == [] + assert len(report.low_confidence_warnings) == 1 + assert report.low_confidence_warnings[0].suspected_name == "auth" + assert report.low_confidence_warnings[0].is_high_confidence is False + + def test_long_flat_name_is_high_confidence(self, fs_manager: MemoriesManager) -> None: + # a flat name >= _HIGH_CONFIDENCE_NAME_LENGTH chars is unlikely to be coincidental prose + long_name = "authentication_flow" + assert len(long_name) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + _write(fs_manager, long_name, "# notes") + _write(fs_manager, "docs", f"the {long_name} is documented here") + report = fs_manager.validate_referential_integrity() + assert len(report.high_confidence_warnings) == 1 + + def test_fuzzy_near_miss_rejects_substring_only_matches(self, fs_manager: MemoriesManager) -> None: + """ + Regression: fuzzy bare-text matching must require meaningful token overlap, not just + substring containment. Without the token-Jaccard floor, generic words like + ``Repository`` would be flagged as near-misses for ``serena_repository_structure`` + purely because they appear as substrings — a noise flood. The cases below are taken + from a real run on the Serena codebase that produced six false positives. + """ + _write(fs_manager, "serena_repository_structure", "# repo structure notes") + _write(fs_manager, "suggested_commands", "# command notes") + _write( + fs_manager, + "docs", + # generic words ("Repository", "repository") and unrelated code identifiers + # ("_create_launch_command", "create_launch_command", "repository_root_path") + # that happen to be substrings of, or share one token with, an existing memory name + """\ + See _create_launch_command and also create_launch_command. + The Repository pattern is documented elsewhere. + Also repository_root_path is set via env var. + The repository contains many tools. + """, + ) + report = fs_manager.validate_referential_integrity() + assert report.is_clean(), ( + f"Expected no findings; got high={report.high_confidence_warnings}, " + f"low={report.low_confidence_warnings}, stale={report.stale_references}" + ) + + def test_long_flat_name_near_miss_is_high_confidence(self, fs_manager: MemoriesManager) -> None: + """ + Regression: bare text that is a *near-match* of an existing long memory name + (e.g. the same name with a missing suffix) should be flagged as a high-confidence + unmarked-reference warning. Long, distinctive multi-word tokens that appear bare in + prose and similarity-match an existing memory are almost certainly intended references. + + Real-world case: ``serena_repository_structure.md`` mentions + ``adding_new_language_support`` in prose; the actual memory is + ``adding_new_language_support_guide``. Exact-substring matching alone misses this. + """ + existing = "adding_new_language_support_guide" + bare = "adding_new_language_support" # prefix of `existing`; clearly the same concept + assert len(bare) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + _write(fs_manager, existing, "# guide content") + _write(fs_manager, "docs", f"See {bare} for details") + + report = fs_manager.validate_referential_integrity() + assert len(report.high_confidence_warnings) == 1, ( + f"expected one high-confidence warning for the near-miss reference, " + f"got {len(report.high_confidence_warnings)} " + f"(high={report.high_confidence_warnings}, low={report.low_confidence_warnings})" + ) + warning = report.high_confidence_warnings[0] + assert warning.suspected_name == existing + assert warning.source_memory == "docs" + + def test_self_reference_is_ignored(self, fs_manager: MemoriesManager) -> None: + # the basename `login` appears in `auth/login`'s own content -> must not be flagged + _write(fs_manager, "auth/login", "this login flow ...") + _write(fs_manager, "auth", "# auth notes") + report = fs_manager.validate_referential_integrity() + # `auth/login` mentioning its own basename `login` should not produce a warning; + # `auth` mentioned in `auth/login` content also shouldn't (it's the prefix), and only + # actual matches matter — assert no warning concerns the source itself + for w in report.high_confidence_warnings + report.low_confidence_warnings: + assert not (w.source_memory == "auth/login" and w.suspected_name in ("login", "auth/login")) + + def test_marked_reference_does_not_produce_unmarked_warning(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth/login", "# notes") + _write(fs_manager, "docs", "see `mem:auth/login` for details") + report = fs_manager.validate_referential_integrity() + # the marked reference is valid; no warning should fire on that occurrence + assert report.is_clean() + + def test_mixed_marked_and_bare_in_same_content(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth/login", "# notes") + _write(fs_manager, "docs", "we have `mem:auth/login` plus a bare auth/login mention") + report = fs_manager.validate_referential_integrity() + # only the bare occurrence should be counted + assert len(report.high_confidence_warnings) == 1 + assert report.high_confidence_warnings[0].occurrences == 1 + + def test_empty_memory_skipped(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth/login", "") + _write(fs_manager, "docs", "# nothing relevant") + # must not raise even though one memory is empty + report = fs_manager.validate_referential_integrity() + assert report.is_clean() + + def test_read_only_status_propagated(self, tmp_path) -> None: + manager = MemoriesManager(serena_data_folder=tmp_path, read_only_memory_patterns=[r"frozen/.*"]) + _write(manager, "frozen/notes", "see bare auth/login mention") + _write(manager, "auth/login", "# notes") + report = manager.validate_referential_integrity() + warnings = report.high_confidence_warnings + assert len(warnings) == 1 + assert warnings[0].source_memory == "frozen/notes" + assert warnings[0].source_is_read_only is True + + +class TestAutoPrefixBareReferences: + def test_prefixes_high_confidence_warning_in_writable_memory(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth/login", "# notes") + _write(fs_manager, "docs", "the auth/login process is documented here") + result = fs_manager.auto_prefix_bare_references() + assert result.total_replacements == 1 + assert len(result.autofixed) == 1 + # verify the file content was actually mutated + assert fs_manager.load_memory("docs") == "the mem:auth/login process is documented here" + + def test_skips_low_confidence_by_default(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth", "# notes") + _write(fs_manager, "docs", "use auth here") + result = fs_manager.auto_prefix_bare_references() + assert result.total_replacements == 0 + assert len(result.skipped_flat) == 1 + # content unchanged + assert fs_manager.load_memory("docs") == "use auth here" + + def test_include_flat_names_widens_scope(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth", "# notes") + _write(fs_manager, "docs", "use auth here") + result = fs_manager.auto_prefix_bare_references(include_flat_names=True) + assert result.total_replacements == 1 + assert fs_manager.load_memory("docs") == "use mem:auth here" + + def test_skips_read_only_source_by_default(self, tmp_path) -> None: + manager = MemoriesManager(serena_data_folder=tmp_path, read_only_memory_patterns=[r"frozen/.*"]) + _write(manager, "frozen/notes", "the auth/login process") + _write(manager, "auth/login", "# notes") + result = manager.auto_prefix_bare_references() + assert result.total_replacements == 0 + assert len(result.skipped_read_only) == 1 + # content unchanged + assert manager.load_memory("frozen/notes") == "the auth/login process" + + def test_include_read_only_modifies_read_only_source(self, tmp_path) -> None: + manager = MemoriesManager(serena_data_folder=tmp_path, read_only_memory_patterns=[r"frozen/.*"]) + _write(manager, "frozen/notes", "the auth/login process") + _write(manager, "auth/login", "# notes") + result = manager.auto_prefix_bare_references(include_read_only=True) + assert result.total_replacements == 1 + assert manager.load_memory("frozen/notes") == "the mem:auth/login process" + + def test_multiple_occurrences_in_same_memory(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth/login", "# notes") + _write(fs_manager, "docs", "auth/login here and auth/login there") + result = fs_manager.auto_prefix_bare_references() + assert result.total_replacements == 2 + assert fs_manager.load_memory("docs") == "mem:auth/login here and mem:auth/login there" + + def test_marked_references_left_alone(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth/login", "# notes") + _write(fs_manager, "docs", "see `mem:auth/login` and also auth/login bare") + result = fs_manager.auto_prefix_bare_references() + # only the bare occurrence is rewritten; the already-marked one stays as-is + assert result.total_replacements == 1 + assert fs_manager.load_memory("docs") == "see `mem:auth/login` and also mem:auth/login bare" + + def test_dry_run_does_not_modify_files(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth/login", "# notes") + _write(fs_manager, "docs", "the auth/login process is documented here") + result = fs_manager.auto_prefix_bare_references(dry_run=True) + # report describes what *would* have been done + assert result.dry_run is True + assert result.total_replacements == 1 + assert len(result.autofixed) == 1 + # but the file on disk is unchanged + assert fs_manager.load_memory("docs") == "the auth/login process is documented here" + # running again non-dry actually rewrites + result = fs_manager.auto_prefix_bare_references() + assert result.dry_run is False + assert fs_manager.load_memory("docs") == "the mem:auth/login process is documented here" + + def test_fuzzy_warnings_are_routed_to_skipped_fuzzy(self, fs_manager: MemoriesManager) -> None: + # set up a fuzzy near-miss case (token shares all parts with a longer existing name) + existing = "adding_new_language_support_guide" + bare = "adding_new_language_support" + _write(fs_manager, existing, "# guide content") + _write(fs_manager, "docs", f"See {bare} for details") + result = fs_manager.auto_prefix_bare_references() + # fuzzy match must NOT be auto-rewritten — it would require substring substitution + assert result.total_replacements == 0 + assert len(result.skipped_fuzzy) == 1 + skipped = result.skipped_fuzzy[0] + assert skipped.actual_token == bare + assert skipped.suspected_name == existing + # source unchanged + assert fs_manager.load_memory("docs") == f"See {bare} for details" + + def test_no_double_prefix_on_repeated_runs(self, fs_manager: MemoriesManager) -> None: + _write(fs_manager, "auth/login", "# notes") + _write(fs_manager, "docs", "the auth/login process") + fs_manager.auto_prefix_bare_references() + second = fs_manager.auto_prefix_bare_references() + # idempotent: the second run should not touch anything + assert second.total_replacements == 0 + assert fs_manager.load_memory("docs") == "the mem:auth/login process" From bc9d4e050caf6d74075ef41193931a416c12baae Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 13 May 2026 16:10:18 +0200 Subject: [PATCH 03/20] Feature: added memory maintenance memory as part of onboarding Rephrased onboarding prompt to adjust to the recommended memory management --- .../generated/generated_prompt_factory.py | 42 ++++++------- src/serena/project.py | 37 +++++++++++ .../prompt_templates/simple_tool_outputs.yml | 61 +++++++++++-------- src/serena/resources/memory_maintenance.md | 40 ++++++++++++ src/serena/tools/workflow_tools.py | 5 +- 5 files changed, 134 insertions(+), 51 deletions(-) create mode 100644 src/serena/resources/memory_maintenance.md diff --git a/src/serena/generated/generated_prompt_factory.py b/src/serena/generated/generated_prompt_factory.py index a1ee2920..fedf4142 100644 --- a/src/serena/generated/generated_prompt_factory.py +++ b/src/serena/generated/generated_prompt_factory.py @@ -14,39 +14,31 @@ class PromptFactory(PromptFactoryBase): """ def get_info_jet_brains_debug_repl_template_string(self) -> str: - return self.get_prompt_template_string("info_jet_brains_debug_repl") - + return self.get_prompt_template_string('info_jet_brains_debug_repl') + def create_info_jet_brains_debug_repl(self) -> str: - return self._render_prompt("info_jet_brains_debug_repl", locals()) + return self._render_prompt('info_jet_brains_debug_repl', locals()) def get_onboarding_prompt_template_string(self) -> str: - return self.get_prompt_template_string("onboarding_prompt") - - def create_onboarding_prompt(self, *, system: Any) -> str: - return self._render_prompt("onboarding_prompt", locals()) + return self.get_prompt_template_string('onboarding_prompt') + + def create_onboarding_prompt(self, *, memory_maintenance_name: Any, system: Any) -> str: + return self._render_prompt('onboarding_prompt', locals()) def get_connection_prompt_template_string(self) -> str: - return self.get_prompt_template_string("connection_prompt") - + return self.get_prompt_template_string('connection_prompt') + def create_connection_prompt(self) -> str: - return self._render_prompt("connection_prompt", locals()) + return self._render_prompt('connection_prompt', locals()) def get_system_prompt_template_string(self) -> str: - return self.get_prompt_template_string("system_prompt") - - def create_system_prompt( - self, - *, - available_markers: Any, - available_tools: Any, - context_system_prompt: Any, - global_memories_list: Any, - mode_system_prompts: Any, - ) -> str: - return self._render_prompt("system_prompt", locals()) + return self.get_prompt_template_string('system_prompt') + + def create_system_prompt(self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, global_memories_list: Any, mode_system_prompts: Any) -> str: + return self._render_prompt('system_prompt', locals()) def get_cc_system_prompt_override_template_string(self) -> str: - return self.get_prompt_template_string("cc_system_prompt_override") - + return self.get_prompt_template_string('cc_system_prompt_override') + def create_cc_system_prompt_override(self) -> str: - return self._render_prompt("cc_system_prompt_override", locals()) + return self._render_prompt('cc_system_prompt_override', locals()) diff --git a/src/serena/project.py b/src/serena/project.py index dceb65a7..21c5b3e1 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -93,6 +93,43 @@ class MemoriesManager: name = cls._prepare_name(name) return cls._MEMORY_REF_PREFIX + name + MEMORY_MAINTENANCE_NAME: str = "memory_maintenance" + _MEMORY_MAINTENANCE_TEMPLATE_PATH: Path = Path(__file__).parent / "resources" / "memory_maintenance.md" + + def ensure_memory_maintenance_memory(self) -> str: + """ + Ensures a memory describing how memories should be maintained exists for this project, + and returns the name to reference it by. + + Precedence: + + 1. If a global copy exists at ``global/memory_maintenance``, return that name; no + project copy is created (the global version takes precedence). + 2. Else if a project copy already exists, return its name unchanged. + 3. Else seed a project copy from the package-shipped template and return that name. + + Existing memory files are never overwritten; users may have customized them. To + refresh from the shipped template, delete the existing memory first. + + :return: the bare name to reference the maintenance memory by (without the ``mem:`` + prefix); either ``"global/memory_maintenance"`` or ``"memory_maintenance"``. + :raises FileNotFoundError: if the shipped template is missing on disk. + :raises AssertionError: if this manager has no associated project directory. + """ + global_name = f"{self.GLOBAL_TOPIC}/{self.MEMORY_MAINTENANCE_NAME}" + if self.get_memory_file_path(global_name).exists(): + return global_name + if self.get_memory_file_path(self.MEMORY_MAINTENANCE_NAME).exists(): + return self.MEMORY_MAINTENANCE_NAME + + # seed a project copy from the shipped template + template_path = self._MEMORY_MAINTENANCE_TEMPLATE_PATH + if not template_path.exists(): + raise FileNotFoundError(f"Memory maintenance template not found at {template_path}") + content = template_path.read_text(encoding=self._encoding) + self.save_memory(self.MEMORY_MAINTENANCE_NAME, content, is_tool_context=False) + return self.MEMORY_MAINTENANCE_NAME + def rename_references_to_memory(self, content: str, old_name: str, new_name: str) -> tuple[str, int]: r""" Replaces all occurrences of a memory reference (e.g. ``mem:foo``) in ``content`` with diff --git a/src/serena/resources/config/prompt_templates/simple_tool_outputs.yml b/src/serena/resources/config/prompt_templates/simple_tool_outputs.yml index c6f2c619..168840ff 100644 --- a/src/serena/resources/config/prompt_templates/simple_tool_outputs.yml +++ b/src/serena/resources/config/prompt_templates/simple_tool_outputs.yml @@ -4,33 +4,44 @@ prompts: onboarding_prompt: | You are viewing the project for the first time. - Your task is to assemble relevant high-level information about the project which - will be saved to memory files in the following steps. - The information should be sufficient to understand what the project is about, - and the most important commands for developing code. + Your task is to assemble durable, non-obvious information about the project and write it + to memory files that future agents will consult. The project is being developed on the system: {{ system }}. - You need to identify at least the following information: - * the project's purpose - * the tech stack used - * the code style and conventions used (including naming, type hints, docstrings, etc.) - * which commands to run when a task is completed (linting, formatting, testing, etc.) - * the rough structure of the codebase - * the commands for testing, formatting, and linting - * the commands for running the entrypoints of the project - * the util commands for the system, like `git`, `ls`, `cd`, `grep`, `find`, etc. Keep in mind that the system is {{ system }}, - so the commands might be different than on a regular unix system. - * whether there are particular guidelines, styles, design patterns, etc. that one should know about + **Before writing anything, read `mem:{{ memory_maintenance_name }}`** using the `read_memory` tool. + It defines the style, naming, reference, and add/update threshold conventions that every + memory in this project must follow. Do not skip this step. - This list is not exhaustive, you can add more information if you think it is relevant. + Target memory layout — use the `write_memory` tool, one call per memory: - For doing that, you will need to acquire information about the project with the corresponding tools. - Read only the necessary files and directories to avoid loading too much data into memory. - If you cannot find everything you need from the project itself, you should ask the user for more information. + * `mem:core` — top-level routing/source map: a terse index pointing at the focused memories + below, plus only those project-wide invariants that don't belong in a focused memory. + * `mem:tech_stack` — language(s), framework(s), build tools, package manager, version + pins where they matter. + * `mem:suggested_commands` — project commands the user will actually run (dev, test, lint, + format, run entrypoints) and any system util commands (`git`, `ls`, `grep`, …) whose + form differs on {{ system }} from a standard unix shell. Skip generic commands that + behave identically across systems. + * `mem:conventions` — code style, naming, type hints, docstring conventions, design + patterns specific to this codebase. + * `mem:task_completion` — exact commands to run when a coding task is considered done + (linter, formatter, test runner, type checker, etc.). - After collecting all the information, you will use the `write_memory` tool (in multiple calls) to save it to various memory files. - A particularly important memory file will be the `suggested_commands.md` file, which should contain - a list of commands that the user should know about to develop code in this project. - Moreover, you should create memory files for the style and conventions and a dedicated memory file for - what should be done when a task is completed. - **Important**: after done with the onboarding task, remember to call the `write_memory` to save the collected information! + If the project has clearly distinct modules (e.g. frontend/backend), create per-module + `mem:/core` memories for module-specific routing instead of pushing everything + into `mem:core`. + + Acquire information using the available tools. Read only the files needed; do not load + entire directory trees. If essential information is missing, ask the user. + + Memory body style (full rules in `mem:{{ memory_maintenance_name }}`): dense agent notes — + terse bullets, invariants, paths, commands. No self-referential cues ("Use when…", + "Start here…", "Read before…"); routing belongs in `mem:core` upstream, not inside the + body it points to. References to other memories use `` `mem:NAME` `` (backticked, with + the `mem:` prefix). + + When all memories are written, sanity-check the references by mentioning that the user can run + `serena memories check` from the project root. + + **Important**: the onboarding is only complete once you have actually called `write_memory` + for each memory above — do not summarize the content in chat and skip writing. diff --git a/src/serena/resources/memory_maintenance.md b/src/serena/resources/memory_maintenance.md new file mode 100644 index 00000000..567d39c3 --- /dev/null +++ b/src/serena/resources/memory_maintenance.md @@ -0,0 +1,40 @@ +# Memory Maintenance + +## Discovery Model + +- Agents start with the full memory name list and should read `mem:core` first as the routing entry point. +- All other memory discovery comes from memory names plus routing already read. +- Routing is useful only before the target memory is selected. Once a memory is open, target-selection text inside it can no longer help decide to read it. +- Therefore, never put self-referential selection cues in a memory body: no "Start here", "Use when", "Use before", "Use after", or "read this when..." openers. + Such self-referential cues are an error mode: they are unreachable for selection, duplicate routing that belongs upstream, and drift independently from the actual routing graph. +- Add routing only when names are insufficient, ambiguous, or a non-obvious dependency/order matters. Do not add tautological routing such as "frontend debugging -> frontend/debugging". +- Routing can live outside core memories when needed, but core memories own most module-level routing because they are read before focused memories in that subfolder. +- Top-level/startup memory is `mem:core`; top-level memory in each memory subfolder is `/core` (for example `mem:frontend/core`). +- Memory references must use a mem: prefix inside backticks, e.g. `mem:frontend/core`. + +## Style + +Dense agent notes, not prose docs. Prefer invariants, paths, commands, terse bullets. Avoid obvious context, rationale, and examples unless they prevent likely mistakes. Keep guidance durable and generalizable, not task-local. + +## Add/update threshold + +Add or update memory only for stable, non-obvious project conventions that avoid complex rediscovery. + +Do not add: quick-read/rg facts; generic language/framework knowledge; one-off task notes; volatile line-level details; behavior likely to change soon. + +Operational constants (commands, paths, version pins): phrase as current behavior; before changing the value, update it in the owning memory and run `serena memories check` to catch references that need updating. + +## Graph Ownership + +- Project-level routing: `mem:core`. +- Module-level routing/source maps: module core memories. +- Focused memories: topic details only; assume relevant routing was read before selection. + +## Maintenance Actions + +- Renamed/removed/split/merged/superseded memory: update all references. Done automatically if handled via Serena's memory rename tool. +- Stale paths/namespaces/commands/behavior: update or remove in owning memory. If working with Serena, call `serena memories check` for a report. +- Split catch-all memory when sections are commonly needed independently or agents must read unrelated domains to reach needed topic. +- Retire/delete memory when owning module/subsystem is removed or replaced; remove all references. +- Added/renamed/split/moved/deleted memory: update affected routing only where names are insufficient. +- Before finishing: list memories for old names and key new names to catch stale/missing references. \ No newline at end of file diff --git a/src/serena/tools/workflow_tools.py b/src/serena/tools/workflow_tools.py index 461da9ea..7d5942d1 100644 --- a/src/serena/tools/workflow_tools.py +++ b/src/serena/tools/workflow_tools.py @@ -54,7 +54,10 @@ class OnboardingTool(Tool): if not write_memory_tool_available: return "Memory writing tool not activated, skipping onboarding." system = platform.system() - return self.prompt_factory.create_onboarding_prompt(system=system) + # seed the project-local memory-maintenance memory (or detect a global override) so + # the prompt can point the agent at the conventions before it writes anything + memory_maintenance_name = self.memories_manager.ensure_memory_maintenance_memory() + return self.prompt_factory.create_onboarding_prompt(system=system, memory_maintenance_name=memory_maintenance_name) class InitialInstructionsTool(Tool, ToolMarkerDoesNotRequireActiveProject): From be5b60bddbcfc1f075a4eda7b9f46e8ae3ec5185 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Thu, 14 May 2026 22:43:00 +0200 Subject: [PATCH 04/20] Make dotall optional in affected tools --- CHANGELOG.md | 8 ++++ src/serena/project.py | 16 +++++++- src/serena/tools/file_tools.py | 65 ++++++++------------------------ src/serena/tools/memory_tools.py | 11 +++++- src/serena/util/text_utils.py | 17 +++++++-- 5 files changed, 60 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 715a2c6f..3e05a886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,14 @@ Status of the `main` branch. Changes prior to the next official version change w - Modal markup cleanup: extract shared CSS classes (`.modal-info`, `.modal-hint`, `.modal-prompt`, `.modal-field`, `.modal-input`, `.modal-select`, `.modal-textarea`, `.modal-actions`, `.btn-secondary`) and remove duplicated inline styles from all seven modals. Inputs and textareas get an accent-colored focus ring; the modal backdrop has a subtle blur. - Add Language: replace the native `` with a filterable combobox — type to filter, keyboard navigation (Up/Down/Enter/Esc), substring highlight, click-outside to close. The typed value is validated against the available languages list before submission. +* Memories: + - Memories can now reference each other using the `` `mem:NAME` `` convention. Renames + propagate to all references automatically. See the [reference convention](https://oraios.github.io/serena/02-usage/045_memories.html#referencing-memories-from-other-memories). + - New `serena memories` CLI command group: `list`, `read`, `write`, `check` (referential + integrity report) and `auto-prefix-references` (heuristic rewrite of bare occurrences). + See the [CLI subcommands](https://oraios.github.io/serena/02-usage/045_memories.html#cli-subcommands). + - Onboarding now seeds a `memory_maintenance` memory describing the memory-style and + routing conventions, and the agent is instructed to read it before writing any other + memories. A `global/memory_maintenance` memory takes precedence over the per-project + seed. See the [memory maintenance section](https://oraios.github.io/serena/02-usage/045_memories.html#the-memory-maintenance-memory). + - Onboarding prompt rewritten to target a layout of `mem:core` (routing entry point), + `mem:suggested_commands`, `mem:tech_stack`, `mem:conventions`, `mem:task_completion`, + and optional per-module `mem:/core` memories, using dense agent-notes style. + * Tools: - `edit_memory`: New `dotall` parameter (default `False`) controlling whether ``.`` matches newlines in regex mode. **Behavior change**: previously DOTALL was forced on; the new default diff --git a/docs/02-usage/040_workflow.md b/docs/02-usage/040_workflow.md index d4482f49..cbf4310d 100644 --- a/docs/02-usage/040_workflow.md +++ b/docs/02-usage/040_workflow.md @@ -143,9 +143,17 @@ and to store memories, which it can then draw upon in future interactions. In general, **memories** provide a way for Serena to store and retrieve information about the project, relevant conventions, and other relevant aspects. +Memories may reference each other using the `` `mem:NAME` `` convention; references +are kept in sync across renames, and a `serena memories check` command is available +to report stale references. -For more information on this, including how to manage -or disable these features, see [Memories & Onboarding](045_memories). +During the first onboarding, Serena seeds a `memory_maintenance` memory describing the +conventions (style, references, routing) that subsequent memories should follow, and the +agent is instructed to read it before writing any project-specific memories. + +For more information on this, including the target memory layout, the `mem:` reference +convention, the `serena memories` CLI subcommands, and how to manage or disable these +features, see [Memories & Onboarding](045_memories). ## Preparing Your Project diff --git a/docs/02-usage/045_memories.md b/docs/02-usage/045_memories.md index 7b364aa9..b578ed3c 100644 --- a/docs/02-usage/045_memories.md +++ b/docs/02-usage/045_memories.md @@ -7,7 +7,7 @@ agent's internal memory management (e.g., `AGENTS.md` files). ## Memories Memories are simple, human-readable Markdown files that both you and -your agent can create, read, and edit. +your agent can create, read, reference, and edit. Serena differentiates between * **project-specific memories**, which are stored in the `.serena/memories/` directory within your project folder, and @@ -18,12 +18,76 @@ inferring appropriateness from the file name. When the agent starts working on a project, it receives the list of available memories. The agent should be instructed to update memories by the user when appropriate. +### Design Rationale + +Serena's memory system is intentionally minimal. It was designed to satisfy the following +criteria: + +1. **Human-readable and editable.** Memories must remain directly readable and editable + in any text editor. The agent is typically the day-to-day consumer, but a human author + or reviewer must always be able to step in without going through the agent. +2. **Versionable with the project.** Project memories live alongside the code and can + be committed, reviewed in PRs, and reverted like any other repository artifact. +3. **Progressive disclosure.** Agents receive the full memory *name list* up + front as part of their initial instructions; any further routing is described inside + the memory content itself - typically a `mem:core` entry point pointing at focused + memories. The agent decides what to read based on names plus the routing it has + already seen. +4. **Prefer routing to search.** Given an intelligent agent and well-structured routing, search is + unnecessary - and it adds noise: any retrieval method (lexical or semantic) + produces both false positives and false negatives. Explicit, name-based routing + decided by the agent is deterministic and avoids both error modes. Basic search via regex/grep + is sufficient to complement the routing when needed and is available to any agent. +5. **Prefer deliberate reads to triggers.** The agent decides what to read and when. The harness does + not inject memory content on the agent's behalf. +6. **Framework-agnostic.** The storage format is plain Markdown files in + a simple directory layout. The only Serena-specific convention is the `mem:` + prefix for references to memories, which does not prevent using the memory files outside + of Serena. +7. **Configurable and composable.** Two orthogonal memory scopes - + [per-project](#memories) (committed alongside the code) and [global](global-memories) + (shared across all your projects) - can be combined freely. Within either scope, + regex patterns in the global or project configuration can mark subsets as read-only + or [hide them entirely](#ignoring-memories) from the agent. This lets a project mix + personal cross-project knowledge with checked-in project conventions, and selectively + freeze either set, without custom plumbing. + +Taken together, these criteria rule out several common alternatives: + +- **Database-backed memory** (SQLite, graph databases, vector stores) is excluded by + criteria 1, 4, and 6. +- **`AGENTS.md` and similar single-file conventions** are excluded by criteria 3 and 5. +- **Hooks and harness-internal memory systems** are excluded by criteria 5 and 6. + +To our knowledge, no existing system satisfies our design goal, which is +why Serena ships its own memory layer rather than reusing one. +The closest existing approaches are in the family of Markdown-based personal knowledge +management tools - **Obsidian**, **Logseq**, **Foam**. + ### Organizing Memories Memories can be organized into **topics** by using `/` in the memory name (e.g. `modules/frontend`). The structure is mapped to the file system, where topics correspond to subdirectories. The `list_memories` tool can filter by topic, allowing the agent to explore even large numbers of memories in a structured way. +(memory-references)= +### Referencing Memories from Other Memories + +Memories may reference each other. Serena recognizes a reference as a memory name prefixed with +`mem:` and wrapped in backticks, for example `` `mem:auth/login` `` or `` `mem:suggested_commands` ``. +This convention has two practical consequences: + +- **Renames keep references intact.** When you rename or move a memory with the `rename_memory` + tool, Serena rewrites every `` `mem:OLD_NAME` `` occurrence across all memories to point to + the new name. References that do not use the `mem:` prefix will not be updated automatically. +- **Integrity checks** (see [below](memory-cli)) report any `` `mem:NAME` `` whose target does + not resolve to an existing memory, and propose similarly-named candidates as likely intended + targets. + +The full convention - including style, add/update thresholds, and how to structure routing across +`core` memories - is shipped to every onboarded project as the `memory_maintenance` memory; see the +[Onboarding section](onboarding) below. + (global-memories)= ### Global Memories @@ -49,7 +113,7 @@ global or project-level [configuration](050_configuration): ignored_memory_patterns: ["_archive/.*", "_episodes/.*"] ``` -Ignored memories are completely excluded — they cannot be accessed via `read_memory`, `write_memory`, +Ignored memories are completely excluded - they cannot be accessed via `read_memory`, `write_memory`, or any other memory tool. To read an ignored memory file, use the `read_file` tool on the raw file path (e.g., `.serena/memories/_archive/2026-03/some-topic.md`). @@ -66,8 +130,8 @@ viewing, creating, editing, and deleting memories while Serena is running. By default, Serena performs an **onboarding process** when it encounters a project for the first time (i.e., when no project memories exist yet). -The goal of the onboarding is for Serena to get familiar with the project — -its structure, build system, testing setup, and other essential aspects — +The goal of the onboarding is for Serena to get familiar with the project - +its structure, build system, testing setup, and other essential aspects - and to store this knowledge as memories for future interactions. In further project activations, Serena will check whether onboarding was already @@ -80,7 +144,32 @@ process if memories are found. performed (by checking if any memories exist). 2. If no memories are found, Serena triggers the onboarding process, which reads key files and directories to understand the project. -3. The gathered information is written into project-specific memory files (see above). +3. Before any project memory is written, Serena materializes a project-local + `memory_maintenance` memory (see below). The agent is then instructed to read it + first and follow the conventions it describes. +4. The gathered information is written into project-specific memory files following + the onboarding prompt instructions and the conventions outlined in `memory_maintenance`. + +(memory-maintenance-memory)= +### The `memory_maintenance` Memory + +To make memory conventions discoverable to both the LLM and the user, Serena seeds +a `memory_maintenance` memory on first onboarding. The seed is copied from a template +shipped with the Serena package and contains the dense agent-notes style, the +`mem:` reference convention, the routing model around `core` memories, the +add/update threshold, and the maintenance actions (rename / delete / split). + +The seeding follows a strict precedence: + +1. If you already maintain a `global/memory_maintenance` memory, Serena uses that + and **does not** create a project-local copy. This is the recommended approach + for teams that want one shared convention document across all projects. +2. Otherwise, if the project already has a `memory_maintenance` memory, it is left + untouched. +3. Otherwise, the shipped template is written to `.serena/memories/memory_maintenance.md`. + +Existing files are never overwritten - you can freely customize the project copy. +To refresh from the shipped template, delete the existing memory first. ### Tips for Onboarding @@ -92,6 +181,29 @@ process if memories are found. - **Review the results**: After onboarding, we recommend having a quick look at the generated memories and editing them or adding new ones as needed. +(memory-cli)= +### CLI Subcommands + +While the recommended way to manage memories is through the **MCP integration**, +Serena also offers memory-related CLI commands. + +The following commands have **no MCP tool counterpart** and are intended for human execution: + +- `serena memories check` — referential-integrity report. By default reports only stale + `` `mem:NAME` `` references whose target does not resolve. Pass `--include-unmarked` to + also report bare occurrences of existing memory names found through a heuristic. +- `serena memories auto-prefix-references` — heuristic rewrite of bare occurrences to add + the `mem:` prefix; supports `--dry-run`. +- `serena memories initialize` will seed the `memory_maintenance` memory for the project. + +The remaining commands mirror the MCP tools, you can thus instruct your agent to manage memories with +serena without having a running MCP server. Discover the full surface and per-command flags via: + +```shell +serena memories --help +serena memories --help +``` + ## Disabling Memories and Onboarding If you do not require the functionality described in this section, you can selectively disable it. diff --git a/src/serena/cli.py b/src/serena/cli.py index 86fc4d5f..ae7a2380 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -9,7 +9,7 @@ import time from collections.abc import Iterator, Sequence from logging import Logger from pathlib import Path -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal import click from sensai.util import logging @@ -43,6 +43,9 @@ from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind from solidlsp.util.subprocess_util import subprocess_kwargs +if TYPE_CHECKING: + from serena.project import MemoriesManager + log = logging.getLogger(__name__) _MAX_CONTENT_WIDTH = 200 @@ -1124,14 +1127,43 @@ class MemoryCommands(AutoRegisteringGroup): ) @staticmethod - def _load_memories_manager(project: str): - """Load the active project and return its memories manager.""" + def _load_memories_manager(project: str) -> "MemoriesManager": + """ + Load the project at ``project`` and return its memories manager. + + Requires the project to already be a Serena project (``.serena/project.yml`` must + exist at the configured location). Never auto-creates the ``.serena`` directory — + if no project configuration is found, the user is directed at ``serena project create``. + """ from serena.project import Project serena_config = SerenaConfig.from_config_file() - proj = Project.load(os.path.abspath(project), serena_config=serena_config) + project_path = os.path.abspath(project) + try: + proj = Project.load(project_path, serena_config=serena_config, autogenerate=False) + except FileNotFoundError as e: + raise click.UsageError( + f"{e}\nNo Serena project found at {project_path}. Create one first with:\n serena project create {project_path}" + ) from e return proj.memories_manager + @staticmethod + @click.command( + "initialize", + help=( + "Initialize this project's memory layout by seeding the `memory_maintenance` memory. " + "Requires the project to already exist as a Serena project (run `serena project create` first); " + "this command does not create a `.serena` directory on its own. " + "If a `global/memory_maintenance` memory exists, it takes precedence and no project copy is created." + ), + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + def initialize(project: str) -> None: + manager = MemoryCommands._load_memories_manager(project) + name = manager.ensure_memory_maintenance_memory() + click.echo(f"Memory maintenance memory ready: `mem:{name}`.") + @staticmethod @click.command( "list", @@ -1184,20 +1216,114 @@ class MemoryCommands(AutoRegisteringGroup): manager = MemoryCommands._load_memories_manager(project) click.echo(manager.save_memory(memory_name, content, is_tool_context=False)) + @staticmethod + @click.command( + "delete", + help="Delete a memory file. Use the `global/` prefix to address a global memory (e.g. `global/style_guide`).", + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.argument("memory_name", type=str) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + def delete(memory_name: str, project: str) -> None: + manager = MemoryCommands._load_memories_manager(project) + click.echo(manager.delete_memory(memory_name, is_tool_context=False)) + + @staticmethod + @click.command( + "rename", + help=( + "Rename or move a memory and update every `mem:OLD_NAME` reference across all memories. " + "Use `/` in the name to organize into topics; use the `global/` prefix to address a " + "global memory. Moving between project and global scope is supported." + ), + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.argument("old_name", type=str) + @click.argument("new_name", type=str) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + def rename(old_name: str, new_name: str, project: str) -> None: + manager = MemoryCommands._load_memories_manager(project) + message, n_refs = manager.rename_memory_and_propagate_references(old_name, new_name, is_tool_context=False) + click.echo(message) + if n_refs > 0: + click.echo(f"Updated {n_refs} `mem:` reference occurrence(s) across the memory graph.") + + @staticmethod + @click.command( + "edit", + help=( + "Replace content matching a pattern in a memory. By default operates in literal " + "(non-regex) mode and refuses to replace more than one occurrence; pass --mode regex " + "to enable regex matching and --allow-multiple-occurrences to permit multiple hits." + ), + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.argument("memory_name", type=str) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + @click.option("--needle", type=str, required=True, help="The text to search for (literal by default; regex if --mode=regex).") + @click.option("--repl", type=str, required=True, help="The replacement text (verbatim).") + @click.option( + "--mode", + type=click.Choice(["literal", "regex"]), + default="literal", + show_default=True, + help="Treat --needle as literal text or as a Python regex (MULTILINE flag is always on; DOTALL via --dotall).", + ) + @click.option( + "--allow-multiple-occurrences", + is_flag=True, + default=False, + help="Permit and apply multiple matches; without this, multiple matches raise an error.", + ) + @click.option( + "--dotall", + is_flag=True, + default=False, + help="Regex mode only: make `.` match newlines as well, enabling multi-line patterns. Use with care.", + ) + def edit( + memory_name: str, + project: str, + needle: str, + repl: str, + mode: Literal["literal", "regex"], + allow_multiple_occurrences: bool, + dotall: bool, + ) -> None: + manager = MemoryCommands._load_memories_manager(project) + click.echo( + manager.edit_memory( + memory_name, + needle, + repl, + mode, + allow_multiple_occurrences, + is_tool_context=False, + dotall=dotall, + ) + ) + @staticmethod @click.command( "check", help=( "Check referential integrity across all memories of the project (and global memories). " - "Reports stale `mem:` references and unmarked-reference warnings split into high- and " - "low-confidence groups. Read-only and never writes. Always exits 0." + "Reports stale `mem:` references. With --include-unmarked, also reports bare occurrences " + "of existing memory names that lack the `mem:` prefix (split into high- and low-confidence). " + "Read-only and never writes. Always exits 0." ), context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, ) @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) - def check(project: str) -> None: + @click.option( + "--include-unmarked", + is_flag=True, + default=False, + help="Also report potentially unmarked references (bare occurrences of memory names without the `mem:` prefix).", + ) + def check(project: str, include_unmarked: bool) -> None: manager = MemoryCommands._load_memories_manager(project) - report = manager.validate_referential_integrity() + report = manager.validate_referential_integrity(include_unmarked=include_unmarked) click.echo(report.format()) @staticmethod diff --git a/src/serena/project.py b/src/serena/project.py index 72cc33c7..21bea043 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -166,6 +166,9 @@ class MemoriesManager: # generic prose word like "repository" is a substring of "serena_repository_structure" # but their token Jaccard is only 1/3, so we reject it at this floor. _FUZZY_BARE_TOKEN_JACCARD_FLOOR: float = 0.5 + + _WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: frozenset[str] = frozenset({"core"}) + """Names that we specifically mention in our instructions and that coincide with common words""" _VERSION_SUFFIX_PATTERN = re.compile(r"_(?:v\d+|old|new|legacy|bak)$", re.IGNORECASE) @classmethod @@ -507,19 +510,19 @@ class MemoriesManager: Outcome of :meth:`MemoriesManager.validate_referential_integrity`. :ivar stale_references: ``mem:NAME`` references whose target memory does not exist. - :ivar high_confidence_warnings: bare references whose suspected target name is unlikely + :ivar high_confidence_unmarked_memories: bare references whose suspected target name is unlikely to be coincidental prose (topic-path or sufficiently long). - :ivar low_confidence_warnings: bare references whose suspected target name could plausibly + :ivar low_confidence_unmarked_memories: bare references whose suspected target name could plausibly appear in ordinary prose (short, flat names). """ stale_references: list["MemoriesManager.StaleReference"] = field(default_factory=list) - high_confidence_warnings: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) - low_confidence_warnings: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) + high_confidence_unmarked_memories: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) + low_confidence_unmarked_memories: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) def is_clean(self) -> bool: """:return: True iff no stale references and no warnings of any confidence level were found.""" - return not (self.stale_references or self.high_confidence_warnings or self.low_confidence_warnings) + return not (self.stale_references or self.high_confidence_unmarked_memories or self.low_confidence_unmarked_memories) def format(self) -> str: """:return: a human-readable rendering suitable for CLI display.""" @@ -543,8 +546,8 @@ class MemoriesManager: # unmarked-reference warnings, split by confidence for label, warnings in ( - ("High-confidence unmarked references", self.high_confidence_warnings), - ("Low-confidence unmarked references", self.low_confidence_warnings), + ("High-confidence unmarked references", self.high_confidence_unmarked_memories), + ("Low-confidence unmarked references", self.low_confidence_unmarked_memories), ): if not warnings: continue @@ -714,6 +717,31 @@ class MemoriesManager: return f"Memory renamed from {old_name} to {new_name}." + def rename_memory_and_propagate_references(self, old_name: str, new_name: str, is_tool_context: bool) -> tuple[str, int]: + """ + Renames a memory and updates every ``mem:OLD_NAME`` reference across all memories. + + Memories whose content does not contain a reference to ``old_name`` are left + untouched (no spurious mtime changes). Memories that do are rewritten via + :meth:`save_memory`. + + :param old_name: the current memory name (the source of the rename) + :param new_name: the target memory name + :param is_tool_context: forwarded to :meth:`save_memory` for read-only enforcement + :return: a tuple of (rename message returned by :meth:`move_memory`, total number of + ``mem:`` reference occurrences rewritten across all memories). + """ + renaming_message = self.move_memory(old_name, new_name, is_tool_context=is_tool_context) + + total_updates = 0 + for memory_name in self.list_memories().get_full_list(): + content = self.load_memory(memory_name) + updated_content, n_replacements = self.rename_references_to_memory(content, old_name, new_name) + if n_replacements > 0: + self.save_memory(memory_name, updated_content, is_tool_context=is_tool_context) + total_updates += n_replacements + return renaming_message, total_updates + def edit_memory( self, name: str, @@ -760,7 +788,7 @@ class MemoriesManager: source_basename = source_memory.rsplit("/", 1)[-1] return suspected_name == source_basename - def validate_referential_integrity(self) -> "MemoriesManager.ReferentialIntegrityReport": + def validate_referential_integrity(self, include_unmarked: bool = True) -> "MemoriesManager.ReferentialIntegrityReport": """ Scans every (non-ignored) memory's content for referential integrity issues. @@ -773,7 +801,8 @@ class MemoriesManager: * **exact unmarked-reference warnings** — bare occurrences of an existing memory name that appear without the ``mem:`` prefix. Split into high-confidence (the suspected name contains a ``/`` or exceeds :attr:`_HIGH_CONFIDENCE_NAME_LENGTH`) - and low-confidence groups. + and low-confidence groups. Candidate names whose basename matches + :attr:`WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES` are skipped. * **fuzzy near-miss warnings** — long, distinctive bare tokens in a memory body that *do not* match an existing name exactly but similarity-match a high-confidence existing name AND share at least :attr:`_FUZZY_BARE_TOKEN_JACCARD_FLOOR` @@ -787,6 +816,8 @@ class MemoriesManager: This method has no side effects. + :param include_unmarked: if False, skip the two unmarked-reference scans (exact and + fuzzy near-miss) entirely; only stale references are reported. :return: a :class:`ReferentialIntegrityReport` summarizing all findings. """ report = MemoriesManager.ReferentialIntegrityReport() @@ -801,6 +832,10 @@ class MemoriesManager: high_confidence_names = [n for n in all_names if "/" in n or len(n) >= self._HIGH_CONFIDENCE_NAME_LENGTH] for source_name in all_names: + # content of this memory should not be analyzed regarding stale references + if source_name == self.MEMORY_MAINTENANCE_NAME: + continue + content = self.load_memory(source_name) if not content: # skip empty memories silently — no references can exist in empty content @@ -823,10 +858,16 @@ class MemoriesManager: ) ) + if not include_unmarked: + continue + # exact unmarked references: bare occurrences of any existing memory name (except self) for candidate_name in all_names: if self._is_self_reference(source_name, candidate_name): continue + # skip candidate names whose basename is an ambiguous common word + if candidate_name.rsplit("/", 1)[-1] in self._WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: + continue n = self._find_bare_occurrences(content, candidate_name) if n == 0: continue @@ -840,9 +881,9 @@ class MemoriesManager: actual_token=candidate_name, ) if is_high_conf: - report.high_confidence_warnings.append(warning) + report.high_confidence_unmarked_memories.append(warning) else: - report.low_confidence_warnings.append(warning) + report.low_confidence_unmarked_memories.append(warning) # fuzzy near-misses: long bare tokens that similarity-match a high-confidence existing name for token, n in self._iter_long_bare_tokens(content): @@ -855,6 +896,8 @@ class MemoriesManager: for existing in high_confidence_names: if self._is_self_reference(source_name, existing): continue + if existing.rsplit("/", 1)[-1] in self._WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: + continue score = self.compute_name_similarity(token, existing) if score < self.NAME_SIMILARITY_THRESHOLD: continue @@ -873,7 +916,7 @@ class MemoriesManager: best_name = existing if best_name is None: continue - report.high_confidence_warnings.append( + report.high_confidence_unmarked_memories.append( MemoriesManager.UnmarkedReferenceWarning( source_memory=source_name, suspected_name=best_name, @@ -933,11 +976,11 @@ class MemoriesManager: validation = self.validate_referential_integrity() # combine the warning groups according to the include_flat_names policy - warnings: list[MemoriesManager.UnmarkedReferenceWarning] = list(validation.high_confidence_warnings) + warnings: list[MemoriesManager.UnmarkedReferenceWarning] = list(validation.high_confidence_unmarked_memories) if include_flat_names: - warnings.extend(validation.low_confidence_warnings) + warnings.extend(validation.low_confidence_unmarked_memories) else: - report.skipped_flat.extend(validation.low_confidence_warnings) + report.skipped_flat.extend(validation.low_confidence_unmarked_memories) # apply replacements per source memory, grouping warnings to avoid re-reading content warnings_by_source: dict[str, list[MemoriesManager.UnmarkedReferenceWarning]] = {} diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index 8c2de38e..b7709a4a 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -82,13 +82,11 @@ class RenameMemoryTool(Tool, ToolMarkerCanEdit): The "global" topic should only be used if explicitly instructed. References to other memories that are marked with the `mem:` prefix will be updated accordingly. """ - renaming_message = self.memories_manager.move_memory(old_name, new_name, is_tool_context=True) - for memory in self.memories_manager.list_memories().get_full_list(): - memory_content = self.memories_manager.load_memory(memory) - updated_content, n_replacements = self.memories_manager.rename_references_to_memory(memory_content, old_name, new_name) - if n_replacements > 0: - log.info(f"Updated {n_replacements} references to memory {old_name} to {new_name}") - self.memories_manager.save_memory(memory, updated_content, is_tool_context=True) + renaming_message, n_references_updated = self.memories_manager.rename_memory_and_propagate_references( + old_name, new_name, is_tool_context=True + ) + if n_references_updated > 0: + log.info(f"Updated {n_references_updated} references to memory {old_name} to {new_name}") return renaming_message diff --git a/test/serena/test_memories_manager.py b/test/serena/test_memories_manager.py index 991feb98..0ff428eb 100644 --- a/test/serena/test_memories_manager.py +++ b/test/serena/test_memories_manager.py @@ -278,21 +278,21 @@ class TestValidateReferentialIntegrity: _write(fs_manager, "auth/login", "# notes") _write(fs_manager, "docs", "the auth/login process is documented elsewhere") report = fs_manager.validate_referential_integrity() - assert len(report.high_confidence_warnings) == 1 - warning = report.high_confidence_warnings[0] + assert len(report.high_confidence_unmarked_memories) == 1 + warning = report.high_confidence_unmarked_memories[0] assert warning.suspected_name == "auth/login" assert warning.is_high_confidence is True assert warning.occurrences == 1 - assert report.low_confidence_warnings == [] + assert report.low_confidence_unmarked_memories == [] def test_low_confidence_unmarked_warning_for_short_flat_name(self, fs_manager: MemoriesManager) -> None: _write(fs_manager, "auth", "# notes") _write(fs_manager, "docs", "you must use auth here") report = fs_manager.validate_referential_integrity() - assert report.high_confidence_warnings == [] - assert len(report.low_confidence_warnings) == 1 - assert report.low_confidence_warnings[0].suspected_name == "auth" - assert report.low_confidence_warnings[0].is_high_confidence is False + assert report.high_confidence_unmarked_memories == [] + assert len(report.low_confidence_unmarked_memories) == 1 + assert report.low_confidence_unmarked_memories[0].suspected_name == "auth" + assert report.low_confidence_unmarked_memories[0].is_high_confidence is False def test_long_flat_name_is_high_confidence(self, fs_manager: MemoriesManager) -> None: # a flat name >= _HIGH_CONFIDENCE_NAME_LENGTH chars is unlikely to be coincidental prose @@ -301,7 +301,7 @@ class TestValidateReferentialIntegrity: _write(fs_manager, long_name, "# notes") _write(fs_manager, "docs", f"the {long_name} is documented here") report = fs_manager.validate_referential_integrity() - assert len(report.high_confidence_warnings) == 1 + assert len(report.high_confidence_unmarked_memories) == 1 def test_fuzzy_near_miss_rejects_substring_only_matches(self, fs_manager: MemoriesManager) -> None: """ @@ -328,8 +328,8 @@ class TestValidateReferentialIntegrity: ) report = fs_manager.validate_referential_integrity() assert report.is_clean(), ( - f"Expected no findings; got high={report.high_confidence_warnings}, " - f"low={report.low_confidence_warnings}, stale={report.stale_references}" + f"Expected no findings; got high={report.high_confidence_unmarked_memories}, " + f"low={report.low_confidence_unmarked_memories}, stale={report.stale_references}" ) def test_long_flat_name_near_miss_is_high_confidence(self, fs_manager: MemoriesManager) -> None: @@ -350,12 +350,12 @@ class TestValidateReferentialIntegrity: _write(fs_manager, "docs", f"See {bare} for details") report = fs_manager.validate_referential_integrity() - assert len(report.high_confidence_warnings) == 1, ( + assert len(report.high_confidence_unmarked_memories) == 1, ( f"expected one high-confidence warning for the near-miss reference, " - f"got {len(report.high_confidence_warnings)} " - f"(high={report.high_confidence_warnings}, low={report.low_confidence_warnings})" + f"got {len(report.high_confidence_unmarked_memories)} " + f"(high={report.high_confidence_unmarked_memories}, low={report.low_confidence_unmarked_memories})" ) - warning = report.high_confidence_warnings[0] + warning = report.high_confidence_unmarked_memories[0] assert warning.suspected_name == existing assert warning.source_memory == "docs" @@ -367,7 +367,7 @@ class TestValidateReferentialIntegrity: # `auth/login` mentioning its own basename `login` should not produce a warning; # `auth` mentioned in `auth/login` content also shouldn't (it's the prefix), and only # actual matches matter — assert no warning concerns the source itself - for w in report.high_confidence_warnings + report.low_confidence_warnings: + for w in report.high_confidence_unmarked_memories + report.low_confidence_unmarked_memories: assert not (w.source_memory == "auth/login" and w.suspected_name in ("login", "auth/login")) def test_marked_reference_does_not_produce_unmarked_warning(self, fs_manager: MemoriesManager) -> None: @@ -382,8 +382,8 @@ class TestValidateReferentialIntegrity: _write(fs_manager, "docs", "we have `mem:auth/login` plus a bare auth/login mention") report = fs_manager.validate_referential_integrity() # only the bare occurrence should be counted - assert len(report.high_confidence_warnings) == 1 - assert report.high_confidence_warnings[0].occurrences == 1 + assert len(report.high_confidence_unmarked_memories) == 1 + assert report.high_confidence_unmarked_memories[0].occurrences == 1 def test_empty_memory_skipped(self, fs_manager: MemoriesManager) -> None: _write(fs_manager, "auth/login", "") @@ -397,7 +397,7 @@ class TestValidateReferentialIntegrity: _write(manager, "frozen/notes", "see bare auth/login mention") _write(manager, "auth/login", "# notes") report = manager.validate_referential_integrity() - warnings = report.high_confidence_warnings + warnings = report.high_confidence_unmarked_memories assert len(warnings) == 1 assert warnings[0].source_memory == "frozen/notes" assert warnings[0].source_is_read_only is True From d4c460583b5912abb63b91e0a5f18e5008ea0fdf Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 13 May 2026 18:52:25 +0200 Subject: [PATCH 06/20] Improve onboarding and memory maintenance wording --- .../config/prompt_templates/simple_tool_outputs.yml | 10 ++-------- src/serena/resources/memory_maintenance.md | 2 ++ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/serena/resources/config/prompt_templates/simple_tool_outputs.yml b/src/serena/resources/config/prompt_templates/simple_tool_outputs.yml index 168840ff..9f5e058d 100644 --- a/src/serena/resources/config/prompt_templates/simple_tool_outputs.yml +++ b/src/serena/resources/config/prompt_templates/simple_tool_outputs.yml @@ -14,8 +14,8 @@ prompts: Target memory layout — use the `write_memory` tool, one call per memory: - * `mem:core` — top-level routing/source map: a terse index pointing at the focused memories - below, plus only those project-wide invariants that don't belong in a focused memory. + * `mem:core` — top-level source map and project-wide invariants that don't belong in a + focused memory. * `mem:tech_stack` — language(s), framework(s), build tools, package manager, version pins where they matter. * `mem:suggested_commands` — project commands the user will actually run (dev, test, lint, @@ -34,12 +34,6 @@ prompts: Acquire information using the available tools. Read only the files needed; do not load entire directory trees. If essential information is missing, ask the user. - Memory body style (full rules in `mem:{{ memory_maintenance_name }}`): dense agent notes — - terse bullets, invariants, paths, commands. No self-referential cues ("Use when…", - "Start here…", "Read before…"); routing belongs in `mem:core` upstream, not inside the - body it points to. References to other memories use `` `mem:NAME` `` (backticked, with - the `mem:` prefix). - When all memories are written, sanity-check the references by mentioning that the user can run `serena memories check` from the project root. diff --git a/src/serena/resources/memory_maintenance.md b/src/serena/resources/memory_maintenance.md index 567d39c3..cfb4c05c 100644 --- a/src/serena/resources/memory_maintenance.md +++ b/src/serena/resources/memory_maintenance.md @@ -11,6 +11,8 @@ - Routing can live outside core memories when needed, but core memories own most module-level routing because they are read before focused memories in that subfolder. - Top-level/startup memory is `mem:core`; top-level memory in each memory subfolder is `/core` (for example `mem:frontend/core`). - Memory references must use a mem: prefix inside backticks, e.g. `mem:frontend/core`. +- Do not add an index that simply re-lists the focused-memory names, the memory name list is already visible. + Only add routing pointers when a target memory's name is not self-explanatory or when a some aspect should be routed to multiple memories. ## Style From 04198c2a5e9418ce30fdffc34b575636b804f1bf Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 13 May 2026 18:53:12 +0200 Subject: [PATCH 07/20] Removed check_onboarding_performed, instead add a hint to activation message --- src/serena/agent.py | 3 ++ .../generated/generated_prompt_factory.py | 40 +++++++++++-------- .../resources/config/modes/no-memories.yml | 1 - .../resources/config/modes/no-onboarding.yml | 1 - src/serena/tools/tools_base.py | 1 + src/serena/tools/workflow_tools.py | 33 +-------------- .../typescript/test_repo/.serena/project.yml | 1 - 7 files changed, 29 insertions(+), 51 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index db67577d..5b1543fb 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -47,6 +47,7 @@ from serena.task_executor import TaskExecutor from serena.tools import ( ActivateProjectTool, GetCurrentConfigTool, + OnboardingTool, OpenDashboardTool, ReadMemoryTool, ReplaceContentTool, @@ -981,6 +982,8 @@ class SerenaAgent: f"\n{json.dumps(project_memories.to_dict())}\n" + "Use the `read_memory` tool to read these memories later if they are relevant to the task." ) + elif self._active_tools.contains_tool_class(OnboardingTool): + msg += "Onboarding has not been performed yet, you should call Serena's `onboarding` tool now to set up project memories." # add prompts for modes that were dynamically activated by the project modes_with_prompts = self._project_prompt_status.get_modes_with_prompts_to_be_provided_for_project_activation(session_id) diff --git a/src/serena/generated/generated_prompt_factory.py b/src/serena/generated/generated_prompt_factory.py index fedf4142..2a43e586 100644 --- a/src/serena/generated/generated_prompt_factory.py +++ b/src/serena/generated/generated_prompt_factory.py @@ -14,31 +14,39 @@ class PromptFactory(PromptFactoryBase): """ def get_info_jet_brains_debug_repl_template_string(self) -> str: - return self.get_prompt_template_string('info_jet_brains_debug_repl') - + return self.get_prompt_template_string("info_jet_brains_debug_repl") + def create_info_jet_brains_debug_repl(self) -> str: - return self._render_prompt('info_jet_brains_debug_repl', locals()) + return self._render_prompt("info_jet_brains_debug_repl", locals()) def get_onboarding_prompt_template_string(self) -> str: - return self.get_prompt_template_string('onboarding_prompt') - + return self.get_prompt_template_string("onboarding_prompt") + def create_onboarding_prompt(self, *, memory_maintenance_name: Any, system: Any) -> str: - return self._render_prompt('onboarding_prompt', locals()) + return self._render_prompt("onboarding_prompt", locals()) def get_connection_prompt_template_string(self) -> str: - return self.get_prompt_template_string('connection_prompt') - + return self.get_prompt_template_string("connection_prompt") + def create_connection_prompt(self) -> str: - return self._render_prompt('connection_prompt', locals()) + return self._render_prompt("connection_prompt", locals()) def get_system_prompt_template_string(self) -> str: - return self.get_prompt_template_string('system_prompt') - - def create_system_prompt(self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, global_memories_list: Any, mode_system_prompts: Any) -> str: - return self._render_prompt('system_prompt', locals()) + return self.get_prompt_template_string("system_prompt") + + def create_system_prompt( + self, + *, + available_markers: Any, + available_tools: Any, + context_system_prompt: Any, + global_memories_list: Any, + mode_system_prompts: Any, + ) -> str: + return self._render_prompt("system_prompt", locals()) def get_cc_system_prompt_override_template_string(self) -> str: - return self.get_prompt_template_string('cc_system_prompt_override') - + return self.get_prompt_template_string("cc_system_prompt_override") + def create_cc_system_prompt_override(self) -> str: - return self._render_prompt('cc_system_prompt_override', locals()) + return self._render_prompt("cc_system_prompt_override", locals()) diff --git a/src/serena/resources/config/modes/no-memories.yml b/src/serena/resources/config/modes/no-memories.yml index 08602141..d4175eef 100644 --- a/src/serena/resources/config/modes/no-memories.yml +++ b/src/serena/resources/config/modes/no-memories.yml @@ -9,4 +9,3 @@ excluded_tools: - rename_memory - list_memories - onboarding - - check_onboarding_performed diff --git a/src/serena/resources/config/modes/no-onboarding.yml b/src/serena/resources/config/modes/no-onboarding.yml index 4b893eff..d441b5d3 100644 --- a/src/serena/resources/config/modes/no-onboarding.yml +++ b/src/serena/resources/config/modes/no-onboarding.yml @@ -3,4 +3,3 @@ prompt: | The onboarding process is not applied. excluded_tools: - onboarding - - check_onboarding_performed diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 565a4055..33b47d15 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -528,6 +528,7 @@ class ToolRegistry: "summarize_changes", "think_about_whether_you_are_done", "switch_modes", + "check_onboarding_performed", ] def __init__(self) -> None: diff --git a/src/serena/tools/workflow_tools.py b/src/serena/tools/workflow_tools.py index 7d5942d1..18ed224b 100644 --- a/src/serena/tools/workflow_tools.py +++ b/src/serena/tools/workflow_tools.py @@ -4,38 +4,7 @@ Tools supporting the general workflow of the agent import platform -from serena.tools import ReadMemoryTool, Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional, WriteMemoryTool - - -class CheckOnboardingPerformedTool(Tool): - """ - Checks whether project onboarding was already performed. - """ - - def apply(self) -> str: - """ - Checks whether project onboarding was already performed. - You should always call this tool before beginning to actually work on the project/after activating a project. - """ - read_memory_tool_available = self.agent.tool_is_exposed(ReadMemoryTool.get_name_from_cls()) - perform_onboarding_tool_available = self.agent.tool_is_exposed(OnboardingTool.get_name_from_cls()) - - if not read_memory_tool_available: - return "Memory reading tool not activated, skipping onboarding check." - project_memories = self.memories_manager.list_project_memories() - if len(project_memories) == 0: - msg = "Onboarding not performed yet (no memories available). " - if perform_onboarding_tool_available: - msg += "You should perform onboarding by calling the `onboarding` tool before proceeding with the task. " - else: - # Not reporting the list of memories here, as they were already reported at project activation - # (with the system prompt if the project was activated at startup) - msg = ( - f"Onboarding was already performed: {len(project_memories)} project memories are available. " - "Consider reading memories if they appear relevant to the task at hand." - ) - msg += " If you have not read the 'Serena Instructions Manual', do so now." - return msg +from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional, WriteMemoryTool class OnboardingTool(Tool): diff --git a/test/resources/repos/typescript/test_repo/.serena/project.yml b/test/resources/repos/typescript/test_repo/.serena/project.yml index 942b95a1..090b7b46 100644 --- a/test/resources/repos/typescript/test_repo/.serena/project.yml +++ b/test/resources/repos/typescript/test_repo/.serena/project.yml @@ -7,7 +7,6 @@ # execute `uv run scripts/print_tool_overview.py`. # # * `activate_project`: Activates a project by name. -# * `check_onboarding_performed`: Checks whether project onboarding was already performed. # * `create_text_file`: Creates/overwrites a file in the project directory. # * `delete_lines`: Deletes a range of lines within a file. # * `delete_memory`: Deletes a memory from Serena's project-specific memory store. From 342b9b0382c6bb14e29ab426030c10edc6a74ac9 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 13 May 2026 21:57:11 +0200 Subject: [PATCH 08/20] Improved memories, consolidation existing with those from new onboarding --- .serena/memories/conventions.md | 26 +++ .serena/memories/core.md | 29 +++ .serena/memories/memory_maintenance.md | 42 +++++ .../serena_core_concepts_and_architecture.md | 175 ------------------ .serena/memories/suggested_commands.md | 40 ++-- .serena/memories/task_completion.md | 11 ++ .serena/memories/tech_stack.md | 10 + 7 files changed, 137 insertions(+), 196 deletions(-) create mode 100644 .serena/memories/conventions.md create mode 100644 .serena/memories/core.md create mode 100644 .serena/memories/memory_maintenance.md delete mode 100644 .serena/memories/serena_core_concepts_and_architecture.md create mode 100644 .serena/memories/task_completion.md create mode 100644 .serena/memories/tech_stack.md diff --git a/.serena/memories/conventions.md b/.serena/memories/conventions.md new file mode 100644 index 00000000..5c18dc52 --- /dev/null +++ b/.serena/memories/conventions.md @@ -0,0 +1,26 @@ +# Conventions + +## Style (project-instructed) +- Idiomatic, object-oriented design. Non-trivial interfaces use **explicitly typed abstractions** (strategy pattern etc.) rather than bare functions/callbacks. +- Avoid low-level data structures where an OO abstraction fits. For simple data containers use **dataclasses**, not dicts/tuples. +- Structure function bodies into **functional blocks separated by blank lines**, each prefixed with a short elliptical phrase (lowercase, no leading capital) describing the block's purpose. +- **Docstrings: reStructuredText.** Param/return/raises use `:param x:`, `:return:`, `:raises X:`. +- Parameter / method / class descriptions begin with a precise elliptical phrase defining *what* the thing is; details in subsequent sentences. + +## Formatting / lint (ruff) +- Line length 140, double quotes, target `py311`. +- Many "annoying" rules are disabled — see `[tool.ruff.lint] ignore` in `pyproject.toml` before adding workarounds (e.g. `Optional[T]` is preferred over `T | None`, `Union` is allowed, relative imports forbidden, `% string formatting` allowed). +- `ruff format` runs on `src scripts test`; same set for `ruff check`. +- mccabe complexity cap: 20. + +## Typing (mypy strict) +- `disallow_untyped_defs = true`, `disallow_incomplete_defs = true`, `strict_equality`, `strict_optional`, `warn_unreachable`, `no_implicit_optional`. +- Excluded paths: `^build/`, `^docs/`, `^test/resources/`. +- Test files relax `disallow_untyped_defs` (still type-checked otherwise). + +## Tests +- Language-server tests are pytest-marker-gated (one marker per language; see `pyproject.toml` `[tool.pytest.ini_options].markers`). Default `poe test` runs unmarked tests + whatever `PYTEST_MARKERS` selects. +- Snapshot tests use **syrupy** with custom `--snapshot-patch-pycharm-diff` plugin (auto-added via `addopts`). + +## Memories +- Follow `mem:memory_maintenance` for any new/updated memory in `.serena/memories/`. diff --git a/.serena/memories/core.md b/.serena/memories/core.md new file mode 100644 index 00000000..d4a76ff1 --- /dev/null +++ b/.serena/memories/core.md @@ -0,0 +1,29 @@ +# Serena — Project Core + +Serena is an MCP-based "IDE for coding agents": semantic code retrieval/editing/refactoring tools driven by language servers. + +## Source map + +- `src/serena/` — agent, MCP server, tools, project/config layer + - `agent.py`, `mcp.py`, `project_server.py`, `cli.py`, `hooks.py` — entrypoints/wiring + - `tools/` — tool implementations (memory_tools, symbol_tools, file_tools, workflow_tools, query_project_tools, config_tools, cmd_tools, jetbrains_tools) + - `tools/tools_base.py` — base classes for all tools + - `config/` — `serena_config.py`, `context_mode.py`, `client_setup.py` + - `resources/config/contexts/*.yml`, `resources/config/modes/*.yml` — context/mode definitions + - `code_editor.py`, `symbol.py`, `ls_manager.py` — symbolic editing / LS lifecycle + - `dashboard.py`, `gui_log_viewer.py` — web dashboard / log viewer + - `prompt_factory.py` + `generated/generated_prompt_factory.py` — prompts (regenerate with `scripts/gen_prompt_factory.py`) +- `src/solidlsp/` — LSP client framework; per-language servers under `language_servers/` +- `src/interprompt/` — prompt template library (synced from external repo; see `.syncCommitId.*`) +- `test/serena/`, `test/solidlsp//` — pytest suites; per-language tests gated by pytest markers +- `test/resources/repos//` — fixture projects used by language-server tests +- `scripts/` — utilities (prompt regen, tool overview, profiling, agno agent) +- `docs/` — Jupyter Book sources; build via poe `doc-build` + +## Project-wide invariants + +- Package name (PyPI): `serena-agent`; import root: `serena`. Wheel includes `serena`, `interprompt`, `solidlsp`. +- Python: `>=3.11, <3.15`. Dependencies are exact-pinned in `pyproject.toml` (uvx installs from git, lockfile ignored — pin exactly). +- Entry points: `serena` → `serena.cli:top_level`; `serena-hooks` → `serena.hooks:hook_commands`. +- Per-project state lives under `/.serena/` (config + `memories/` as `.md` files). +- See `mem:tech_stack`, `mem:suggested_commands`, `mem:conventions`, `mem:task_completion`. diff --git a/.serena/memories/memory_maintenance.md b/.serena/memories/memory_maintenance.md new file mode 100644 index 00000000..cfb4c05c --- /dev/null +++ b/.serena/memories/memory_maintenance.md @@ -0,0 +1,42 @@ +# Memory Maintenance + +## Discovery Model + +- Agents start with the full memory name list and should read `mem:core` first as the routing entry point. +- All other memory discovery comes from memory names plus routing already read. +- Routing is useful only before the target memory is selected. Once a memory is open, target-selection text inside it can no longer help decide to read it. +- Therefore, never put self-referential selection cues in a memory body: no "Start here", "Use when", "Use before", "Use after", or "read this when..." openers. + Such self-referential cues are an error mode: they are unreachable for selection, duplicate routing that belongs upstream, and drift independently from the actual routing graph. +- Add routing only when names are insufficient, ambiguous, or a non-obvious dependency/order matters. Do not add tautological routing such as "frontend debugging -> frontend/debugging". +- Routing can live outside core memories when needed, but core memories own most module-level routing because they are read before focused memories in that subfolder. +- Top-level/startup memory is `mem:core`; top-level memory in each memory subfolder is `/core` (for example `mem:frontend/core`). +- Memory references must use a mem: prefix inside backticks, e.g. `mem:frontend/core`. +- Do not add an index that simply re-lists the focused-memory names, the memory name list is already visible. + Only add routing pointers when a target memory's name is not self-explanatory or when a some aspect should be routed to multiple memories. + +## Style + +Dense agent notes, not prose docs. Prefer invariants, paths, commands, terse bullets. Avoid obvious context, rationale, and examples unless they prevent likely mistakes. Keep guidance durable and generalizable, not task-local. + +## Add/update threshold + +Add or update memory only for stable, non-obvious project conventions that avoid complex rediscovery. + +Do not add: quick-read/rg facts; generic language/framework knowledge; one-off task notes; volatile line-level details; behavior likely to change soon. + +Operational constants (commands, paths, version pins): phrase as current behavior; before changing the value, update it in the owning memory and run `serena memories check` to catch references that need updating. + +## Graph Ownership + +- Project-level routing: `mem:core`. +- Module-level routing/source maps: module core memories. +- Focused memories: topic details only; assume relevant routing was read before selection. + +## Maintenance Actions + +- Renamed/removed/split/merged/superseded memory: update all references. Done automatically if handled via Serena's memory rename tool. +- Stale paths/namespaces/commands/behavior: update or remove in owning memory. If working with Serena, call `serena memories check` for a report. +- Split catch-all memory when sections are commonly needed independently or agents must read unrelated domains to reach needed topic. +- Retire/delete memory when owning module/subsystem is removed or replaced; remove all references. +- Added/renamed/split/moved/deleted memory: update affected routing only where names are insufficient. +- Before finishing: list memories for old names and key new names to catch stale/missing references. \ No newline at end of file diff --git a/.serena/memories/serena_core_concepts_and_architecture.md b/.serena/memories/serena_core_concepts_and_architecture.md deleted file mode 100644 index 49edd2aa..00000000 --- a/.serena/memories/serena_core_concepts_and_architecture.md +++ /dev/null @@ -1,175 +0,0 @@ -# Serena Core Concepts and Architecture - -## High-Level Architecture - -Serena is built around a dual-layer architecture: - -1. **SerenaAgent** - The main orchestrator that manages projects, tools, and user interactions -2. **SolidLanguageServer** - A unified wrapper around Language Server Protocol (LSP) implementations - -## Core Components - -### 1. SerenaAgent (`src/serena/agent.py`) - -The central coordinator that: -- Manages active projects and their configurations -- Coordinates between different tools and contexts -- Handles language server lifecycle -- Manages memory persistence -- Provides MCP (Model Context Protocol) server interface - -Key responsibilities: -- **Project Management** - Activating, switching between projects -- **Tool Registry** - Loading and managing available tools based on context/mode -- **Language Server Integration** - Starting/stopping language servers per project -- **Memory Management** - Persistent storage of project knowledge -- **Task Execution** - Coordinating complex multi-step operations - -### 2. SolidLanguageServer (`src/solidlsp/ls.py`) - -A unified abstraction over multiple language servers that provides: -- **Language-agnostic interface** for symbol operations -- **Caching layer** for performance optimization -- **Error handling and recovery** for unreliable language servers -- **Uniform API** regardless of underlying LSP implementation - -Core capabilities: -- Symbol discovery and navigation -- Code completion and hover information -- Find references and definitions -- Document and workspace symbol search -- File watching and change notifications - -### 3. Tool System (`src/serena/tools/`) - -Modular tool architecture with several categories: - -#### File Tools (`file_tools.py`) -- File system operations (read, write, list directories) -- Text search and pattern matching -- Regex-based replacements - -#### Symbol Tools (`symbol_tools.py`) -- Language-aware symbol finding and navigation -- Symbol body replacement and insertion -- Reference finding across codebase - -#### Memory Tools (`memory_tools.py`) -- Project knowledge persistence -- Memory retrieval and management -- Onboarding information storage - -#### Configuration Tools (`config_tools.py`) -- Project activation and switching -- Mode and context management -- Tool inclusion/exclusion - -### 4. Configuration System (`src/serena/config/`) - -Multi-layered configuration supporting: -- **Contexts** - Define available tools and their behavior -- **Modes** - Specify operational patterns (interactive, editing, etc.) -- **Projects** - Per-project settings and language server configs -- **Tool Sets** - Grouped tool collections for different use cases - -## Language Server Integration - -### Language Support Model - -Each supported language has: -1. **Language Server Implementation** (`src/solidlsp/language_servers/`) -2. **Runtime Dependencies** - Managed downloads of language servers -3. **Test Repository** (`test/resources/repos//`) -4. **Test Suite** (`test/solidlsp//`) - -### Language Server Lifecycle - -1. **Discovery** - Find language servers or download them automatically -2. **Initialization** - Start server process and perform LSP handshake -3. **Project Setup** - Open workspace and configure language-specific settings -4. **Operation** - Handle requests/responses with caching and error recovery -5. **Shutdown** - Clean shutdown of server processes - -### Supported Languages - -Current language support includes: -- **C#** - Microsoft.CodeAnalysis.LanguageServer (.NET 9) -- **Python** - Pyright or Jedi -- **TypeScript/JavaScript** - TypeScript Language Server -- **Rust** - rust-analyzer -- **Go** - gopls -- **Java** - Eclipse JDT Language Server -- **Kotlin** - Kotlin Language Server -- **PHP** - Intelephense -- **Ruby** - Solargraph -- **Clojure** - clojure-lsp -- **Elixir** - ElixirLS -- **Dart** - Dart Language Server -- **C/C++** - clangd -- **Terraform** - terraform-ls - -## Memory and Knowledge Management - -### Memory System -- **Markdown-based storage** in `.serena/memories/` directory -- **Contextual retrieval** - memories loaded based on relevance -- **Project-specific** knowledge persistence -- **Onboarding support** - guided setup for new projects - -### Knowledge Categories -- **Project Structure** - Directory layouts, build systems -- **Architecture Patterns** - How the codebase is organized -- **Development Workflows** - Testing, building, deployment -- **Domain Knowledge** - Business logic and requirements - -## MCP Server Interface - -Serena exposes its functionality through Model Context Protocol: -- **Tool Discovery** - AI agents can enumerate available tools -- **Context-Aware Operations** - Tools behave based on active project/mode -- **Stateful Sessions** - Maintains project state across interactions -- **Error Handling** - Graceful degradation when tools fail - -## Error Handling and Resilience - -### Language Server Reliability -- **Timeout Management** - Configurable timeouts for LSP requests -- **Process Recovery** - Automatic restart of crashed language servers -- **Fallback Behavior** - Graceful degradation when LSP unavailable -- **Caching Strategy** - Reduces impact of server failures - -### Project Activation Safety -- **Validation** - Verify project structure before activation -- **Error Isolation** - Project failures don't affect other projects -- **Recovery Mechanisms** - Automatic cleanup and retry logic - -## Performance Considerations - -### Caching Strategy -- **Symbol Cache** - In-memory caching of expensive symbol operations -- **File System Cache** - Reduced disk I/O for repeated operations -- **Language Server Cache** - Persistent cache across sessions - -### Resource Management -- **Language Server Pooling** - Reuse servers across projects when possible -- **Memory Management** - Automatic cleanup of unused resources -- **Background Operations** - Async operations don't block user interactions - -## Extension Points - -### Adding New Languages -1. Implement language server class in `src/solidlsp/language_servers/` -2. Add runtime dependencies configuration -3. Create test repository and test suite -4. Update language enumeration and configuration - -### Adding New Tools -1. Inherit from `Tool` base class in `tools_base.py` -2. Implement required methods and parameter validation -3. Register tool in appropriate tool registry -4. Add to context/mode configurations as needed - -### Custom Contexts and Modes -- Define new contexts in YAML configuration files -- Specify tool sets and operational patterns -- Configure for specific development workflows \ No newline at end of file diff --git a/.serena/memories/suggested_commands.md b/.serena/memories/suggested_commands.md index 2ecbe796..5ae6638f 100644 --- a/.serena/memories/suggested_commands.md +++ b/.serena/memories/suggested_commands.md @@ -1,26 +1,24 @@ # Suggested Commands -## Development Tasks (using uv and poe) +Run via `uv run poe ` (or `poe ` inside the activated venv). Poe executor is `simple`, so plain `poe` works without uv re-resolving. -The following tasks should generally be executed using `uv run poe `. +## Dev loop +- `poe test` — pytest on `test/` (per-language tests are marker-gated; pass `-m ` to enable). +- `poe lint` — ruff format-check + ruff check (no fixes). +- `poe format` — ruff `--fix` then `ruff format` (mutates files). +- `poe type-check` — mypy on `src/serena`, `src/solidlsp`, and `test/` (test pass disables `no-untyped-def`). +- Single test file: `uv run pytest test/path/to/test_x.py -vv`. Language-gated: add `-m python` etc. -- `format`: This is the **only** allowed command for formatting. Run as `uv run poe format`. -- `type-check`: This is the **only** allowed command for type checking. Run as `uv run poe type-check`. -- `test`: This is the preferred command for running tests (`uv run poe test [args]`). You can select subsets of tests with markers, - the current markers are - ```toml - markers = [ - "python: language server running for Python", - "go: language server running for Go", - "java: language server running for Java", - "rust: language server running for Rust", - "typescript: language server running for TypeScript", - "php: language server running for PHP", - "snapshot: snapshot tests for symbolic editing operations", - ] - ``` - By default, `uv run poe test` uses the markers set in the env var `PYTEST_MARKERS`, or, if it unset, uses `-m "not java and not rust and not isolated process"`. - You can override this behavior by simply passing the `-m` option to `uv run poe test`, e.g. `uv run poe test -m "python or go"`. +## Docs +- `poe doc-build` — clean + autogen + sphinx (uses `rm -rf`; needs a unix-like shell, e.g. Git Bash on Windows). -For finishing a task, make sure format, type-check and test pass! Run them at the end of the task -and if needed fix any issues that come up and run them again until they pass. \ No newline at end of file +## Entrypoints +- `uv run serena ...` — main CLI (`serena.cli:top_level`). +- `uv run serena-hooks ...` — hook helpers. +- `python scripts/gen_prompt_factory.py` — regenerate `src/serena/generated/generated_prompt_factory.py` after editing prompt templates. + +## Windows shell notes (PowerShell 7+ is the project shell) +- Use `Remove-Item -Recurse -Force ` instead of `rm -rf` (the `doc-clean` poe task uses unix `rm -rf` and requires bash). +- Env vars: `$env:NAME = 'value'` (not `export`). +- Path separator in `pyproject.toml` poe tasks uses forward slashes; PowerShell accepts them in arguments. +- `git`, `uv`, `poe`, `pytest` behave identically to unix. diff --git a/.serena/memories/task_completion.md b/.serena/memories/task_completion.md new file mode 100644 index 00000000..5ca33012 --- /dev/null +++ b/.serena/memories/task_completion.md @@ -0,0 +1,11 @@ +# Task Completion Checklist + +After any code change in `src/` or `test/`, run: + +1. `uv run poe format` — applies ruff fixes + formatting (mutates files). +2. `uv run poe type-check` — mypy strict on `src/serena`, `src/solidlsp`, `test/`. +3. `uv run poe test` — pytest on affected files or affected languages, using `-m` markers. + +If prompt templates changed: `uv run python scripts/gen_prompt_factory.py` (regenerates `src/serena/generated/generated_prompt_factory.py`; use `uv run poe format` and commit the result). + +If memories were edited/renamed/split: run `uv run serena memories check` from the project root to find broken `mem:` references. diff --git a/.serena/memories/tech_stack.md b/.serena/memories/tech_stack.md new file mode 100644 index 00000000..f868cca6 --- /dev/null +++ b/.serena/memories/tech_stack.md @@ -0,0 +1,10 @@ +# Tech Stack + +- Language: Python 3.11–3.14 (`requires-python = ">=3.11, <3.15"`). +- Package/dep manager: **uv** (uv.lock present). Exact version pins in `pyproject.toml` because `uvx` installs from git and ignores the lockfile. +- Build backend: `hatchling`. Packages: `src/serena`, `src/interprompt`, `src/solidlsp`. +- Task runner: **poethepoet** (`poe `). Poe executor is `simple` (does NOT shell out via uv) — avoids env recreation while MCP server is running. +- Key runtime deps: `mcp`, `flask` (dashboard), `pydantic`, `pygls` + `lsprotocol` (LSP), `anthropic`, `jinja2`, `ruamel.yaml`, `pywebview`/`pystray` (GUI), `pythonnet` (Windows only). +- Dev deps: `ruff` (lint+format), `mypy` (strict), `pytest` (+ `pytest-xdist`, `pytest-timeout`, `syrupy` snapshots), `sphinx`/`jupyter-book` for docs. +- Optional extras: `agno` (Agno agent integration), `google` (gemini). +- LSP client core lives under `src/solidlsp/`; one subdir per supported language server under `language_servers/`. From 27b3622c3829b7638278e36880e2c8cc347a2277 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 13 May 2026 23:14:59 +0200 Subject: [PATCH 09/20] Improve fuzzy memory name matching logic and UX in CLI --- docs/02-usage/045_memories.md | 6 +- src/serena/cli.py | 30 ++++- src/serena/project.py | 154 ++++++++++++++------- test/serena/test_memories_manager.py | 194 +++++++++++++++++++++++++-- 4 files changed, 320 insertions(+), 64 deletions(-) diff --git a/docs/02-usage/045_memories.md b/docs/02-usage/045_memories.md index b578ed3c..e14d3b31 100644 --- a/docs/02-usage/045_memories.md +++ b/docs/02-usage/045_memories.md @@ -189,9 +189,9 @@ Serena also offers memory-related CLI commands. The following commands have **no MCP tool counterpart** and are intended for human execution: -- `serena memories check` — referential-integrity report. By default reports only stale - `` `mem:NAME` `` references whose target does not resolve. Pass `--include-unmarked` to - also report bare occurrences of existing memory names found through a heuristic. +- `serena memories check` — referential-integrity report. By default reports stale + `` `mem:NAME` `` references; additional scans (bare occurrences and fuzzy near-misses) + are opt-in via flags. Run `serena memories check --help` for the full flag list. - `serena memories auto-prefix-references` — heuristic rewrite of bare occurrences to add the `mem:` prefix; supports `--dry-run`. - `serena memories initialize` will seed the `memory_maintenance` memory for the project. diff --git a/src/serena/cli.py b/src/serena/cli.py index ae7a2380..27022662 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -1308,9 +1308,10 @@ class MemoryCommands(AutoRegisteringGroup): "check", help=( "Check referential integrity across all memories of the project (and global memories). " - "Reports stale `mem:` references. With --include-unmarked, also reports bare occurrences " - "of existing memory names that lack the `mem:` prefix (split into high- and low-confidence). " - "Read-only and never writes. Always exits 0." + "By default reports only stale `mem:` references. Pass --include-unmarked to also report " + "bare occurrences of existing memory names (exact matches) and --fuzzy-matching (only " + "meaningful in combination with --include-unmarked) to additionally report fuzzy " + "near-misses. Read-only and never writes. Always exits 0." ), context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, ) @@ -1319,11 +1320,28 @@ class MemoryCommands(AutoRegisteringGroup): "--include-unmarked", is_flag=True, default=False, - help="Also report potentially unmarked references (bare occurrences of memory names without the `mem:` prefix).", + help="Also report bare exact occurrences of existing memory names (i.e. without the `mem:` prefix).", ) - def check(project: str, include_unmarked: bool) -> None: + @click.option( + "--fuzzy-matching", + is_flag=True, + default=False, + help=( + "Additionally report fuzzy near-misses (long bare tokens that similarity-match an existing " + "memory name). Only meaningful together with --include-unmarked; ignored otherwise." + ), + ) + def check(project: str, include_unmarked: bool, fuzzy_matching: bool) -> None: + if fuzzy_matching and not include_unmarked: + click.echo( + "Warning: --fuzzy-matching has no effect without --include-unmarked; ignoring it.", + err=True, + ) manager = MemoryCommands._load_memories_manager(project) - report = manager.validate_referential_integrity(include_unmarked=include_unmarked) + report = manager.validate_referential_integrity( + include_unmarked=include_unmarked, + include_fuzzy_matching=fuzzy_matching, + ) click.echo(report.format()) @staticmethod diff --git a/src/serena/project.py b/src/serena/project.py index 21bea043..480a4a37 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -165,7 +165,17 @@ class MemoriesManager: # tokenized name parts. Containment / SequenceMatcher signals alone over-match — e.g. a # generic prose word like "repository" is a substring of "serena_repository_structure" # but their token Jaccard is only 1/3, so we reject it at this floor. - _FUZZY_BARE_TOKEN_JACCARD_FLOOR: float = 0.5 + _FUZZY_BARE_TOKEN_JACCARD_FLOOR: float = 0.6 + # In :meth:`compute_name_similarity` case 2 (basenames differ), when the two names also + # carry *differing* topic prefixes, demand at least one strong basename signal: token + # Jaccard >= this floor, basename containment, or typo-level seq ratio. Without this + # gate, names like ``frontend/x-subtleties`` and ``backend/y-subtleties`` get matched + # purely on the shared trailing token via the SequenceMatcher signal. + _BASENAME_JACCARD_FLOOR: float = 0.34 + _BASENAME_TYPO_SEQ_FLOOR: float = 0.75 + # Hard cap on candidates proposed per stale reference. Beyond a small number the list + # stops aiding disambiguation and becomes noise. + MAX_STALE_REFERENCE_CANDIDATES: int = 3 _WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: frozenset[str] = frozenset({"core"}) """Names that we specifically mention in our instructions and that coincide with common words""" @@ -204,6 +214,8 @@ class MemoriesManager: auth/login ~ security/login -> 0.50 (basename only; disjoint topics) auth ~ authentication -> 0.64 (substring containment) foo ~ for -> 0.00 (short-name floor) + frontend/x-subtleties ~ backend/y-subtleties -> 0.00 (different topics + only a + shared generic trailing token) :param a: first memory name :param b: second memory name @@ -234,22 +246,34 @@ class MemoriesManager: ) return 0.5 + 0.5 * prefix_jaccard - # case 2: basenames differ -- combine generic similarity signals via max - tokens_a, tokens_b = cls._tokenize_name(norm_a), cls._tokenize_name(norm_b) - jaccard = len(tokens_a & tokens_b) / len(tokens_a | tokens_b) if (tokens_a | tokens_b) else 0.0 - seq = SequenceMatcher(None, norm_a, norm_b).ratio() + # case 2: basenames differ -- score the *basenames* only, so that a shared topic prefix + # plus a generic trailing token (e.g. "frontend/X-subtleties" vs "frontend/Y-subtleties") + # does not inflate the SequenceMatcher signal into a false positive. + basename_tokens_a, basename_tokens_b = cls._tokenize_name(basename_a), cls._tokenize_name(basename_b) + union_basename = basename_tokens_a | basename_tokens_b + basename_jaccard = (len(basename_tokens_a & basename_tokens_b) / len(union_basename)) if union_basename else 0.0 + basename_contained = bool(basename_a) and bool(basename_b) and (basename_a in basename_b or basename_b in basename_a) + seq = SequenceMatcher(None, basename_a, basename_b).ratio() + + # when both names carry *differing* topic prefixes, demand at least one strong basename + # signal — otherwise a single coincidental shared token at the end of long basenames + # under different topic roots slips through purely on the seq-ratio + if prefix_a and prefix_b and prefix_a != prefix_b: + if basename_jaccard < cls._BASENAME_JACCARD_FLOOR and not basename_contained and seq < cls._BASENAME_TYPO_SEQ_FLOOR: + return 0.0 + # containment is a meaningful rename signal on its own (e.g. auth -> authentication), - # so we lift it above the threshold floor when one name fully contains the other, + # so we lift it above the threshold floor when one basename fully contains the other, # with the length-ratio gating how confident we are contained = 0.0 - if norm_a and norm_b and (norm_a in norm_b or norm_b in norm_a): - length_ratio = min(len(norm_a), len(norm_b)) / max(len(norm_a), len(norm_b)) + if basename_contained: + length_ratio = min(len(basename_a), len(basename_b)) / max(len(basename_a), len(basename_b)) contained = 0.5 + 0.5 * length_ratio - # short-name floor: for very short names, only accept candidates with a strong token signal - if min(len(norm_a), len(norm_b)) <= cls._SHORT_NAME_FLOOR and jaccard < 1.0: + # short-name floor: for very short basenames, only accept candidates with a strong token signal + if min(len(basename_a), len(basename_b)) <= cls._SHORT_NAME_FLOOR and basename_jaccard < 1.0: return 0.0 - return max(jaccard, seq, contained) + return max(basename_jaccard, seq, contained) @classmethod def _find_stale_reference_candidates(cls, missing_name: str, existing_names: list[str], threshold: float | None = None) -> list[str]: @@ -259,7 +283,9 @@ class MemoriesManager: :param threshold: optional override for the minimum similarity required; defaults to :attr:`NAME_SIMILARITY_THRESHOLD`. :return: existing memory names whose similarity to ``missing_name`` meets or exceeds - ``threshold``, sorted in descending order of similarity (ties broken alphabetically). + ``threshold``, sorted in descending order of similarity (ties broken alphabetically) + and capped at :attr:`MAX_STALE_REFERENCE_CANDIDATES` entries to keep the output + focused on the most plausible matches. """ cutoff = cls.NAME_SIMILARITY_THRESHOLD if threshold is None else threshold scored: list[tuple[float, str]] = [] @@ -268,7 +294,7 @@ class MemoriesManager: if score >= cutoff: scored.append((score, existing)) scored.sort(key=lambda pair: (-pair[0], pair[1])) - return [name for _, name in scored] + return [name for _, name in scored[: cls.MAX_STALE_REFERENCE_CANDIDATES]] # character class delimiting a memory name; used to anchor regex matches so we never # consume a partial name (kept in sync with the boundary rule in rename_references_to_memory) @@ -544,28 +570,41 @@ class MemoriesManager: tb.with_line(" candidates: (none above similarity threshold)") tb.with_line("") - # unmarked-reference warnings, split by confidence - for label, warnings in ( - ("High-confidence unmarked references", self.high_confidence_unmarked_memories), - ("Low-confidence unmarked references", self.low_confidence_unmarked_memories), - ): - if not warnings: - continue - tb.with_line(f"{label} ({len(warnings)}):") - for w in warnings: - ro_tag = " [read-only source]" if w.source_is_read_only else "" - occ = "occurrence" if w.occurrences == 1 else "occurrences" - if w.is_exact_match: + # unmarked-reference warnings — split exact vs fuzzy near-miss, then by confidence + exact_high = [w for w in self.high_confidence_unmarked_memories if w.is_exact_match] + exact_low = [w for w in self.low_confidence_unmarked_memories if w.is_exact_match] + fuzzy = [w for w in self.high_confidence_unmarked_memories if not w.is_exact_match] + [ + w for w in self.low_confidence_unmarked_memories if not w.is_exact_match + ] + + # the "may be intentional" caveat is meaningful only for exact matches (where the bare + # text equals a real memory name and may simply be ordinary prose); print it once, + # before the first exact-match section + if exact_high or exact_low: + tb.with_line("Possibly unmarked references — exact matches:") + tb.with_line(" (note: bare matches in prose may be intentional and not actual references)") + for label, warnings in (("high confidence", exact_high), ("low confidence", exact_low)): + if not warnings: + continue + tb.with_line(f" {label} ({len(warnings)}):") + for w in warnings: + ro_tag = " [read-only source]" if w.source_is_read_only else "" + occ = "occurrence" if w.occurrences == 1 else "occurrences" tb.with_line( - f" - {w.occurrences} {occ} of `{w.suspected_name}` in `{w.source_memory}`{ro_tag}" + f" - {w.occurrences} {occ} of `{w.suspected_name}` in `{w.source_memory}`{ro_tag}" f" (suggested: `mem:{w.suspected_name}`)" ) - else: - # fuzzy near-miss: the bare text differs from the proposed target - tb.with_line( - f" - {w.occurrences} {occ} of `{w.actual_token}` in `{w.source_memory}`{ro_tag}" - f" — near-miss (suggested: `mem:{w.suspected_name}`)" - ) + tb.with_line("") + + if fuzzy: + tb.with_line(f"Possibly unmarked references — fuzzy near-misses ({len(fuzzy)}):") + for w in fuzzy: + ro_tag = " [read-only source]" if w.source_is_read_only else "" + occ = "occurrence" if w.occurrences == 1 else "occurrences" + tb.with_line( + f" - {w.occurrences} {occ} of `{w.actual_token}` in `{w.source_memory}`{ro_tag}" + f" — near-miss (suggested: `mem:{w.suspected_name}`)" + ) tb.with_line("") return tb.build() @@ -788,36 +827,42 @@ class MemoriesManager: source_basename = source_memory.rsplit("/", 1)[-1] return suspected_name == source_basename - def validate_referential_integrity(self, include_unmarked: bool = True) -> "MemoriesManager.ReferentialIntegrityReport": + def validate_referential_integrity( + self, include_unmarked: bool = True, include_fuzzy_matching: bool = True + ) -> "MemoriesManager.ReferentialIntegrityReport": """ Scans every (non-ignored) memory's content for referential integrity issues. The scan covers both project-local and global memories. Three kinds of finding are - produced: + produced, each independently gated: * **stale references** — occurrences of ``mem:NAME`` where ``NAME`` does not resolve to an existing memory. For each, a list of similarly-named existing memories is - proposed as candidate intended targets (see :meth:`compute_name_similarity`). + proposed as candidate intended targets (see :meth:`compute_name_similarity`), + capped at :attr:`MAX_STALE_REFERENCE_CANDIDATES`. Always reported. * **exact unmarked-reference warnings** — bare occurrences of an existing memory name that appear without the ``mem:`` prefix. Split into high-confidence (the suspected name contains a ``/`` or exceeds :attr:`_HIGH_CONFIDENCE_NAME_LENGTH`) - and low-confidence groups. Candidate names whose basename matches - :attr:`WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES` are skipped. + and low-confidence groups. Candidate names whose basename is in + :attr:`_WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES` are skipped. Gated by + ``include_unmarked``. * **fuzzy near-miss warnings** — long, distinctive bare tokens in a memory body that *do not* match an existing name exactly but similarity-match a - high-confidence existing name AND share at least :attr:`_FUZZY_BARE_TOKEN_JACCARD_FLOOR` - of their tokenized name parts (e.g. ``adding_new_language_support`` in prose when - the memory is named ``adding_new_language_support_guide``). Always reported as - high-confidence; the actual bare text is preserved on - :attr:`UnmarkedReferenceWarning.actual_token`. + high-confidence existing name AND share at least + :attr:`_FUZZY_BARE_TOKEN_JACCARD_FLOOR` of their tokenized name parts. Always + reported as high-confidence; the actual bare text is preserved on + :attr:`UnmarkedReferenceWarning.actual_token`. Gated by ``include_unmarked + and include_fuzzy_matching`` — fuzzy matches are noisy and only meaningful when + unmarked-reference checking is enabled. Self-references (a memory's content mentioning its own name or basename) and empty - memories are skipped silently. + memories are skipped silently. This method has no side effects. - This method has no side effects. - - :param include_unmarked: if False, skip the two unmarked-reference scans (exact and - fuzzy near-miss) entirely; only stale references are reported. + :param include_unmarked: if False, skip both the exact unmarked scan and the fuzzy + near-miss scan; only stale references are reported. + :param include_fuzzy_matching: if False, skip the fuzzy near-miss scan even when + unmarked-reference checking is otherwise enabled. Has no effect when + ``include_unmarked`` is False. :return: a :class:`ReferentialIntegrityReport` summarizing all findings. """ report = MemoriesManager.ReferentialIntegrityReport() @@ -885,11 +930,19 @@ class MemoriesManager: else: report.low_confidence_unmarked_memories.append(warning) + if not include_fuzzy_matching: + continue + # fuzzy near-misses: long bare tokens that similarity-match a high-confidence existing name for token, n in self._iter_long_bare_tokens(content): if token in existing_set: # exact-match path already handled this token (or self-referenced and was skipped) continue + # filter bare tokens that coincide with common English words; the same filter is + # applied to candidate names in the exact-match loop, this is the symmetric side + token_norm = self._normalize_for_similarity(token) + if token_norm in self._WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: + continue token_tokens = self._tokenize_name(token) best_name: str | None = None best_score = 0.0 @@ -898,6 +951,13 @@ class MemoriesManager: continue if existing.rsplit("/", 1)[-1] in self._WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: continue + # require substring containment between bare token and existing name: fuzzy + # matches without a containment relationship are rarely useful as rewrite + # suggestions (auto_prefix_bare_references can't fix them anyway) and dominate + # the false-positive rate, e.g. "grid-layout" vs "layout-grid-subtleties". + existing_norm = self._normalize_for_similarity(existing) + if token_norm not in existing_norm and existing_norm not in token_norm: + continue score = self.compute_name_similarity(token, existing) if score < self.NAME_SIMILARITY_THRESHOLD: continue diff --git a/test/serena/test_memories_manager.py b/test/serena/test_memories_manager.py index 0ff428eb..3974d159 100644 --- a/test/serena/test_memories_manager.py +++ b/test/serena/test_memories_manager.py @@ -134,17 +134,12 @@ class TestRenameReferencesToMemory: @pytest.mark.parametrize( "delimiter_pair", [ + # one case per structural class of surrounding character (asymmetric pair, whitespace, + # punctuation, none); the boundary regex only distinguishes "name char" vs "non-name char", + # so further variants exercise the same code path. ("`", "`"), - ('"', '"'), - ("'", "'"), ("(", ")"), - ("[", "]"), - ("{", "}"), (" ", " "), - ("\n", "\n"), - ("\t", "\t"), - (",", ","), - (";", ";"), ("", ""), # raw, no delimiters ], ) @@ -217,6 +212,27 @@ class TestNameSimilarity: "security/login", "auth/login" ) + def test_cross_topic_suffix_coincidence_is_rejected(self, manager: MemoriesManager) -> None: + """ + Regression: when two long names live under *different* topic prefixes and share + only a single trailing token (e.g. ``-subtleties``), the score must collapse to + zero. Without the cross-topic gate, SequenceMatcher returns ~0.55+ on these + purely on shared characters, flooding stale-reference candidate lists with + unrelated memories from neighbouring subsystems. + """ + target = "frontend/plugin-api-runtime-subtleties" + # different topic prefix + only the generic "subtleties" trailing token shared -> 0 + for other in ( + "backend/rpc-db-worker-subtleties", + "render-wasm/ffi-rendering-subtleties", + "common/grid-layout-subtleties", + ): + assert manager.compute_name_similarity(target, other) == 0.0, f"cross-topic suffix coincidence with {other} must be rejected" + # contrast: *same* prefix is treated leniently — the gate must not apply, so the + # score is non-zero even if it still falls under the similarity threshold + same_prefix_score = manager.compute_name_similarity(target, "frontend/routing-app-shell-subtleties") + assert same_prefix_score > 0.0 + def test_find_candidates_ranks_by_descending_similarity(self, manager: MemoriesManager) -> None: existing = ["auth/login", "security/login", "database", "authentication"] candidates = manager._find_stale_reference_candidates("login", existing) @@ -359,6 +375,168 @@ class TestValidateReferentialIntegrity: assert warning.suspected_name == existing assert warning.source_memory == "docs" + def test_fuzzy_accepts_bare_token_when_jaccard_just_above_floor(self, fs_manager: MemoriesManager) -> None: + """ + Edge case: bare token is a clean substring of the existing memory name AND + their tokenized Jaccard sits just above ``_FUZZY_BARE_TOKEN_JACCARD_FLOOR`` + (with floor = 0.6, this means 2/3 ≈ 0.667). + """ + # existing has 3 name-tokens; bare has 2 of them -> jaccard = 2/3 = 0.667 + existing = "data-model-extra" + bare = "data-model" + assert len(bare) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + _write(fs_manager, existing, "# extra notes") + _write(fs_manager, "docs", f"see {bare} for naming") + report = fs_manager.validate_referential_integrity() + fuzzy = [w for w in report.high_confidence_unmarked_memories if not w.is_exact_match] + assert len(fuzzy) == 1 + assert fuzzy[0].suspected_name == existing + assert fuzzy[0].actual_token == bare + + def test_fuzzy_rejects_bare_token_when_jaccard_just_below_floor(self, fs_manager: MemoriesManager) -> None: + """ + Edge case: bare token is still a clean substring of the existing memory name, + but their tokenized Jaccard is just below ``_FUZZY_BARE_TOKEN_JACCARD_FLOOR`` + (2/4 = 0.5 with the floor at 0.6). Must be rejected — without this gate, + long candidate names would attract many borderline matches. + """ + existing = "data-model-extra-stuff" # 4 tokens + bare = "data-model" # 2 tokens; jaccard = 2/4 = 0.5 + assert len(bare) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + _write(fs_manager, existing, "# notes") + _write(fs_manager, "docs", f"see {bare} for naming") + report = fs_manager.validate_referential_integrity() + assert report.high_confidence_unmarked_memories == [] + assert report.low_confidence_unmarked_memories == [] + + def test_fuzzy_requires_substring_containment(self, fs_manager: MemoriesManager) -> None: + """ + Edge case: bare token and existing name share several tokens (and even a topic + prefix) but neither is a substring of the other. Without the containment + requirement, sequence-ratio alone would mis-rank long names like + ``frontend/X-subtleties`` vs ``frontend/Y-subtleties`` as near-misses. + """ + existing = "frontend/text-editor-workflow" + bare = "frontend/text-editor-features" # shares "frontend", "text", "editor"; not a substring + assert len(bare) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + _write(fs_manager, existing, "# workflow notes") + _write(fs_manager, "docs", f"see {bare} for layout") + report = fs_manager.validate_referential_integrity() + assert report.high_confidence_unmarked_memories == [] + assert report.low_confidence_unmarked_memories == [] + + def test_fuzzy_accepts_reverse_containment(self, fs_manager: MemoriesManager) -> None: + """ + Edge case: the *existing* memory name is a substring of the longer bare token + (rather than the other way around). The containment gate must hold in both + directions, so this is still flagged. + """ + existing = "playwright-gestures" + bare = "playwright-gestures-guide" # existing ⊂ bare + assert len(bare) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + _write(fs_manager, existing, "# notes") + _write(fs_manager, "docs", f"see {bare} for details") + report = fs_manager.validate_referential_integrity() + fuzzy = [w for w in report.high_confidence_unmarked_memories if not w.is_exact_match] + assert len(fuzzy) == 1 + assert fuzzy[0].suspected_name == existing + assert fuzzy[0].actual_token == bare + + def test_unmarked_exact_match_suppressed_for_ignored_word(self, fs_manager: MemoriesManager) -> None: + """ + Names in ``_WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES`` (e.g. ``core``) are + common-word collisions and must not produce bare-occurrence warnings even + when an exact memory of that name exists. + """ + _write(fs_manager, "core", "# project core notes") + _write(fs_manager, "docs", "this is the core of the system") + report = fs_manager.validate_referential_integrity() + # bare "core" matches the candidate name `core` literally but is filtered out + assert all(w.suspected_name != "core" for w in report.high_confidence_unmarked_memories) + assert all(w.suspected_name != "core" for w in report.low_confidence_unmarked_memories) + + def test_unmarked_exact_match_suppressed_for_topic_with_ignored_basename(self, fs_manager: MemoriesManager) -> None: + """ + The ignore filter applies to the *basename* of a candidate name, so a memory + named ``frontend/core`` is also exempt from bare-occurrence warnings (the + bare text ``frontend/core`` is more likely a directory reference than a + forgotten memory link). + """ + _write(fs_manager, "frontend/core", "# notes") + _write(fs_manager, "docs", "the frontend/core module is documented here") + report = fs_manager.validate_referential_integrity() + assert all(w.suspected_name != "frontend/core" for w in report.high_confidence_unmarked_memories) + assert all(w.suspected_name != "frontend/core" for w in report.low_confidence_unmarked_memories) + + def test_unmarked_exact_match_not_suppressed_for_non_ignored_basename(self, fs_manager: MemoriesManager) -> None: + """ + Sanity: the ignore filter must not over-apply to names whose basename happens + to share a prefix with an ignored word. + """ + _write(fs_manager, "frontend/core-utils", "# notes") + _write(fs_manager, "docs", "the frontend/core-utils module is documented here") + report = fs_manager.validate_referential_integrity() + assert any(w.suspected_name == "frontend/core-utils" for w in report.high_confidence_unmarked_memories) + + def test_stale_reference_candidates_are_capped(self, fs_manager: MemoriesManager) -> None: + """ + The candidate list proposed for an unresolved ``mem:`` reference is capped at + :attr:`MAX_STALE_REFERENCE_CANDIDATES`. Beyond a small number the list stops + aiding disambiguation and becomes noise. + """ + cap = MemoriesManager.MAX_STALE_REFERENCE_CANDIDATES + # five memories all sharing basename `login` -> all tie at compute_name_similarity == 1.0 + candidate_names = ["admin/login", "api/login", "auth/login", "security/login", "users/login"] + for name in candidate_names: + _write(fs_manager, name, "# notes") + _write(fs_manager, "docs", "old reference: `mem:login`") + report = fs_manager.validate_referential_integrity() + assert len(report.stale_references) == 1 + candidates = report.stale_references[0].candidates + # cap enforced; alphabetical tiebreak among equally-scored candidates + assert len(candidates) == cap + assert candidates == candidate_names[:cap] + + def test_include_unmarked_false_skips_both_scans(self, fs_manager: MemoriesManager) -> None: + """ + With ``include_unmarked=False`` only stale references are reported; both the + exact unmarked scan and the fuzzy near-miss scan are entirely skipped. + """ + _write(fs_manager, "auth/login", "# notes") + _write(fs_manager, "playwright-gestures", "# notes") + _write( + fs_manager, + "docs", + "bare auth/login mention and a playwright-gestures-guide near-miss; also `mem:nope`", + ) + report = fs_manager.validate_referential_integrity(include_unmarked=False) + assert report.high_confidence_unmarked_memories == [] + assert report.low_confidence_unmarked_memories == [] + # stale reference is still reported regardless of the flag + assert len(report.stale_references) == 1 + assert report.stale_references[0].referenced_name == "nope" + + def test_include_fuzzy_matching_false_keeps_exact_only(self, fs_manager: MemoriesManager) -> None: + """ + With ``include_unmarked=True`` but ``include_fuzzy_matching=False``, exact + bare-occurrence warnings still fire but fuzzy near-misses are suppressed. + """ + _write(fs_manager, "auth/login", "# notes") + _write(fs_manager, "playwright-gestures", "# notes") + _write( + fs_manager, + "docs", + "bare auth/login mention and a playwright-gestures-guide near-miss", + ) + report = fs_manager.validate_referential_integrity(include_unmarked=True, include_fuzzy_matching=False) + # exact match for auth/login is kept + exact = [w for w in report.high_confidence_unmarked_memories if w.is_exact_match] + fuzzy = [w for w in report.high_confidence_unmarked_memories if not w.is_exact_match] + assert len(exact) == 1 + assert exact[0].suspected_name == "auth/login" + # fuzzy near-miss for `playwright-gestures-guide` is suppressed + assert fuzzy == [] + def test_self_reference_is_ignored(self, fs_manager: MemoriesManager) -> None: # the basename `login` appears in `auth/login`'s own content -> must not be flagged _write(fs_manager, "auth/login", "this login flow ...") From 833f518c3d855acdcbee13ff1678b99cfd8803de Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 13 May 2026 23:34:40 +0200 Subject: [PATCH 10/20] Changelog, minor fixes --- CHANGELOG.md | 12 ++++++------ docs/02-usage/045_memories.md | 6 ++++-- pyproject.toml | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f310bc59..b9f67003 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,16 +13,15 @@ Status of the `main` branch. Changes prior to the next official version change w * Memories: - Memories can now reference each other using the `` `mem:NAME` `` convention. Renames propagate to all references automatically. See the [reference convention](https://oraios.github.io/serena/02-usage/045_memories.html#referencing-memories-from-other-memories). - - New `serena memories` CLI command group: `list`, `read`, `write`, `check` (referential - integrity report) and `auto-prefix-references` (heuristic rewrite of bare occurrences). - See the [CLI subcommands](https://oraios.github.io/serena/02-usage/045_memories.html#cli-subcommands). - Onboarding now seeds a `memory_maintenance` memory describing the memory-style and routing conventions, and the agent is instructed to read it before writing any other memories. A `global/memory_maintenance` memory takes precedence over the per-project seed. See the [memory maintenance section](https://oraios.github.io/serena/02-usage/045_memories.html#the-memory-maintenance-memory). - - Onboarding prompt rewritten to target a layout of `mem:core` (routing entry point), - `mem:suggested_commands`, `mem:tech_stack`, `mem:conventions`, `mem:task_completion`, - and optional per-module `mem:/core` memories, using dense agent-notes style. + +* CLI: + - Add `serena memories` CLI command group: `list`, `read`, `write`, `check` (referential + integrity report) and `auto-prefix-references` (heuristic rewrite of bare occurrences). + See the [CLI subcommands](https://oraios.github.io/serena/02-usage/045_memories.html#cli-subcommands). * Tools: - `edit_memory`: New `dotall` parameter (default `False`) controlling whether ``.`` matches @@ -31,6 +30,7 @@ Status of the `main` branch. Changes prior to the next official version change w avoids accidentally consuming large spans of content. Pass `dotall=True` for multi-line edits. - `search_for_pattern`: New `dotall` parameter (default `True`, preserving existing behavior) exposing the DOTALL flag, and the docstring was tightened. + - Delete `check_onboarding_performed` tool (instead extend project activation message), # v1.3.0 (2026-05-11) diff --git a/docs/02-usage/045_memories.md b/docs/02-usage/045_memories.md index e14d3b31..6c3560a9 100644 --- a/docs/02-usage/045_memories.md +++ b/docs/02-usage/045_memories.md @@ -4,6 +4,7 @@ Serena provides the functionality of a fully featured agent, and a useful aspect Despite its simplicity, we received positive feedback from many users who tend to combine it with their agent's internal memory management (e.g., `AGENTS.md` files). +(memories)= ## Memories Memories are simple, human-readable Markdown files that both you and @@ -45,10 +46,10 @@ criteria: prefix for references to memories, which does not prevent using the memory files outside of Serena. 7. **Configurable and composable.** Two orthogonal memory scopes - - [per-project](#memories) (committed alongside the code) and [global](global-memories) + [per-project](memories) (committed alongside the code) and [global](global-memories) (shared across all your projects) - can be combined freely. Within either scope, regex patterns in the global or project configuration can mark subsets as read-only - or [hide them entirely](#ignoring-memories) from the agent. This lets a project mix + or [hide them entirely](ignoring-memories) from the agent. This lets a project mix personal cross-project knowledge with checked-in project conventions, and selectively freeze either set, without custom plumbing. @@ -103,6 +104,7 @@ Since global memories are not versioned alongside your project files, it can be helpful to track global memories with git (i.e. to make `~/.serena/memories/` a git repository) in order to have a history of changes and the possibility to revert them if needed. +(ignoring-memories)= ### Ignoring Memories Projects that accumulate large numbers of archived memory files can use `ignored_memory_patterns` diff --git a/pyproject.toml b/pyproject.toml index f3e3e76b..e50286b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -361,7 +361,7 @@ markers = [ [tool.codespell] # Ref: https://github.com/codespell-project/codespell#using-a-config-file -skip = '.git*,*.svg,*.lock,*.min.*' +skip = '.git*,*.svg,*.lock,*.min.*,*test_memories_manager.py' check-hidden = true ignore-regex = '\.\w+' ignore-words-list = 'paket,EDN,als' From e3b2553d5b6bb1fa0f58d52b499ef927413ba4c6 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Thu, 14 May 2026 00:09:10 +0200 Subject: [PATCH 11/20] Refactoring: move memory reference analysis to separate module --- src/serena/memory_reference_analysis.py | 780 ++++++++++++++++++++++++ src/serena/project.py | 713 +--------------------- test/serena/test_memories_manager.py | 43 +- 3 files changed, 828 insertions(+), 708 deletions(-) create mode 100644 src/serena/memory_reference_analysis.py diff --git a/src/serena/memory_reference_analysis.py b/src/serena/memory_reference_analysis.py new file mode 100644 index 00000000..47b61f56 --- /dev/null +++ b/src/serena/memory_reference_analysis.py @@ -0,0 +1,780 @@ +""" +Memory reference matching, similarity scoring, and integrity reporting. + +This module contains the pure helpers and data types used by +:class:`serena.project.MemoriesManager` to (a) detect and rank references +between memories, and (b) report referential integrity issues. The +:class:`MemoryReferenceAnalyzer` class drives the validation and autofix +workflows against a live ``MemoriesManager`` instance. + +Kept separate from the manager so the matching heuristics can be evolved +and tested in isolation from filesystem and lifecycle concerns. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from typing import TYPE_CHECKING + +from sensai.util.string import TextBuilder + +if TYPE_CHECKING: + from serena.project import MemoriesManager + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +MEMORY_REF_PREFIX: str = "mem:" +"""Reference prefix used inside memory bodies to point at another memory.""" + +NAME_SIMILARITY_THRESHOLD: float = 0.55 +"""Minimum :func:`compute_name_similarity` score for a candidate to be reported.""" + +HIGH_CONFIDENCE_NAME_LENGTH: int = 10 +"""Length threshold above which a flat memory name is considered "name-shaped" (i.e. +unlikely to coincide with ordinary prose).""" + +SHORT_NAME_FLOOR: int = 3 +"""Below this basename length, only an exact token match keeps similarity above 0.""" + +FUZZY_BARE_TOKEN_JACCARD_FLOOR: float = 0.6 +"""For fuzzy bare-text matching, the bare token and the candidate existing name must +share at least this fraction of their tokenized name parts. Containment / SequenceMatcher +signals alone over-match — e.g. a generic prose word like ``repository`` is a substring of +``serena_repository_structure`` but their token Jaccard is only 1/3, so we reject it at +this floor.""" + +BASENAME_JACCARD_FLOOR: float = 0.34 +"""In :func:`compute_name_similarity` case 2 (basenames differ), when the two names also +carry *differing* topic prefixes, demand at least one strong basename signal: token +Jaccard >= this floor, basename containment, or typo-level seq ratio. Without this gate, +names like ``frontend/x-subtleties`` and ``backend/y-subtleties`` get matched purely on the +shared trailing token via the SequenceMatcher signal.""" + +BASENAME_TYPO_SEQ_FLOOR: float = 0.75 +"""Companion to :data:`BASENAME_JACCARD_FLOOR`: a basename seq ratio above this is treated +as typo-level edit distance and bypasses the Jaccard/containment requirement.""" + +MAX_STALE_REFERENCE_CANDIDATES: int = 3 +"""Hard cap on candidates proposed per stale reference. Beyond a small number the list +stops aiding disambiguation and becomes noise.""" + +WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: frozenset[str] = frozenset({"core"}) +"""Names that coincide with common English words and are filtered out of the unmarked- +reference scan to avoid false positives.""" + +NAME_CHAR_CLASS: str = r"[A-Za-z0-9_\-/]" +"""Character class delimiting a memory name. Used to anchor regex matches so we never +consume a partial name (kept in sync with the boundary rule in +:meth:`MemoriesManager.rename_references_to_memory`).""" + +_VERSION_SUFFIX_PATTERN: re.Pattern[str] = re.compile(r"_(?:v\d+|old|new|legacy|bak)$", re.IGNORECASE) + + +# --------------------------------------------------------------------------- +# Pure helpers: normalization, tokenization, similarity +# --------------------------------------------------------------------------- + + +def normalize_for_similarity(name: str) -> str: + """ + :param name: a memory name + :return: a lowercased copy of ``name`` with version/legacy-style trailing suffixes stripped + (e.g. ``"auth_v2"`` -> ``"auth"``); used as the canonical form for similarity scoring. + """ + return _VERSION_SUFFIX_PATTERN.sub("", name.lower()) + + +def tokenize_name(name: str) -> set[str]: + """ + :param name: a memory name + :return: the set of lowercase tokens extracted by splitting ``name`` on ``/``, ``_``, ``-`` + and at camelCase boundaries; empty tokens are dropped. + """ + parts = re.split(r"[/_\-]|(?<=[a-z])(?=[A-Z])", name) + return {p.lower() for p in parts if p} + + +def compute_name_similarity(a: str, b: str) -> float: + """ + Computes a similarity score in ``[0, 1]`` between two memory names. Examples + (with default threshold ``0.55``):: + + auth/login == auth/login -> 1.00 (exact) + auth_v1 ~ auth_v2 -> 1.00 (version suffix normalized) + login ~ auth/login -> 1.00 (flat <-> topic move) + auth/login ~ auth/v2/login -> 0.75 (shared prefix token + basename) + auth/login ~ security/login -> 0.50 (basename only; disjoint topics) + auth ~ authentication -> 0.64 (substring containment) + foo ~ for -> 0.00 (short-name floor) + frontend/x-subtleties ~ backend/y-subtleties -> 0.00 (different topics + only a + shared generic trailing token) + + :param a: first memory name + :param b: second memory name + :return: the similarity score + """ + # normalize: lowercase + strip version/legacy suffixes + norm_a, norm_b = normalize_for_similarity(a), normalize_for_similarity(b) + if norm_a == norm_b: + return 1.0 + + # split into prefix / basename + basename_a = norm_a.rsplit("/", 1)[-1] + basename_b = norm_b.rsplit("/", 1)[-1] + prefix_a = norm_a.rsplit("/", 1)[0] if "/" in norm_a else "" + prefix_b = norm_b.rsplit("/", 1)[0] if "/" in norm_b else "" + + # case 1: basenames match -- score is determined entirely by prefix relationship + if basename_a == basename_b: + if not prefix_a or not prefix_b: + # one side is flat -> canonical topic move, strongly confident + return 1.0 + # both sides carry a topic prefix -> blend basename match (0.5 floor) with prefix token similarity + prefix_tokens_a, prefix_tokens_b = tokenize_name(prefix_a), tokenize_name(prefix_b) + prefix_jaccard = ( + len(prefix_tokens_a & prefix_tokens_b) / len(prefix_tokens_a | prefix_tokens_b) if (prefix_tokens_a | prefix_tokens_b) else 0.0 + ) + return 0.5 + 0.5 * prefix_jaccard + + # case 2: basenames differ -- score the *basenames* only, so that a shared topic prefix + # plus a generic trailing token (e.g. "frontend/X-subtleties" vs "frontend/Y-subtleties") + # does not inflate the SequenceMatcher signal into a false positive. + basename_tokens_a, basename_tokens_b = tokenize_name(basename_a), tokenize_name(basename_b) + union_basename = basename_tokens_a | basename_tokens_b + basename_jaccard = (len(basename_tokens_a & basename_tokens_b) / len(union_basename)) if union_basename else 0.0 + basename_contained = bool(basename_a) and bool(basename_b) and (basename_a in basename_b or basename_b in basename_a) + seq = SequenceMatcher(None, basename_a, basename_b).ratio() + + # when both names carry *differing* topic prefixes, demand at least one strong basename + # signal — otherwise a single coincidental shared token at the end of long basenames + # under different topic roots slips through purely on the seq-ratio + if prefix_a and prefix_b and prefix_a != prefix_b: + if basename_jaccard < BASENAME_JACCARD_FLOOR and not basename_contained and seq < BASENAME_TYPO_SEQ_FLOOR: + return 0.0 + + # containment is a meaningful rename signal on its own (e.g. auth -> authentication), + # so we lift it above the threshold floor when one basename fully contains the other, + # with the length-ratio gating how confident we are + contained = 0.0 + if basename_contained: + length_ratio = min(len(basename_a), len(basename_b)) / max(len(basename_a), len(basename_b)) + contained = 0.5 + 0.5 * length_ratio + + # short-name floor: for very short basenames, only accept candidates with a strong token signal + if min(len(basename_a), len(basename_b)) <= SHORT_NAME_FLOOR and basename_jaccard < 1.0: + return 0.0 + return max(basename_jaccard, seq, contained) + + +def find_stale_reference_candidates(missing_name: str, existing_names: list[str], threshold: float | None = None) -> list[str]: + """ + :param missing_name: the unresolved reference target (without the ``mem:`` prefix). + :param existing_names: the names of all currently existing memories. + :param threshold: optional override for the minimum similarity required; defaults to + :data:`NAME_SIMILARITY_THRESHOLD`. + :return: existing memory names whose similarity to ``missing_name`` meets or exceeds + ``threshold``, sorted in descending order of similarity (ties broken alphabetically) + and capped at :data:`MAX_STALE_REFERENCE_CANDIDATES` entries to keep the output + focused on the most plausible matches. + """ + cutoff = NAME_SIMILARITY_THRESHOLD if threshold is None else threshold + scored: list[tuple[float, str]] = [] + for existing in existing_names: + score = compute_name_similarity(missing_name, existing) + if score >= cutoff: + scored.append((score, existing)) + scored.sort(key=lambda pair: (-pair[0], pair[1])) + return [name for _, name in scored[:MAX_STALE_REFERENCE_CANDIDATES]] + + +def iter_referenced_names_in_content(content: str) -> Iterator[str]: + """ + :param content: arbitrary memory content + :return: an iterator over the memory names appearing as ``mem:NAME`` references in + ``content``; duplicate occurrences are yielded once each. + """ + # use the same boundary rule as MemoriesManager.rename_references_to_memory so the two stay consistent + pattern = rf"(? int: + """ + :param content: arbitrary memory content + :param name: a memory name to look for + :return: the count of bare (i.e. not already ``mem:``-prefixed) word-boundary-anchored + occurrences of ``name`` in ``content``. The match must not be preceded by ``mem:`` + and must not be embedded within a longer memory-name-like run of characters. + """ + pattern = ( + rf"(? tuple[str, int]: + """ + :param content: arbitrary memory content + :param name: the memory name whose bare occurrences should be rewritten + :return: ``(new_content, n_replacements)`` after prefixing each bare occurrence of + ``name`` with ``mem:`` (using the same boundary rule as :func:`find_bare_occurrences`). + """ + pattern = ( + rf"(? Iterator[tuple[str, int]]: + """ + :param content: arbitrary memory content + :return: an iterator over ``(token, count)`` pairs for each distinct name-shaped + token in ``content`` whose length meets :data:`HIGH_CONFIDENCE_NAME_LENGTH` + and which is not preceded by ``mem:`` (those are already valid references). + The boundaries follow :data:`NAME_CHAR_CLASS`, so embedded substrings of + longer name-character runs are not matched. + """ + pattern = ( + rf"(? bool: + """ + :return: True iff ``suspected_name`` would refer to ``source_memory`` itself + (including the case where ``suspected_name`` equals the basename of a topic-path + source memory). + """ + if suspected_name == source_memory: + return True + source_basename = source_memory.rsplit("/", 1)[-1] + return suspected_name == source_basename + + +# --------------------------------------------------------------------------- +# Report dataclasses +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class StaleReference: + """ + A ``mem:NAME`` reference whose target memory does not exist. + + :ivar source_memory: the name of the memory whose content contains the broken reference. + :ivar referenced_name: the name following ``mem:`` that did not resolve to an existing memory. + :ivar candidates: existing memory names proposed as likely intended targets, ranked by + decreasing similarity. May be empty if no candidate exceeded the similarity threshold. + :ivar source_is_read_only: whether the source memory is read-only. + """ + + source_memory: str + referenced_name: str + candidates: list[str] + source_is_read_only: bool + + +@dataclass(frozen=True) +class UnmarkedReferenceWarning: + """ + A bare occurrence in a memory's content that looks like a forgotten reference to an + existing memory. + + Two flavours of finding share this class: + + * **exact match** — the bare text equals an existing memory name verbatim + (``actual_token`` either equals ``suspected_name`` or is left empty). + * **fuzzy near-miss** — the bare text does not equal any existing memory name, but + a long, distinctive token in the body similarity-matches a high-confidence existing + memory name (``actual_token`` is the bare text actually found, distinct from + ``suspected_name``). Such findings are reported but **not** rewritten by + :meth:`MemoryReferenceAnalyzer.auto_prefix_bare_references`, since they would + require substring substitution rather than a prefix addition. + + :ivar source_memory: the name of the memory whose content contains the bare occurrence. + :ivar suspected_name: the existing memory name proposed as the intended target. + :ivar occurrences: the number of occurrences of ``actual_token`` in the source memory's content. + :ivar is_high_confidence: True when ``suspected_name`` contains a ``/`` separator or + exceeds the configured length threshold; such names are unlikely to coincide with + ordinary prose. False otherwise (a low-confidence warning). + :ivar source_is_read_only: whether the source memory is read-only. + :ivar actual_token: the bare text actually found in the source memory's content. + Defaults to ``""``, which is taken to mean ``suspected_name`` (the exact-match case). + When non-empty and different from ``suspected_name``, this is a fuzzy near-miss. + """ + + source_memory: str + suspected_name: str + occurrences: int + is_high_confidence: bool + source_is_read_only: bool + actual_token: str = "" + + @property + def is_exact_match(self) -> bool: + """:return: True iff the bare text in the body equals ``suspected_name`` (i.e. not a fuzzy near-miss).""" + return self.actual_token == "" or self.actual_token == self.suspected_name + + +@dataclass(frozen=True) +class AutofixedReference: + """ + A bare occurrence rewritten to include the ``mem:`` prefix. + + :ivar source_memory: the name of the memory whose content was modified. + :ivar referenced_name: the memory name whose bare occurrences were prefixed. + :ivar n_replacements: the number of bare occurrences replaced in the source memory. + """ + + source_memory: str + referenced_name: str + n_replacements: int + + +@dataclass +class ReferentialIntegrityReport: + """ + Outcome of :meth:`MemoryReferenceAnalyzer.validate_referential_integrity`. + + :ivar stale_references: ``mem:NAME`` references whose target memory does not exist. + :ivar high_confidence_unmarked_memories: bare references whose suspected target name is unlikely + to be coincidental prose (topic-path or sufficiently long). + :ivar low_confidence_unmarked_memories: bare references whose suspected target name could plausibly + appear in ordinary prose (short, flat names). + """ + + stale_references: list[StaleReference] = field(default_factory=list) + high_confidence_unmarked_memories: list[UnmarkedReferenceWarning] = field(default_factory=list) + low_confidence_unmarked_memories: list[UnmarkedReferenceWarning] = field(default_factory=list) + + def is_clean(self) -> bool: + """:return: True iff no stale references and no warnings of any confidence level were found.""" + return not (self.stale_references or self.high_confidence_unmarked_memories or self.low_confidence_unmarked_memories) + + def format(self) -> str: + """:return: a human-readable rendering suitable for CLI display.""" + tb = TextBuilder() + if self.is_clean(): + tb.with_line("✓ No referential integrity issues found.") + return tb.build() + + # stale references — grouped by writable / read-only for clarity + if self.stale_references: + tb.with_line(f"Stale references ({len(self.stale_references)}):") + for ref in self.stale_references: + ro_tag = " [read-only source]" if ref.source_is_read_only else "" + tb.with_line(f" - `mem:{ref.referenced_name}` in `{ref.source_memory}`{ro_tag}") + if ref.candidates: + candidates_str = ", ".join(f"`mem:{c}`" for c in ref.candidates) + tb.with_line(f" candidates: {candidates_str}") + else: + tb.with_line(" candidates: (none above similarity threshold)") + tb.with_line("") + + # unmarked-reference warnings — split exact vs fuzzy near-miss, then by confidence + exact_high = [w for w in self.high_confidence_unmarked_memories if w.is_exact_match] + exact_low = [w for w in self.low_confidence_unmarked_memories if w.is_exact_match] + fuzzy = [w for w in self.high_confidence_unmarked_memories if not w.is_exact_match] + [ + w for w in self.low_confidence_unmarked_memories if not w.is_exact_match + ] + + # the "may be intentional" caveat is meaningful only for exact matches (where the bare + # text equals a real memory name and may simply be ordinary prose); print it once, + # before the first exact-match section + if exact_high or exact_low: + tb.with_line("Possibly unmarked references — exact matches:") + tb.with_line(" (note: bare matches in prose may be intentional and not actual references)") + for label, warnings in (("high confidence", exact_high), ("low confidence", exact_low)): + if not warnings: + continue + tb.with_line(f" {label} ({len(warnings)}):") + for w in warnings: + ro_tag = " [read-only source]" if w.source_is_read_only else "" + occ = "occurrence" if w.occurrences == 1 else "occurrences" + tb.with_line( + f" - {w.occurrences} {occ} of `{w.suspected_name}` in `{w.source_memory}`{ro_tag}" + f" (suggested: `mem:{w.suspected_name}`)" + ) + tb.with_line("") + + if fuzzy: + tb.with_line(f"Possibly unmarked references — fuzzy near-misses ({len(fuzzy)}):") + for w in fuzzy: + ro_tag = " [read-only source]" if w.source_is_read_only else "" + occ = "occurrence" if w.occurrences == 1 else "occurrences" + tb.with_line( + f" - {w.occurrences} {occ} of `{w.actual_token}` in `{w.source_memory}`{ro_tag}" + f" — near-miss (suggested: `mem:{w.suspected_name}`)" + ) + tb.with_line("") + return tb.build() + + +@dataclass +class AutofixReport: + """ + Outcome of :meth:`MemoryReferenceAnalyzer.auto_prefix_bare_references`. + + :ivar autofixed: per-(source, target) records of bare references that were rewritten. + When the call was a dry run, these records describe what *would* have been + written; the files themselves are unchanged. + :ivar dry_run: True if the run was a preview that did not write any files. + :ivar skipped_read_only: warnings whose source memory was read-only and therefore not + modified (only populated when ``include_read_only`` is False). + :ivar skipped_flat: warnings skipped because their suspected target had no ``/`` + separator and was not long enough to be high-confidence (only populated when + ``include_flat_names`` is False). + :ivar skipped_global: warnings skipped because their source memory was global and + ``include_global`` was False. + :ivar skipped_fuzzy: warnings whose bare text in the source memory differs from the + suspected target (fuzzy near-misses). These require substring substitution rather + than a prefix addition and are never autofixed; surface them to the user for + manual review instead. + """ + + autofixed: list[AutofixedReference] = field(default_factory=list) + dry_run: bool = False + skipped_read_only: list[UnmarkedReferenceWarning] = field(default_factory=list) + skipped_flat: list[UnmarkedReferenceWarning] = field(default_factory=list) + skipped_global: list[UnmarkedReferenceWarning] = field(default_factory=list) + skipped_fuzzy: list[UnmarkedReferenceWarning] = field(default_factory=list) + + @property + def total_replacements(self) -> int: + return sum(a.n_replacements for a in self.autofixed) + + def format(self) -> str: + """:return: a human-readable rendering suitable for CLI display.""" + tb = TextBuilder() + verb = "Would apply" if self.dry_run else "Applied" + if self.autofixed: + tb.with_line( + f"{verb} {self.total_replacements} replacement(s) across " + f"{len(self.autofixed)} memory/target pair(s)" + (" (dry run; no files were modified)" if self.dry_run else "") + ":" + ) + for a in self.autofixed: + tb.with_line(f" - {a.n_replacements} x `{a.referenced_name}` -> `mem:{a.referenced_name}` in `{a.source_memory}`") + tb.with_line("") + else: + tb.with_line("No replacements" + (" would be" if self.dry_run else "") + " applied.") + tb.with_line("") + + for label, items in ( + ("Skipped (read-only source; pass --include-read-only to override)", self.skipped_read_only), + ("Skipped (flat name; pass --include-flat-names to include)", self.skipped_flat), + ("Skipped (global memory; pass --include-global to include)", self.skipped_global), + ( + "Skipped (fuzzy near-miss; bare text differs from target name — not safe to auto-rewrite, review manually)", + self.skipped_fuzzy, + ), + ): + if not items: + continue + tb.with_line(f"{label}:") + for w in items: + if w.is_exact_match: + tb.with_line(f" - `{w.suspected_name}` in `{w.source_memory}`") + else: + tb.with_line(f" - `{w.actual_token}` in `{w.source_memory}` (suggested: `mem:{w.suspected_name}`)") + tb.with_line("") + return tb.build() + + +# --------------------------------------------------------------------------- +# Analyzer: ties the helpers above to a live MemoriesManager instance +# --------------------------------------------------------------------------- + + +class MemoryReferenceAnalyzer: + """ + Drives validation and autofix workflows against a :class:`MemoriesManager`. + + Composition rather than inheritance — the analyzer only needs the manager's + memory enumeration, loading, saving, and is-global helpers, and is otherwise + independent of project-management concerns. + """ + + def __init__(self, manager: MemoriesManager): + self._manager = manager + + def validate_referential_integrity( + self, include_unmarked: bool = True, include_fuzzy_matching: bool = True + ) -> ReferentialIntegrityReport: + """ + Scans every (non-ignored) memory's content for referential integrity issues. + + The scan covers both project-local and global memories. Three kinds of finding are + produced, each independently gated: + + * **stale references** — occurrences of ``mem:NAME`` where ``NAME`` does not resolve + to an existing memory. For each, a list of similarly-named existing memories is + proposed as candidate intended targets (see :func:`compute_name_similarity`), + capped at :data:`MAX_STALE_REFERENCE_CANDIDATES`. Always reported. + * **exact unmarked-reference warnings** — bare occurrences of an existing memory + name that appear without the ``mem:`` prefix. Split into high-confidence (the + suspected name contains a ``/`` or exceeds :data:`HIGH_CONFIDENCE_NAME_LENGTH`) + and low-confidence groups. Candidate names whose basename is in + :data:`WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES` are skipped. Gated by + ``include_unmarked``. + * **fuzzy near-miss warnings** — long, distinctive bare tokens in a memory body + that *do not* match an existing name exactly but similarity-match a + high-confidence existing name AND share at least + :data:`FUZZY_BARE_TOKEN_JACCARD_FLOOR` of their tokenized name parts. Always + reported as high-confidence; the actual bare text is preserved on + :attr:`UnmarkedReferenceWarning.actual_token`. Gated by ``include_unmarked + and include_fuzzy_matching`` — fuzzy matches are noisy and only meaningful when + unmarked-reference checking is enabled. + + Self-references (a memory's content mentioning its own name or basename) and empty + memories are skipped silently. This method has no side effects. + + :param include_unmarked: if False, skip both the exact unmarked scan and the fuzzy + near-miss scan; only stale references are reported. + :param include_fuzzy_matching: if False, skip the fuzzy near-miss scan even when + unmarked-reference checking is otherwise enabled. Has no effect when + ``include_unmarked`` is False. + :return: a :class:`ReferentialIntegrityReport` summarizing all findings. + """ + report = ReferentialIntegrityReport() + + # gather all existing memory names (project + global) with their read-only status + memories_list = self._manager.list_memories() + all_names = sorted(set(memories_list.memories) | set(memories_list.read_only_memories)) + read_only_names = set(memories_list.read_only_memories) + existing_set = set(all_names) + # only existing names that are themselves "memory-name-like" qualify as fuzzy candidates, + # otherwise short flat names would match arbitrary long prose words via containment + high_confidence_names = [n for n in all_names if "/" in n or len(n) >= HIGH_CONFIDENCE_NAME_LENGTH] + + for source_name in all_names: + # content of this memory should not be analyzed regarding stale references + if source_name == self._manager.MEMORY_MAINTENANCE_NAME: + continue + + content = self._manager.load_memory(source_name) + if not content: + # skip empty memories silently — no references can exist in empty content + continue + source_is_read_only = source_name in read_only_names + + # stale references: every mem:NAME whose NAME is not an existing memory + stale_seen: set[str] = set() + for referenced in iter_referenced_names_in_content(content): + if referenced in existing_set or referenced in stale_seen: + continue + stale_seen.add(referenced) + candidates = find_stale_reference_candidates(referenced, all_names) + report.stale_references.append( + StaleReference( + source_memory=source_name, + referenced_name=referenced, + candidates=candidates, + source_is_read_only=source_is_read_only, + ) + ) + + if not include_unmarked: + continue + + # exact unmarked references: bare occurrences of any existing memory name (except self) + for candidate_name in all_names: + if is_self_reference(source_name, candidate_name): + continue + # skip candidate names whose basename is an ambiguous common word + if candidate_name.rsplit("/", 1)[-1] in WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: + continue + n = find_bare_occurrences(content, candidate_name) + if n == 0: + continue + is_high_conf = "/" in candidate_name or len(candidate_name) >= HIGH_CONFIDENCE_NAME_LENGTH + warning = UnmarkedReferenceWarning( + source_memory=source_name, + suspected_name=candidate_name, + occurrences=n, + is_high_confidence=is_high_conf, + source_is_read_only=source_is_read_only, + actual_token=candidate_name, + ) + if is_high_conf: + report.high_confidence_unmarked_memories.append(warning) + else: + report.low_confidence_unmarked_memories.append(warning) + + if not include_fuzzy_matching: + continue + + # fuzzy near-misses: long bare tokens that similarity-match a high-confidence existing name + for token, n in iter_long_bare_tokens(content): + if token in existing_set: + # exact-match path already handled this token (or self-referenced and was skipped) + continue + # filter bare tokens that coincide with common English words; the same filter is + # applied to candidate names in the exact-match loop, this is the symmetric side + token_norm = normalize_for_similarity(token) + if token_norm in WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: + continue + token_tokens = tokenize_name(token) + best_name: str | None = None + best_score = 0.0 + for existing in high_confidence_names: + if existing.rsplit("/", 1)[-1] in WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: + continue + # require substring containment between bare token and existing name: fuzzy + # matches without a containment relationship are rarely useful as rewrite + # suggestions (auto_prefix_bare_references can't fix them anyway) and dominate + # the false-positive rate, e.g. "grid-layout" vs "layout-grid-subtleties". + existing_norm = normalize_for_similarity(existing) + if token_norm not in existing_norm and existing_norm not in token_norm: + continue + score = compute_name_similarity(token, existing) + if score < NAME_SIMILARITY_THRESHOLD: + continue + # safeguard against substring-only matches: require meaningful token overlap so + # generic prose words don't get flagged just because they happen to be substrings + # of an existing memory name + existing_tokens = tokenize_name(existing) + union = token_tokens | existing_tokens + if not union: + continue + jaccard = len(token_tokens & existing_tokens) / len(union) + if jaccard < FUZZY_BARE_TOKEN_JACCARD_FLOOR: + continue + if score > best_score: + best_score = score + best_name = existing + if best_name is None: + continue + # skip self-suggestions: a memory's body containing a near-miss of its own + # name (e.g. capitalised "Conventions" in source `conventions`) would otherwise + # produce a warning pointing the source at itself + if normalize_for_similarity(best_name) == normalize_for_similarity(source_name): + continue + report.high_confidence_unmarked_memories.append( + UnmarkedReferenceWarning( + source_memory=source_name, + suspected_name=best_name, + occurrences=n, + is_high_confidence=True, + source_is_read_only=source_is_read_only, + actual_token=token, + ) + ) + + return report + + def auto_prefix_bare_references( + self, + include_flat_names: bool = False, + include_read_only: bool = False, + include_global: bool = False, + dry_run: bool = False, + ) -> AutofixReport: + """ + Rewrites *exact* bare occurrences of existing memory names by adding the ``mem:`` prefix. + + .. warning:: + **This is a heuristic, file-mutating operation** (unless ``dry_run`` is True). A + bare word that happens to coincide with a memory name will be rewritten as a + reference, even if it was intended as ordinary prose. Pass ``dry_run=True`` to + preview the rewrites before applying them. + + Scope is intentionally narrower than what :meth:`validate_referential_integrity` + reports: + + * Only **exact** bare occurrences are rewritten — i.e. the bare text in the source + body must equal an existing memory name verbatim. Fuzzy near-miss findings + (where the actual token differs from the suspected target) require substring + substitution rather than a prefix addition and are routed into + :attr:`AutofixReport.skipped_fuzzy` for manual review. + * By default the rewrite is restricted to *high-confidence* findings only — those + whose suspected target name contains a ``/`` separator or exceeds the + configured length threshold — and skips global memories and read-only memories. + The defaults intentionally err toward false negatives over false positives. + + :param include_flat_names: if True, also rewrite low-confidence findings (flat, + short memory names). Increases recall but markedly raises false-positive risk. + :param include_read_only: if True, also rewrite occurrences inside read-only + memories. Use with care, as read-only memories are typically considered + authoritative. + :param include_global: if True, also rewrite occurrences inside global memories. + Modifying a global memory affects every project that consumes it. + :param dry_run: if True, the report describes the rewrites that *would* be applied + but no files are modified. + :return: an :class:`AutofixReport` describing every replacement (made or planned) + and every warning that was deliberately skipped. + """ + report = AutofixReport(dry_run=dry_run) + + # build the integrity report first so autofix decisions are made on the same data + validation = self.validate_referential_integrity() + + # combine the warning groups according to the include_flat_names policy + warnings: list[UnmarkedReferenceWarning] = list(validation.high_confidence_unmarked_memories) + if include_flat_names: + warnings.extend(validation.low_confidence_unmarked_memories) + else: + report.skipped_flat.extend(validation.low_confidence_unmarked_memories) + + # apply replacements per source memory, grouping warnings to avoid re-reading content + warnings_by_source: dict[str, list[UnmarkedReferenceWarning]] = {} + for w in warnings: + # fuzzy near-misses are never auto-rewritten — the bare text differs from the target + if not w.is_exact_match: + report.skipped_fuzzy.append(w) + continue + # filter remaining warnings according to scope flags + if w.source_is_read_only and not include_read_only: + report.skipped_read_only.append(w) + continue + if self._manager._is_global(w.source_memory) and not include_global: + report.skipped_global.append(w) + continue + warnings_by_source.setdefault(w.source_memory, []).append(w) + + for source_memory, source_warnings in warnings_by_source.items(): + content = self._manager.load_memory(source_memory) + total_n = 0 + per_target_records: list[AutofixedReference] = [] + + # apply replacements sequentially; each pass uses the (potentially updated) content + for w in source_warnings: + content, n = add_bare_occurrences_prefix(content, w.suspected_name) + if n > 0: + per_target_records.append( + AutofixedReference( + source_memory=source_memory, + referenced_name=w.suspected_name, + n_replacements=n, + ) + ) + total_n += n + + if total_n > 0: + if not dry_run: + # use is_tool_context=False so we don't trip read-only protection when --include-read-only is set + self._manager.save_memory(source_memory, content, is_tool_context=False) + report.autofixed.extend(per_target_records) + + return report diff --git a/src/serena/project.py b/src/serena/project.py index 480a4a37..5ba84ca6 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -3,9 +3,7 @@ import os import re import shutil import threading -from collections.abc import Iterator, Sequence -from dataclasses import dataclass, field -from difflib import SequenceMatcher +from collections.abc import Sequence from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, Optional @@ -20,6 +18,11 @@ from serena.config.serena_config import ( ) from serena.constants import SERENA_FILE_ENCODING from serena.ls_manager import LanguageServerFactory, LanguageServerManager +from serena.memory_reference_analysis import ( + AutofixReport, + MemoryReferenceAnalyzer, + ReferentialIntegrityReport, +) from serena.util.file_system import GitignoreParser, match_path from serena.util.text_utils import ContentReplacer, MatchedConsecutiveLines, search_files from solidlsp import SolidLanguageServer @@ -157,221 +160,6 @@ class MemoriesManager: # use a callable replacement to avoid backreference interpretation of characters in ref_new return re.subn(pattern, lambda _m: ref_new, content) - NAME_SIMILARITY_THRESHOLD: float = 0.55 - _HIGH_CONFIDENCE_NAME_LENGTH: int = 10 - _SHORT_NAME_FLOOR: int = 3 - # For fuzzy bare-text matching (validate_referential_integrity), an additional safeguard: - # the bare token and the candidate existing name must share at least this fraction of their - # tokenized name parts. Containment / SequenceMatcher signals alone over-match — e.g. a - # generic prose word like "repository" is a substring of "serena_repository_structure" - # but their token Jaccard is only 1/3, so we reject it at this floor. - _FUZZY_BARE_TOKEN_JACCARD_FLOOR: float = 0.6 - # In :meth:`compute_name_similarity` case 2 (basenames differ), when the two names also - # carry *differing* topic prefixes, demand at least one strong basename signal: token - # Jaccard >= this floor, basename containment, or typo-level seq ratio. Without this - # gate, names like ``frontend/x-subtleties`` and ``backend/y-subtleties`` get matched - # purely on the shared trailing token via the SequenceMatcher signal. - _BASENAME_JACCARD_FLOOR: float = 0.34 - _BASENAME_TYPO_SEQ_FLOOR: float = 0.75 - # Hard cap on candidates proposed per stale reference. Beyond a small number the list - # stops aiding disambiguation and becomes noise. - MAX_STALE_REFERENCE_CANDIDATES: int = 3 - - _WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: frozenset[str] = frozenset({"core"}) - """Names that we specifically mention in our instructions and that coincide with common words""" - _VERSION_SUFFIX_PATTERN = re.compile(r"_(?:v\d+|old|new|legacy|bak)$", re.IGNORECASE) - - @classmethod - def _normalize_for_similarity(cls, name: str) -> str: - """ - :param name: a memory name - :return: a lowercased copy of ``name`` with version/legacy-style trailing suffixes stripped - (e.g. ``"auth_v2"`` -> ``"auth"``); used as the canonical form for similarity scoring. - """ - return cls._VERSION_SUFFIX_PATTERN.sub("", name.lower()) - - @classmethod - def _tokenize_name(cls, name: str) -> set[str]: - """ - :param name: a memory name - :return: the set of lowercase tokens extracted by splitting ``name`` on ``/``, ``_``, ``-`` - and at camelCase boundaries; empty tokens are dropped. - """ - # split on path/word separators and at camelCase boundaries - parts = re.split(r"[/_\-]|(?<=[a-z])(?=[A-Z])", name) - return {p.lower() for p in parts if p} - - @classmethod - def compute_name_similarity(cls, a: str, b: str) -> float: - """ - Computes a similarity score in ``[0, 1]`` between two memory names. Examples - (with default threshold ``0.55``):: - - auth/login == auth/login -> 1.00 (exact) - auth_v1 ~ auth_v2 -> 1.00 (version suffix normalized) - login ~ auth/login -> 1.00 (flat <-> topic move) - auth/login ~ auth/v2/login -> 0.75 (shared prefix token + basename) - auth/login ~ security/login -> 0.50 (basename only; disjoint topics) - auth ~ authentication -> 0.64 (substring containment) - foo ~ for -> 0.00 (short-name floor) - frontend/x-subtleties ~ backend/y-subtleties -> 0.00 (different topics + only a - shared generic trailing token) - - :param a: first memory name - :param b: second memory name - :return: the similarity score - """ - # normalize: lowercase + strip version/legacy suffixes - norm_a, norm_b = cls._normalize_for_similarity(a), cls._normalize_for_similarity(b) - if norm_a == norm_b: - return 1.0 - - # split into prefix / basename - basename_a = norm_a.rsplit("/", 1)[-1] - basename_b = norm_b.rsplit("/", 1)[-1] - prefix_a = norm_a.rsplit("/", 1)[0] if "/" in norm_a else "" - prefix_b = norm_b.rsplit("/", 1)[0] if "/" in norm_b else "" - - # case 1: basenames match -- score is determined entirely by prefix relationship - if basename_a == basename_b: - if not prefix_a or not prefix_b: - # one side is flat -> canonical topic move, strongly confident - return 1.0 - # both sides carry a topic prefix -> blend basename match (0.5 floor) with prefix token similarity - prefix_tokens_a, prefix_tokens_b = cls._tokenize_name(prefix_a), cls._tokenize_name(prefix_b) - prefix_jaccard = ( - len(prefix_tokens_a & prefix_tokens_b) / len(prefix_tokens_a | prefix_tokens_b) - if (prefix_tokens_a | prefix_tokens_b) - else 0.0 - ) - return 0.5 + 0.5 * prefix_jaccard - - # case 2: basenames differ -- score the *basenames* only, so that a shared topic prefix - # plus a generic trailing token (e.g. "frontend/X-subtleties" vs "frontend/Y-subtleties") - # does not inflate the SequenceMatcher signal into a false positive. - basename_tokens_a, basename_tokens_b = cls._tokenize_name(basename_a), cls._tokenize_name(basename_b) - union_basename = basename_tokens_a | basename_tokens_b - basename_jaccard = (len(basename_tokens_a & basename_tokens_b) / len(union_basename)) if union_basename else 0.0 - basename_contained = bool(basename_a) and bool(basename_b) and (basename_a in basename_b or basename_b in basename_a) - seq = SequenceMatcher(None, basename_a, basename_b).ratio() - - # when both names carry *differing* topic prefixes, demand at least one strong basename - # signal — otherwise a single coincidental shared token at the end of long basenames - # under different topic roots slips through purely on the seq-ratio - if prefix_a and prefix_b and prefix_a != prefix_b: - if basename_jaccard < cls._BASENAME_JACCARD_FLOOR and not basename_contained and seq < cls._BASENAME_TYPO_SEQ_FLOOR: - return 0.0 - - # containment is a meaningful rename signal on its own (e.g. auth -> authentication), - # so we lift it above the threshold floor when one basename fully contains the other, - # with the length-ratio gating how confident we are - contained = 0.0 - if basename_contained: - length_ratio = min(len(basename_a), len(basename_b)) / max(len(basename_a), len(basename_b)) - contained = 0.5 + 0.5 * length_ratio - - # short-name floor: for very short basenames, only accept candidates with a strong token signal - if min(len(basename_a), len(basename_b)) <= cls._SHORT_NAME_FLOOR and basename_jaccard < 1.0: - return 0.0 - return max(basename_jaccard, seq, contained) - - @classmethod - def _find_stale_reference_candidates(cls, missing_name: str, existing_names: list[str], threshold: float | None = None) -> list[str]: - """ - :param missing_name: the unresolved reference target (without the ``mem:`` prefix). - :param existing_names: the names of all currently existing memories. - :param threshold: optional override for the minimum similarity required; defaults to - :attr:`NAME_SIMILARITY_THRESHOLD`. - :return: existing memory names whose similarity to ``missing_name`` meets or exceeds - ``threshold``, sorted in descending order of similarity (ties broken alphabetically) - and capped at :attr:`MAX_STALE_REFERENCE_CANDIDATES` entries to keep the output - focused on the most plausible matches. - """ - cutoff = cls.NAME_SIMILARITY_THRESHOLD if threshold is None else threshold - scored: list[tuple[float, str]] = [] - for existing in existing_names: - score = cls.compute_name_similarity(missing_name, existing) - if score >= cutoff: - scored.append((score, existing)) - scored.sort(key=lambda pair: (-pair[0], pair[1])) - return [name for _, name in scored[: cls.MAX_STALE_REFERENCE_CANDIDATES]] - - # character class delimiting a memory name; used to anchor regex matches so we never - # consume a partial name (kept in sync with the boundary rule in rename_references_to_memory) - _NAME_CHAR_CLASS: str = r"[A-Za-z0-9_\-/]" - - @classmethod - def _iter_referenced_names_in_content(cls, content: str) -> Iterator[str]: - """ - :param content: arbitrary memory content - :return: an iterator over the memory names appearing as ``mem:NAME`` references in - ``content``; duplicate occurrences are yielded once each. - """ - # use the same boundary rule as rename_references_to_memory so the two stay consistent - pattern = rf"(? int: - """ - :param content: arbitrary memory content - :param name: a memory name to look for - :return: the count of bare (i.e. not already ``mem:``-prefixed) word-boundary-anchored - occurrences of ``name`` in ``content``. The match must not be preceded by ``mem:`` - and must not be embedded within a longer memory-name-like run of characters. - """ - # negative lookbehind for the mem: prefix prevents double-counting valid references; - # the additional name-char lookbehind/lookahead prevents matching inside longer names - pattern = ( - rf"(? tuple[str, int]: - """ - :param content: arbitrary memory content - :param name: the memory name whose bare occurrences should be rewritten - :return: ``(new_content, n_replacements)`` after prefixing each bare occurrence of - ``name`` with ``mem:`` (using the same boundary rule as - :meth:`_find_bare_occurrences`). - """ - pattern = ( - rf"(? Iterator[tuple[str, int]]: - """ - :param content: arbitrary memory content - :return: an iterator over ``(token, count)`` pairs for each distinct name-shaped - token in ``content`` whose length meets :attr:`_HIGH_CONFIDENCE_NAME_LENGTH` - and which is not preceded by ``mem:`` (those are already valid references). - The boundaries follow :attr:`_NAME_CHAR_CLASS`, so embedded substrings of - longer name-character runs are not matched. - """ - # require minimum length so we only scan tokens that are unlikely to be coincidental prose - pattern = ( - rf"(? Path: name = self._prepare_name(name) parts = name.split("/") @@ -458,225 +246,6 @@ class MemoriesManager: def get_full_list(self) -> list[str]: return sorted(self.memories + self.read_only_memories) - @dataclass(frozen=True) - class StaleReference: - """ - A ``mem:NAME`` reference whose target memory does not exist. - - :ivar source_memory: the name of the memory whose content contains the broken reference. - :ivar referenced_name: the name following ``mem:`` that did not resolve to an existing memory. - :ivar candidates: existing memory names proposed as likely intended targets, ranked by - decreasing similarity. May be empty if no candidate exceeded the similarity threshold. - :ivar source_is_read_only: whether the source memory is read-only. - """ - - source_memory: str - referenced_name: str - candidates: list[str] - source_is_read_only: bool - - @dataclass(frozen=True) - class UnmarkedReferenceWarning: - """ - A bare occurrence in a memory's content that looks like a forgotten reference to an - existing memory. - - Two flavours of finding share this class: - - * **exact match** — the bare text equals an existing memory name verbatim - (``actual_token`` either equals ``suspected_name`` or is left empty). - * **fuzzy near-miss** — the bare text does not equal any existing memory name, but - a long, distinctive token in the body similarity-matches a high-confidence existing - memory name (``actual_token`` is the bare text actually found, distinct from - ``suspected_name``). Such findings are reported but **not** rewritten by - :meth:`auto_prefix_bare_references`, since they would require substring - substitution rather than a prefix addition. - - :ivar source_memory: the name of the memory whose content contains the bare occurrence. - :ivar suspected_name: the existing memory name proposed as the intended target. - :ivar occurrences: the number of occurrences of ``actual_token`` in the source memory's content. - :ivar is_high_confidence: True when ``suspected_name`` contains a ``/`` separator or - exceeds the configured length threshold; such names are unlikely to coincide with - ordinary prose. False otherwise (a low-confidence warning). - :ivar source_is_read_only: whether the source memory is read-only. - :ivar actual_token: the bare text actually found in the source memory's content. - Defaults to ``""``, which is taken to mean ``suspected_name`` (the exact-match case). - When non-empty and different from ``suspected_name``, this is a fuzzy near-miss. - """ - - source_memory: str - suspected_name: str - occurrences: int - is_high_confidence: bool - source_is_read_only: bool - actual_token: str = "" - - @property - def is_exact_match(self) -> bool: - """:return: True iff the bare text in the body equals ``suspected_name`` (i.e. not a fuzzy near-miss).""" - return self.actual_token == "" or self.actual_token == self.suspected_name - - @dataclass(frozen=True) - class AutofixedReference: - """ - A bare occurrence rewritten to include the ``mem:`` prefix. - - :ivar source_memory: the name of the memory whose content was modified. - :ivar referenced_name: the memory name whose bare occurrences were prefixed. - :ivar n_replacements: the number of bare occurrences replaced in the source memory. - """ - - source_memory: str - referenced_name: str - n_replacements: int - - @dataclass - class ReferentialIntegrityReport: - """ - Outcome of :meth:`MemoriesManager.validate_referential_integrity`. - - :ivar stale_references: ``mem:NAME`` references whose target memory does not exist. - :ivar high_confidence_unmarked_memories: bare references whose suspected target name is unlikely - to be coincidental prose (topic-path or sufficiently long). - :ivar low_confidence_unmarked_memories: bare references whose suspected target name could plausibly - appear in ordinary prose (short, flat names). - """ - - stale_references: list["MemoriesManager.StaleReference"] = field(default_factory=list) - high_confidence_unmarked_memories: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) - low_confidence_unmarked_memories: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) - - def is_clean(self) -> bool: - """:return: True iff no stale references and no warnings of any confidence level were found.""" - return not (self.stale_references or self.high_confidence_unmarked_memories or self.low_confidence_unmarked_memories) - - def format(self) -> str: - """:return: a human-readable rendering suitable for CLI display.""" - tb = TextBuilder() - if self.is_clean(): - tb.with_line("✓ No referential integrity issues found.") - return tb.build() - - # stale references — grouped by writable / read-only for clarity - if self.stale_references: - tb.with_line(f"Stale references ({len(self.stale_references)}):") - for ref in self.stale_references: - ro_tag = " [read-only source]" if ref.source_is_read_only else "" - tb.with_line(f" - `mem:{ref.referenced_name}` in `{ref.source_memory}`{ro_tag}") - if ref.candidates: - candidates_str = ", ".join(f"`mem:{c}`" for c in ref.candidates) - tb.with_line(f" candidates: {candidates_str}") - else: - tb.with_line(" candidates: (none above similarity threshold)") - tb.with_line("") - - # unmarked-reference warnings — split exact vs fuzzy near-miss, then by confidence - exact_high = [w for w in self.high_confidence_unmarked_memories if w.is_exact_match] - exact_low = [w for w in self.low_confidence_unmarked_memories if w.is_exact_match] - fuzzy = [w for w in self.high_confidence_unmarked_memories if not w.is_exact_match] + [ - w for w in self.low_confidence_unmarked_memories if not w.is_exact_match - ] - - # the "may be intentional" caveat is meaningful only for exact matches (where the bare - # text equals a real memory name and may simply be ordinary prose); print it once, - # before the first exact-match section - if exact_high or exact_low: - tb.with_line("Possibly unmarked references — exact matches:") - tb.with_line(" (note: bare matches in prose may be intentional and not actual references)") - for label, warnings in (("high confidence", exact_high), ("low confidence", exact_low)): - if not warnings: - continue - tb.with_line(f" {label} ({len(warnings)}):") - for w in warnings: - ro_tag = " [read-only source]" if w.source_is_read_only else "" - occ = "occurrence" if w.occurrences == 1 else "occurrences" - tb.with_line( - f" - {w.occurrences} {occ} of `{w.suspected_name}` in `{w.source_memory}`{ro_tag}" - f" (suggested: `mem:{w.suspected_name}`)" - ) - tb.with_line("") - - if fuzzy: - tb.with_line(f"Possibly unmarked references — fuzzy near-misses ({len(fuzzy)}):") - for w in fuzzy: - ro_tag = " [read-only source]" if w.source_is_read_only else "" - occ = "occurrence" if w.occurrences == 1 else "occurrences" - tb.with_line( - f" - {w.occurrences} {occ} of `{w.actual_token}` in `{w.source_memory}`{ro_tag}" - f" — near-miss (suggested: `mem:{w.suspected_name}`)" - ) - tb.with_line("") - return tb.build() - - @dataclass - class AutofixReport: - """ - Outcome of :meth:`MemoriesManager.auto_prefix_bare_references`. - - :ivar autofixed: per-(source, target) records of bare references that were rewritten. - When the call was a dry run, these records describe what *would* have been - written; the files themselves are unchanged. - :ivar dry_run: True if the run was a preview that did not write any files. - :ivar skipped_read_only: warnings whose source memory was read-only and therefore not - modified (only populated when ``include_read_only`` is False). - :ivar skipped_flat: warnings skipped because their suspected target had no ``/`` - separator and was not long enough to be high-confidence (only populated when - ``include_flat_names`` is False). - :ivar skipped_global: warnings skipped because their source memory was global and - ``include_global`` was False. - :ivar skipped_fuzzy: warnings whose bare text in the source memory differs from the - suspected target (fuzzy near-misses). These require substring substitution rather - than a prefix addition and are never autofixed; surface them to the user for - manual review instead. - """ - - autofixed: list["MemoriesManager.AutofixedReference"] = field(default_factory=list) - dry_run: bool = False - skipped_read_only: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) - skipped_flat: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) - skipped_global: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) - skipped_fuzzy: list["MemoriesManager.UnmarkedReferenceWarning"] = field(default_factory=list) - - @property - def total_replacements(self) -> int: - return sum(a.n_replacements for a in self.autofixed) - - def format(self) -> str: - """:return: a human-readable rendering suitable for CLI display.""" - tb = TextBuilder() - verb = "Would apply" if self.dry_run else "Applied" - if self.autofixed: - tb.with_line( - f"{verb} {self.total_replacements} replacement(s) across " - f"{len(self.autofixed)} memory/target pair(s)" + (" (dry run; no files were modified)" if self.dry_run else "") + ":" - ) - for a in self.autofixed: - tb.with_line(f" - {a.n_replacements} x `{a.referenced_name}` -> `mem:{a.referenced_name}` in `{a.source_memory}`") - tb.with_line("") - else: - tb.with_line("No replacements" + (" would be" if self.dry_run else "") + " applied.") - tb.with_line("") - - for label, items in ( - ("Skipped (read-only source; pass --include-read-only to override)", self.skipped_read_only), - ("Skipped (flat name; pass --include-flat-names to include)", self.skipped_flat), - ("Skipped (global memory; pass --include-global to include)", self.skipped_global), - ( - "Skipped (fuzzy near-miss; bare text differs from target name — not safe to auto-rewrite, review manually)", - self.skipped_fuzzy, - ), - ): - if not items: - continue - tb.with_line(f"{label}:") - for w in items: - if w.is_exact_match: - tb.with_line(f" - `{w.suspected_name}` in `{w.source_memory}`") - else: - tb.with_line(f" - `{w.actual_token}` in `{w.source_memory}` (suggested: `mem:{w.suspected_name}`)") - tb.with_line("") - return tb.build() - def _list_memories(self, search_dir: Path, base_dir: Path, prefix: str = "") -> MemoriesList: result = self.MemoriesList() if not search_dir.exists(): @@ -816,178 +385,19 @@ class MemoriesManager: f.write(updated_content) return f"Memory {name} edited successfully." - def _is_self_reference(self, source_memory: str, suspected_name: str) -> bool: - """ - :return: True iff ``suspected_name`` would refer to ``source_memory`` itself - (including the case where ``suspected_name`` equals the basename of a topic-path - source memory). - """ - if suspected_name == source_memory: - return True - source_basename = source_memory.rsplit("/", 1)[-1] - return suspected_name == source_basename - def validate_referential_integrity( self, include_unmarked: bool = True, include_fuzzy_matching: bool = True - ) -> "MemoriesManager.ReferentialIntegrityReport": + ) -> ReferentialIntegrityReport: """ - Scans every (non-ignored) memory's content for referential integrity issues. + Validates referential integrity across this manager's memories. - The scan covers both project-local and global memories. Three kinds of finding are - produced, each independently gated: - - * **stale references** — occurrences of ``mem:NAME`` where ``NAME`` does not resolve - to an existing memory. For each, a list of similarly-named existing memories is - proposed as candidate intended targets (see :meth:`compute_name_similarity`), - capped at :attr:`MAX_STALE_REFERENCE_CANDIDATES`. Always reported. - * **exact unmarked-reference warnings** — bare occurrences of an existing memory - name that appear without the ``mem:`` prefix. Split into high-confidence (the - suspected name contains a ``/`` or exceeds :attr:`_HIGH_CONFIDENCE_NAME_LENGTH`) - and low-confidence groups. Candidate names whose basename is in - :attr:`_WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES` are skipped. Gated by - ``include_unmarked``. - * **fuzzy near-miss warnings** — long, distinctive bare tokens in a memory body - that *do not* match an existing name exactly but similarity-match a - high-confidence existing name AND share at least - :attr:`_FUZZY_BARE_TOKEN_JACCARD_FLOOR` of their tokenized name parts. Always - reported as high-confidence; the actual bare text is preserved on - :attr:`UnmarkedReferenceWarning.actual_token`. Gated by ``include_unmarked - and include_fuzzy_matching`` — fuzzy matches are noisy and only meaningful when - unmarked-reference checking is enabled. - - Self-references (a memory's content mentioning its own name or basename) and empty - memories are skipped silently. This method has no side effects. - - :param include_unmarked: if False, skip both the exact unmarked scan and the fuzzy - near-miss scan; only stale references are reported. - :param include_fuzzy_matching: if False, skip the fuzzy near-miss scan even when - unmarked-reference checking is otherwise enabled. Has no effect when - ``include_unmarked`` is False. - :return: a :class:`ReferentialIntegrityReport` summarizing all findings. + Thin wrapper around :meth:`MemoryReferenceAnalyzer.validate_referential_integrity`; + see that method for the full description of behavior and parameters. """ - report = MemoriesManager.ReferentialIntegrityReport() - - # gather all existing memory names (project + global) with their read-only status - memories_list = self.list_memories() - all_names = sorted(set(memories_list.memories) | set(memories_list.read_only_memories)) - read_only_names = set(memories_list.read_only_memories) - existing_set = set(all_names) - # only existing names that are themselves "memory-name-like" qualify as fuzzy candidates, - # otherwise short flat names would match arbitrary long prose words via containment - high_confidence_names = [n for n in all_names if "/" in n or len(n) >= self._HIGH_CONFIDENCE_NAME_LENGTH] - - for source_name in all_names: - # content of this memory should not be analyzed regarding stale references - if source_name == self.MEMORY_MAINTENANCE_NAME: - continue - - content = self.load_memory(source_name) - if not content: - # skip empty memories silently — no references can exist in empty content - continue - source_is_read_only = source_name in read_only_names - - # stale references: every mem:NAME whose NAME is not an existing memory - stale_seen: set[str] = set() - for referenced in self._iter_referenced_names_in_content(content): - if referenced in existing_set or referenced in stale_seen: - continue - stale_seen.add(referenced) - candidates = self._find_stale_reference_candidates(referenced, all_names) - report.stale_references.append( - MemoriesManager.StaleReference( - source_memory=source_name, - referenced_name=referenced, - candidates=candidates, - source_is_read_only=source_is_read_only, - ) - ) - - if not include_unmarked: - continue - - # exact unmarked references: bare occurrences of any existing memory name (except self) - for candidate_name in all_names: - if self._is_self_reference(source_name, candidate_name): - continue - # skip candidate names whose basename is an ambiguous common word - if candidate_name.rsplit("/", 1)[-1] in self._WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: - continue - n = self._find_bare_occurrences(content, candidate_name) - if n == 0: - continue - is_high_conf = "/" in candidate_name or len(candidate_name) >= self._HIGH_CONFIDENCE_NAME_LENGTH - warning = MemoriesManager.UnmarkedReferenceWarning( - source_memory=source_name, - suspected_name=candidate_name, - occurrences=n, - is_high_confidence=is_high_conf, - source_is_read_only=source_is_read_only, - actual_token=candidate_name, - ) - if is_high_conf: - report.high_confidence_unmarked_memories.append(warning) - else: - report.low_confidence_unmarked_memories.append(warning) - - if not include_fuzzy_matching: - continue - - # fuzzy near-misses: long bare tokens that similarity-match a high-confidence existing name - for token, n in self._iter_long_bare_tokens(content): - if token in existing_set: - # exact-match path already handled this token (or self-referenced and was skipped) - continue - # filter bare tokens that coincide with common English words; the same filter is - # applied to candidate names in the exact-match loop, this is the symmetric side - token_norm = self._normalize_for_similarity(token) - if token_norm in self._WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: - continue - token_tokens = self._tokenize_name(token) - best_name: str | None = None - best_score = 0.0 - for existing in high_confidence_names: - if self._is_self_reference(source_name, existing): - continue - if existing.rsplit("/", 1)[-1] in self._WORDS_TO_IGNORE_AS_MEMORY_NAME_CANDIDATES: - continue - # require substring containment between bare token and existing name: fuzzy - # matches without a containment relationship are rarely useful as rewrite - # suggestions (auto_prefix_bare_references can't fix them anyway) and dominate - # the false-positive rate, e.g. "grid-layout" vs "layout-grid-subtleties". - existing_norm = self._normalize_for_similarity(existing) - if token_norm not in existing_norm and existing_norm not in token_norm: - continue - score = self.compute_name_similarity(token, existing) - if score < self.NAME_SIMILARITY_THRESHOLD: - continue - # safeguard against substring-only matches: require meaningful token overlap so - # generic prose words don't get flagged just because they happen to be substrings - # of an existing memory name - existing_tokens = self._tokenize_name(existing) - union = token_tokens | existing_tokens - if not union: - continue - jaccard = len(token_tokens & existing_tokens) / len(union) - if jaccard < self._FUZZY_BARE_TOKEN_JACCARD_FLOOR: - continue - if score > best_score: - best_score = score - best_name = existing - if best_name is None: - continue - report.high_confidence_unmarked_memories.append( - MemoriesManager.UnmarkedReferenceWarning( - source_memory=source_name, - suspected_name=best_name, - occurrences=n, - is_high_confidence=True, - source_is_read_only=source_is_read_only, - actual_token=token, - ) - ) - - return report + return MemoryReferenceAnalyzer(self).validate_referential_integrity( + include_unmarked=include_unmarked, + include_fuzzy_matching=include_fuzzy_matching, + ) def auto_prefix_bare_references( self, @@ -995,94 +405,19 @@ class MemoriesManager: include_read_only: bool = False, include_global: bool = False, dry_run: bool = False, - ) -> "MemoriesManager.AutofixReport": + ) -> AutofixReport: """ - Rewrites *exact* bare occurrences of existing memory names by adding the ``mem:`` prefix. + Rewrites bare occurrences of existing memory names to include the ``mem:`` prefix. - .. warning:: - **This is a heuristic, file-mutating operation** (unless ``dry_run`` is True). A - bare word that happens to coincide with a memory name will be rewritten as a - reference, even if it was intended as ordinary prose. Pass ``dry_run=True`` to - preview the rewrites before applying them. - - Scope is intentionally narrower than what :meth:`validate_referential_integrity` - reports: - - * Only **exact** bare occurrences are rewritten — i.e. the bare text in the source - body must equal an existing memory name verbatim. Fuzzy near-miss findings - (where the actual token differs from the suspected target) require substring - substitution rather than a prefix addition and are routed into - :attr:`AutofixReport.skipped_fuzzy` for manual review. - * By default the rewrite is restricted to *high-confidence* findings only — those - whose suspected target name contains a ``/`` separator or exceeds the - configured length threshold — and skips global memories and read-only memories. - The defaults intentionally err toward false negatives over false positives. - - :param include_flat_names: if True, also rewrite low-confidence findings (flat, - short memory names). Increases recall but markedly raises false-positive risk. - :param include_read_only: if True, also rewrite occurrences inside read-only - memories. Use with care, as read-only memories are typically considered - authoritative. - :param include_global: if True, also rewrite occurrences inside global memories. - Modifying a global memory affects every project that consumes it. - :param dry_run: if True, the report describes the rewrites that *would* be applied - but no files are modified. - :return: an :class:`AutofixReport` describing every replacement (made or planned) - and every warning that was deliberately skipped. + Thin wrapper around :meth:`MemoryReferenceAnalyzer.auto_prefix_bare_references`; + see that method for the full description of behavior and parameters. """ - report = MemoriesManager.AutofixReport(dry_run=dry_run) - - # build the integrity report first so autofix decisions are made on the same data - validation = self.validate_referential_integrity() - - # combine the warning groups according to the include_flat_names policy - warnings: list[MemoriesManager.UnmarkedReferenceWarning] = list(validation.high_confidence_unmarked_memories) - if include_flat_names: - warnings.extend(validation.low_confidence_unmarked_memories) - else: - report.skipped_flat.extend(validation.low_confidence_unmarked_memories) - - # apply replacements per source memory, grouping warnings to avoid re-reading content - warnings_by_source: dict[str, list[MemoriesManager.UnmarkedReferenceWarning]] = {} - for w in warnings: - # fuzzy near-misses are never auto-rewritten — the bare text differs from the target - if not w.is_exact_match: - report.skipped_fuzzy.append(w) - continue - # filter remaining warnings according to scope flags - if w.source_is_read_only and not include_read_only: - report.skipped_read_only.append(w) - continue - if self._is_global(w.source_memory) and not include_global: - report.skipped_global.append(w) - continue - warnings_by_source.setdefault(w.source_memory, []).append(w) - - for source_memory, source_warnings in warnings_by_source.items(): - content = self.load_memory(source_memory) - total_n = 0 - per_target_records: list[MemoriesManager.AutofixedReference] = [] - - # apply replacements sequentially; each pass uses the (potentially updated) content - for w in source_warnings: - content, n = self._add_bare_occurrences_prefix(content, w.suspected_name) - if n > 0: - per_target_records.append( - MemoriesManager.AutofixedReference( - source_memory=source_memory, - referenced_name=w.suspected_name, - n_replacements=n, - ) - ) - total_n += n - - if total_n > 0: - if not dry_run: - # use is_tool_context=False so we don't trip read-only protection when --include-read-only is set - self.save_memory(source_memory, content, is_tool_context=False) - report.autofixed.extend(per_target_records) - - return report + return MemoryReferenceAnalyzer(self).auto_prefix_bare_references( + include_flat_names=include_flat_names, + include_read_only=include_read_only, + include_global=include_global, + dry_run=dry_run, + ) class Project(ToStringMixin): diff --git a/test/serena/test_memories_manager.py b/test/serena/test_memories_manager.py index 3974d159..2211c991 100644 --- a/test/serena/test_memories_manager.py +++ b/test/serena/test_memories_manager.py @@ -8,6 +8,13 @@ import os import pytest +from serena.memory_reference_analysis import ( + HIGH_CONFIDENCE_NAME_LENGTH, + MAX_STALE_REFERENCE_CANDIDATES, + NAME_SIMILARITY_THRESHOLD, + compute_name_similarity, + find_stale_reference_candidates, +) from serena.project import MemoriesManager @@ -170,7 +177,7 @@ class TestNameSimilarity: ], ) def test_high_similarity_cases(self, manager: MemoriesManager, a: str, b: str, expected: float) -> None: - assert manager.compute_name_similarity(a, b) == pytest.approx(expected) + assert compute_name_similarity(a, b) == pytest.approx(expected) @pytest.mark.parametrize( ("a", "b"), @@ -189,7 +196,7 @@ class TestNameSimilarity: ], ) def test_below_threshold_cases(self, manager: MemoriesManager, a: str, b: str) -> None: - assert manager.compute_name_similarity(a, b) < MemoriesManager.NAME_SIMILARITY_THRESHOLD + assert compute_name_similarity(a, b) < NAME_SIMILARITY_THRESHOLD @pytest.mark.parametrize( ("a", "b"), @@ -204,13 +211,11 @@ class TestNameSimilarity: ], ) def test_above_threshold_cases(self, manager: MemoriesManager, a: str, b: str) -> None: - assert manager.compute_name_similarity(a, b) >= MemoriesManager.NAME_SIMILARITY_THRESHOLD + assert compute_name_similarity(a, b) >= NAME_SIMILARITY_THRESHOLD def test_symmetry(self, manager: MemoriesManager) -> None: # similarity is order-independent - assert manager.compute_name_similarity("auth/login", "security/login") == manager.compute_name_similarity( - "security/login", "auth/login" - ) + assert compute_name_similarity("auth/login", "security/login") == compute_name_similarity("security/login", "auth/login") def test_cross_topic_suffix_coincidence_is_rejected(self, manager: MemoriesManager) -> None: """ @@ -227,15 +232,15 @@ class TestNameSimilarity: "render-wasm/ffi-rendering-subtleties", "common/grid-layout-subtleties", ): - assert manager.compute_name_similarity(target, other) == 0.0, f"cross-topic suffix coincidence with {other} must be rejected" + assert compute_name_similarity(target, other) == 0.0, f"cross-topic suffix coincidence with {other} must be rejected" # contrast: *same* prefix is treated leniently — the gate must not apply, so the # score is non-zero even if it still falls under the similarity threshold - same_prefix_score = manager.compute_name_similarity(target, "frontend/routing-app-shell-subtleties") + same_prefix_score = compute_name_similarity(target, "frontend/routing-app-shell-subtleties") assert same_prefix_score > 0.0 def test_find_candidates_ranks_by_descending_similarity(self, manager: MemoriesManager) -> None: existing = ["auth/login", "security/login", "database", "authentication"] - candidates = manager._find_stale_reference_candidates("login", existing) + candidates = find_stale_reference_candidates("login", existing) # both auth/login and security/login share basename `login` -> tied at 1.0; expect them first, alphabetical tiebreak assert candidates[:2] == ["auth/login", "security/login"] # `database` is unrelated and must not appear @@ -243,13 +248,13 @@ class TestNameSimilarity: def test_find_candidates_returns_empty_below_threshold(self, manager: MemoriesManager) -> None: existing = ["completely_unrelated", "another_thing"] - assert manager._find_stale_reference_candidates("foo", existing) == [] + assert find_stale_reference_candidates("foo", existing) == [] def test_find_candidates_custom_threshold(self, manager: MemoriesManager) -> None: # raising the threshold filters out borderline matches existing = ["authentication"] - loose = manager._find_stale_reference_candidates("auth", existing, threshold=0.2) - strict = manager._find_stale_reference_candidates("auth", existing, threshold=0.95) + loose = find_stale_reference_candidates("auth", existing, threshold=0.2) + strict = find_stale_reference_candidates("auth", existing, threshold=0.95) assert loose == ["authentication"] assert strict == [] @@ -313,7 +318,7 @@ class TestValidateReferentialIntegrity: def test_long_flat_name_is_high_confidence(self, fs_manager: MemoriesManager) -> None: # a flat name >= _HIGH_CONFIDENCE_NAME_LENGTH chars is unlikely to be coincidental prose long_name = "authentication_flow" - assert len(long_name) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + assert len(long_name) >= HIGH_CONFIDENCE_NAME_LENGTH _write(fs_manager, long_name, "# notes") _write(fs_manager, "docs", f"the {long_name} is documented here") report = fs_manager.validate_referential_integrity() @@ -361,7 +366,7 @@ class TestValidateReferentialIntegrity: """ existing = "adding_new_language_support_guide" bare = "adding_new_language_support" # prefix of `existing`; clearly the same concept - assert len(bare) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + assert len(bare) >= HIGH_CONFIDENCE_NAME_LENGTH _write(fs_manager, existing, "# guide content") _write(fs_manager, "docs", f"See {bare} for details") @@ -384,7 +389,7 @@ class TestValidateReferentialIntegrity: # existing has 3 name-tokens; bare has 2 of them -> jaccard = 2/3 = 0.667 existing = "data-model-extra" bare = "data-model" - assert len(bare) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + assert len(bare) >= HIGH_CONFIDENCE_NAME_LENGTH _write(fs_manager, existing, "# extra notes") _write(fs_manager, "docs", f"see {bare} for naming") report = fs_manager.validate_referential_integrity() @@ -402,7 +407,7 @@ class TestValidateReferentialIntegrity: """ existing = "data-model-extra-stuff" # 4 tokens bare = "data-model" # 2 tokens; jaccard = 2/4 = 0.5 - assert len(bare) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + assert len(bare) >= HIGH_CONFIDENCE_NAME_LENGTH _write(fs_manager, existing, "# notes") _write(fs_manager, "docs", f"see {bare} for naming") report = fs_manager.validate_referential_integrity() @@ -418,7 +423,7 @@ class TestValidateReferentialIntegrity: """ existing = "frontend/text-editor-workflow" bare = "frontend/text-editor-features" # shares "frontend", "text", "editor"; not a substring - assert len(bare) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + assert len(bare) >= HIGH_CONFIDENCE_NAME_LENGTH _write(fs_manager, existing, "# workflow notes") _write(fs_manager, "docs", f"see {bare} for layout") report = fs_manager.validate_referential_integrity() @@ -433,7 +438,7 @@ class TestValidateReferentialIntegrity: """ existing = "playwright-gestures" bare = "playwright-gestures-guide" # existing ⊂ bare - assert len(bare) >= MemoriesManager._HIGH_CONFIDENCE_NAME_LENGTH + assert len(bare) >= HIGH_CONFIDENCE_NAME_LENGTH _write(fs_manager, existing, "# notes") _write(fs_manager, "docs", f"see {bare} for details") report = fs_manager.validate_referential_integrity() @@ -484,7 +489,7 @@ class TestValidateReferentialIntegrity: :attr:`MAX_STALE_REFERENCE_CANDIDATES`. Beyond a small number the list stops aiding disambiguation and becomes noise. """ - cap = MemoriesManager.MAX_STALE_REFERENCE_CANDIDATES + cap = MAX_STALE_REFERENCE_CANDIDATES # five memories all sharing basename `login` -> all tie at compute_name_similarity == 1.0 candidate_names = ["admin/login", "api/login", "auth/login", "security/login", "users/login"] for name in candidate_names: From eaa6a744c1c90ce27afdc08c101753365a26b351 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Thu, 14 May 2026 00:19:16 +0200 Subject: [PATCH 12/20] Docstring --- src/serena/tools/memory_tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index b7709a4a..a1349c79 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -81,6 +81,7 @@ class RenameMemoryTool(Tool, ToolMarkerCanEdit): Rename or move a memory, use "/" in the name to organize into topics. The "global" topic should only be used if explicitly instructed. References to other memories that are marked with the `mem:` prefix will be updated accordingly. + References in read-only memories are not affected. """ renaming_message, n_references_updated = self.memories_manager.rename_memory_and_propagate_references( old_name, new_name, is_tool_context=True From b8a5eef5bc7c1e7f84cbd530524cc235f6157a44 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 14 May 2026 23:12:39 +0200 Subject: [PATCH 13/20] Change regex multi-line handling * Combine DOTALL and MULTILINE into a single flag for "multi-line mode", replacing the dotall parameter with multiline/regex_multiline * Apply the parameter in SearchForPatternTool * Remove the dotall flag from editing tools (ReplaceContentTool, EditMemoryTool) because edits should always use multi-line mode, but keep the multiline parameter as an option in ContentReplacer --- CHANGELOG.md | 9 ++----- src/serena/cli.py | 10 +------- src/serena/project.py | 12 ++++----- src/serena/tools/file_tools.py | 8 +++--- src/serena/tools/memory_tools.py | 9 ++----- src/serena/util/text_utils.py | 44 +++++++++++++++----------------- 6 files changed, 35 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9f67003..47046db5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,13 +24,8 @@ Status of the `main` branch. Changes prior to the next official version change w See the [CLI subcommands](https://oraios.github.io/serena/02-usage/045_memories.html#cli-subcommands). * Tools: - - `edit_memory`: New `dotall` parameter (default `False`) controlling whether ``.`` matches - newlines in regex mode. **Behavior change**: previously DOTALL was forced on; the new default - restricts ``.`` to a single line, which is safer for typical line-bounded memory edits and - avoids accidentally consuming large spans of content. Pass `dotall=True` for multi-line edits. - - `search_for_pattern`: New `dotall` parameter (default `True`, preserving existing behavior) - exposing the DOTALL flag, and the docstring was tightened. - - Delete `check_onboarding_performed` tool (instead extend project activation message), + - `search_for_pattern`: Add parameter `multiline` + - Delete `check_onboarding_performed` tool (instead extend project activation message) # v1.3.0 (2026-05-11) diff --git a/src/serena/cli.py b/src/serena/cli.py index 27022662..5f87a36b 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -1267,7 +1267,7 @@ class MemoryCommands(AutoRegisteringGroup): type=click.Choice(["literal", "regex"]), default="literal", show_default=True, - help="Treat --needle as literal text or as a Python regex (MULTILINE flag is always on; DOTALL via --dotall).", + help="Treat --needle as literal text or as a Python regex (MULTILINE and DOTALL flags enabled).", ) @click.option( "--allow-multiple-occurrences", @@ -1275,12 +1275,6 @@ class MemoryCommands(AutoRegisteringGroup): default=False, help="Permit and apply multiple matches; without this, multiple matches raise an error.", ) - @click.option( - "--dotall", - is_flag=True, - default=False, - help="Regex mode only: make `.` match newlines as well, enabling multi-line patterns. Use with care.", - ) def edit( memory_name: str, project: str, @@ -1288,7 +1282,6 @@ class MemoryCommands(AutoRegisteringGroup): repl: str, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, - dotall: bool, ) -> None: manager = MemoryCommands._load_memories_manager(project) click.echo( @@ -1299,7 +1292,6 @@ class MemoryCommands(AutoRegisteringGroup): mode, allow_multiple_occurrences, is_tool_context=False, - dotall=dotall, ) ) diff --git a/src/serena/project.py b/src/serena/project.py index 5ba84ca6..8fc59817 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -358,7 +358,7 @@ class MemoriesManager: mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, is_tool_context: bool, - dotall: bool = True, + regex_multiline: bool = True, ) -> str: """ Edit a memory by replacing content matching a pattern. @@ -369,7 +369,7 @@ class MemoriesManager: :param mode: "literal" or "regex" :param allow_multiple_occurrences: :param is_tool_context: whether the call originates from a tool invocation (affects write-access checks) - :param dotall: whether to compile the regex with the DOTALL flag (``.`` matches newlines). Only relevant in regex mode. + :param regex_multiline: whether to apply multi-line regex matching, enabling the flags re.DOTALL and re.MULTILINE """ name = self._prepare_name(name) self._check_not_ignored(name) @@ -379,7 +379,7 @@ class MemoriesManager: raise FileNotFoundError(f"Memory {name} not found.") with open(memory_file_path, encoding=self._encoding) as f: original_content = f.read() - replacer = ContentReplacer(mode=mode, allow_multiple_occurrences=allow_multiple_occurrences, dotall=dotall) + replacer = ContentReplacer(mode=mode, allow_multiple_occurrences=allow_multiple_occurrences, regex_multiline=regex_multiline) updated_content = replacer.replace(original_content, needle, repl) with open(memory_file_path, "w", encoding=self._encoding) as f: f.write(updated_content) @@ -738,7 +738,7 @@ class Project(ToStringMixin): context_lines_after: int = 0, paths_include_glob: str | None = None, paths_exclude_glob: str | None = None, - dotall: bool = True, + multiline: bool = True, ) -> list[MatchedConsecutiveLines]: """ Search for a pattern across all (non-ignored) source files @@ -749,7 +749,7 @@ class Project(ToStringMixin): :param context_lines_after: Number of lines of context to include after each match :param paths_include_glob: Glob pattern to filter which files to include in the search :param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob. - :param dotall: Whether to compile the regex with the DOTALL flag (``.`` matches newlines). + :param multiline: Whether to compile the regex with the DOTALL flag (``.`` matches newlines). :return: List of matched consecutive lines with context """ relative_file_paths = self.gather_source_files(relative_path=relative_path) @@ -762,7 +762,7 @@ class Project(ToStringMixin): context_lines_after=context_lines_after, paths_include_glob=paths_include_glob, paths_exclude_glob=paths_exclude_glob, - dotall=dotall, + multiline=multiline, ) def retrieve_content_around_line( diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 83aa837e..830e0bfe 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -332,7 +332,7 @@ class SearchForPatternTool(Tool): paths_exclude_glob: str = "", relative_path: str = "", restrict_search_to_code_files: bool = False, - dotall: bool = True, + multiline: bool = True, max_answer_chars: int = -1, ) -> str: """ @@ -347,7 +347,7 @@ class SearchForPatternTool(Tool): :param relative_path: restricts the search to this file or subdirectory of the project root. Must exist. :param restrict_search_to_code_files: whether to search only files containing analyzable code symbols (useful when looking for class/method definitions); otherwise also search non-code files. - :param dotall: whether to compile the regex with the DOTALL flag (``.`` matches newlines, enabling multi-line matches). + :param multiline: whether to apply multi-line matching, enabling the flags re.DOTALL and re.MULTILINE :param max_answer_chars: if the output exceeds this many characters, a progressively shortened summary is returned instead. ``-1`` uses the configured default. :return: A mapping from file paths to matched consecutive lines (0-based line numbers). @@ -368,7 +368,7 @@ class SearchForPatternTool(Tool): context_lines_after=context_lines_after, paths_include_glob=paths_include_glob.strip(), paths_exclude_glob=paths_exclude_glob.strip(), - dotall=dotall, + multiline=multiline, ) else: if os.path.isfile(abs_path): @@ -392,7 +392,7 @@ class SearchForPatternTool(Tool): root_path=self.get_project_root(), paths_include_glob=paths_include_glob, paths_exclude_glob=paths_exclude_glob, - dotall=dotall, + multiline=multiline, ) # group matches by file diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index a1349c79..f7be84d0 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -103,7 +103,6 @@ class EditMemoryTool(Tool, ToolMarkerCanEdit): repl: str, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool = False, - dotall: bool = False, ) -> str: r""" Replace content matching a regular expression in a memory. @@ -112,16 +111,12 @@ class EditMemoryTool(Tool, ToolMarkerCanEdit): :param needle: the string or regex pattern to search for. In regex mode, be careful to not replace too much! If `mode` is "literal", this string will be matched exactly. If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module, - with the MULTILINE flag enabled and DOTALL controlled by the `dotall` parameter). + with the MULTILINE and DOTALL flags enabled). :param repl: the replacement string (verbatim). :param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted. :param allow_multiple_occurrences: whether to allow matching and replacing multiple occurrences. If false and multiple occurrences are found, an error will be returned. - :param dotall: whether to compile the regex with the DOTALL flag, so that ``.`` matches newlines. - Only relevant in regex mode. Defaults to False, restricting ``.`` to a single line, which is the safer - choice for typical line-bounded memory edits and avoids accidentally consuming entire blocks of content. - Set to True deliberately when you intend a multi-line replacement (alternatively, use ``[\s\S]*?``). """ return self.memories_manager.edit_memory( - memory_name, needle, repl, mode, allow_multiple_occurrences, is_tool_context=True, dotall=dotall + memory_name, needle, repl, mode, allow_multiple_occurrences, is_tool_context=True, regex_multiline=True ) diff --git a/src/serena/util/text_utils.py b/src/serena/util/text_utils.py index ddadf549..16bee9d9 100644 --- a/src/serena/util/text_utils.py +++ b/src/serena/util/text_utils.py @@ -145,7 +145,7 @@ def search_text( context_lines_before: int = 0, context_lines_after: int = 0, is_glob: bool = False, - dotall: bool = True, + multiline: bool = True, ) -> list[MatchedConsecutiveLines]: """ Search for a pattern in text content. Supports both regex and glob-like patterns. @@ -160,13 +160,9 @@ def search_text( :param context_lines_after: Number of context lines to include after matches :param is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py") and will be converted to regex internally - :param dotall: Whether to compile the regex with the DOTALL flag (``.`` matches newlines). - Only takes effect when `allow_multiline_match` is True (the line-by-line branch never spans newlines). - + :param multiline: whether to apply multi-line matching, enabling the flags re.DOTALL and re.MULTILINE :return: List of `TextSearchMatch` objects - :raises: ValueError if the pattern is not valid - """ if source_file_path and content is None: with open(source_file_path) as f: @@ -184,7 +180,8 @@ def search_text( pattern = glob_to_regex(pattern) if allow_multiline_match: # For multiline matches, optionally use DOTALL so '.' matches newlines - compiled_pattern = re.compile(pattern, re.DOTALL if dotall else 0) + flags = (re.MULTILINE | re.DOTALL) if multiline else 0 + compiled_pattern = re.compile(pattern, flags) # Search across the entire content as a single string for match in compiled_pattern.finditer(content): start_pos = match.start() @@ -330,22 +327,22 @@ def search_files( context_lines_after: int = 0, paths_include_glob: str | None = None, paths_exclude_glob: str | None = None, - dotall: bool = True, + multiline: bool = True, ) -> list[MatchedConsecutiveLines]: """ Search for a pattern in a list of files. - :param relative_file_paths: List of relative file paths in which to search - :param pattern: Pattern to search for - :param root_path: Root path to resolve relative paths against (by default, current working directory). - :param file_reader: Function to read a file, by default will just use os.open. + :param relative_file_paths: list of relative file paths in which to search + :param pattern: pattern to search for + :param root_path: root path to resolve relative paths against (by default, current working directory). + :param file_reader: function to read a file, by default will just use os.open. All files that can't be read by it will be skipped. - :param context_lines_before: Number of context lines to include before matches - :param context_lines_after: Number of context lines to include after matches - :param paths_include_glob: Optional glob pattern to include files from the list - :param paths_exclude_glob: Optional glob pattern to exclude files from the list - :param dotall: Whether to compile the regex with the DOTALL flag (``.`` matches newlines). - :return: List of MatchedConsecutiveLines objects + :param context_lines_before: number of context lines to include before matches + :param context_lines_after: number of context lines to include after matches + :param paths_include_glob: optional glob pattern to include files from the list + :param paths_exclude_glob: optional glob pattern to exclude files from the list + :param multiline: whether to apply multi-line matching, enabling the flags re.DOTALL and re.MULTILINE (default: True) + :return: list of MatchedConsecutiveLines objects """ # Pre-filter paths (done sequentially to avoid overhead) # Use proper glob matching instead of gitignore patterns @@ -380,7 +377,7 @@ def search_files( allow_multiline_match=True, context_lines_before=context_lines_before, context_lines_after=context_lines_after, - dotall=dotall, + multiline=multiline, ) if len(search_results) > 0: log.debug(f"Found {len(search_results)} matches in {path}") @@ -436,19 +433,18 @@ class ContentReplacer: provides dual modes for maximum flexibility. """ - def __init__(self, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, dotall: bool = True): + def __init__(self, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, regex_multiline: bool = True): """ :param mode: the mode indicating whether to the needle in replacements corresponds to a regular expression (mode "regex") or to a literal string (mode "literal") :param allow_multiple_occurrences: whether it is allowed that the search expression matches multiple occurrences. If False, an error will be raised if more than one match is found. - :param dotall: whether to compile the regex with the DOTALL flag, which makes ``.`` match newlines. - Only relevant in regex mode. The MULTILINE flag is always enabled (so ``^`` and ``$`` match line boundaries). + :param regex_multiline: whether to apply multi-line regex matching, enabling the flags re.DOTALL and re.MULTILINE """ self.mode = mode self.allow_multiple_occurrences = allow_multiple_occurrences - self.dotall = dotall + self.regex_multiline = regex_multiline @staticmethod def _create_replacement_function(regex_pattern: str, repl_template: str, regex_flags: int) -> Callable[[re.Match], str]: @@ -514,7 +510,7 @@ class ContentReplacer: else: raise ValueError(f"Invalid mode: '{self.mode}', expected 'literal' or 'regex'.") - regex_flags = re.MULTILINE | (re.DOTALL if self.dotall else 0) + regex_flags = (re.MULTILINE | re.DOTALL) if self.regex_multiline else 0 # create replacement function with validation and backreference handling repl_fn = self._create_replacement_function(regex, repl, regex_flags=regex_flags) From 2b52dd7636b9390b3ed97c8b952fea658bf13c93 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 14 May 2026 23:48:33 +0200 Subject: [PATCH 14/20] Improve memory management documentation and memory terminology * Improve memory_maintenance memory * Drop "routing" terminology, use "references" instead --- .serena/memories/memory_maintenance.md | 47 ++++++++----------- CHANGELOG.md | 12 ++--- docs/02-usage/040_workflow.md | 2 +- docs/02-usage/045_memories.md | 16 +++---- .../prompt_templates/simple_tool_outputs.yml | 4 +- src/serena/resources/memory_maintenance.md | 47 ++++++++----------- 6 files changed, 55 insertions(+), 73 deletions(-) diff --git a/.serena/memories/memory_maintenance.md b/.serena/memories/memory_maintenance.md index cfb4c05c..6f84514d 100644 --- a/.serena/memories/memory_maintenance.md +++ b/.serena/memories/memory_maintenance.md @@ -2,41 +2,32 @@ ## Discovery Model -- Agents start with the full memory name list and should read `mem:core` first as the routing entry point. -- All other memory discovery comes from memory names plus routing already read. -- Routing is useful only before the target memory is selected. Once a memory is open, target-selection text inside it can no longer help decide to read it. -- Therefore, never put self-referential selection cues in a memory body: no "Start here", "Use when", "Use before", "Use after", or "read this when..." openers. - Such self-referential cues are an error mode: they are unreachable for selection, duplicate routing that belongs upstream, and drift independently from the actual routing graph. -- Add routing only when names are insufficient, ambiguous, or a non-obvious dependency/order matters. Do not add tautological routing such as "frontend debugging -> frontend/debugging". -- Routing can live outside core memories when needed, but core memories own most module-level routing because they are read before focused memories in that subfolder. -- Top-level/startup memory is `mem:core`; top-level memory in each memory subfolder is `/core` (for example `mem:frontend/core`). +- Core principle: progressive discovery through references, building a graph of memories. +- Initially, agents are provided with the list of all memories (names only). +- Agents should read `mem:core` as the top-level entry point (graph root). + This memory should contain references to other memories covering major project domains. + The referenced memories shall, in turn, shall contain references to even more specific memories, and so on. + The depth of the graph shall depend on the project complexity. +- Use topics/folders to group related memories in order to make the content structure explicit. + Folders can mirror project structure (e.g. modules like frontend/backend) or topics like debugging, architecture, etc. - Memory references must use a mem: prefix inside backticks, e.g. `mem:frontend/core`. -- Do not add an index that simply re-lists the focused-memory names, the memory name list is already visible. - Only add routing pointers when a target memory's name is not self-explanatory or when a some aspect should be routed to multiple memories. + The surrounding text should clearly indicate when to read the memory/which content to expect. + The text should provide more precise guidance than the memory name alone, + i.e. avoid a reference like "frontend debugging: `mem:frontend/debugging` and instead make clear which aspects of frontend debugging are covered. +- Memories themselves should not contain information about when to read them; this is the responsibility of the referring memory. ## Style -Dense agent notes, not prose docs. Prefer invariants, paths, commands, terse bullets. Avoid obvious context, rationale, and examples unless they prevent likely mistakes. Keep guidance durable and generalizable, not task-local. +Dense agent notes, not prose docs. Prefer invariants, terse bullets. +Avoid obvious context, rationale, and examples unless they prevent likely mistakes. +Keep guidance durable and generalizable, not task-local. ## Add/update threshold -Add or update memory only for stable, non-obvious project conventions that avoid complex rediscovery. - -Do not add: quick-read/rg facts; generic language/framework knowledge; one-off task notes; volatile line-level details; behavior likely to change soon. - -Operational constants (commands, paths, version pins): phrase as current behavior; before changing the value, update it in the owning memory and run `serena memories check` to catch references that need updating. - -## Graph Ownership - -- Project-level routing: `mem:core`. -- Module-level routing/source maps: module core memories. -- Focused memories: topic details only; assume relevant routing was read before selection. +Add or update memories only with stable, non-obvious project conventions that avoid complex rediscovery in the future. +Do not add: quick-read facts; generic language/framework knowledge; one-off task notes; volatile line-level details; behavior likely to change soon. ## Maintenance Actions -- Renamed/removed/split/merged/superseded memory: update all references. Done automatically if handled via Serena's memory rename tool. -- Stale paths/namespaces/commands/behavior: update or remove in owning memory. If working with Serena, call `serena memories check` for a report. -- Split catch-all memory when sections are commonly needed independently or agents must read unrelated domains to reach needed topic. -- Retire/delete memory when owning module/subsystem is removed or replaced; remove all references. -- Added/renamed/split/moved/deleted memory: update affected routing only where names are insufficient. -- Before finishing: list memories for old names and key new names to catch stale/missing references. \ No newline at end of file +- Renaming memories: References are updated automatically if handled via Serena's memory rename tool. +- Checking for stale memories (e.g. after deletion): Call `serena memories check` for a report. \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 47046db5..e22f5712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,13 @@ Status of the `main` branch. Changes prior to the next official version change w - Add Language: replace the native `