From 3cde03a8fb85b47921d3de135b2c9b3e03dc29c9 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 16 Oct 2025 22:09:39 +0200 Subject: [PATCH] Introduce constant for Serena file encoding * SERENA_FILE_ENCODING for Serena-specific files (config files, memories) * DEFAULT_SOURCE_FILE_ENCODING for source files (renamed from DEFAULT_ENCODING) --- src/serena/agent.py | 3 ++- src/serena/code_editor.py | 4 ++-- src/serena/config/context_mode.py | 5 +++-- src/serena/config/serena_config.py | 13 +++++++------ src/serena/constants.py | 6 +++++- src/serena/text_utils.py | 4 ++-- src/serena/tools/memory_tools.py | 1 + src/serena/util/general.py | 6 ++++-- 8 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 1c95b5fb..9358db1d 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -22,6 +22,7 @@ from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, ToolSet, get_serena_managed_in_project_dir +from serena.constants import SERENA_FILE_ENCODING from serena.dashboard import SerenaDashboardAPI from serena.project import Project from serena.prompt_factory import SerenaPromptFactory @@ -47,7 +48,7 @@ class MemoriesManager: def __init__(self, project_root: str): self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" self._memory_dir.mkdir(parents=True, exist_ok=True) - self._encoding = "utf-8" + self._encoding = SERENA_FILE_ENCODING def _get_memory_file_path(self, name: str) -> Path: # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not. diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index 5dc3fe65..3d99d2bb 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -12,7 +12,7 @@ from solidlsp.ls import LSPFileBuffer from solidlsp.ls_types import extract_text_edits from solidlsp.ls_utils import PathUtils, TextUtils -from .constants import DEFAULT_ENCODING +from .constants import DEFAULT_SOURCE_FILE_ENCODING from .project import Project from .tools.jetbrains_plugin_client import JetBrainsPluginClient @@ -30,7 +30,7 @@ class CodeEditor(Generic[TSymbol], ABC): self.agent = agent # set encoding based on active project, if available - encoding = DEFAULT_ENCODING + encoding = DEFAULT_SOURCE_FILE_ENCODING if agent is not None: project = agent.get_active_project() if project is not None: diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index 44cd42fb..ed2eb11c 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -17,6 +17,7 @@ from serena.constants import ( DEFAULT_CONTEXT, DEFAULT_MODES, INTERNAL_MODE_YAMLS_DIR, + SERENA_FILE_ENCODING, SERENAS_OWN_CONTEXT_YAMLS_DIR, SERENAS_OWN_MODE_YAMLS_DIR, USER_CONTEXT_YAMLS_DIR, @@ -56,7 +57,7 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a mode from a YAML file.""" - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) return cls(name=name, **data) @@ -139,7 +140,7 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a context from a YAML file.""" - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) # Ensure backwards compatibility for tool_description_overrides diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 00bbc901..5b9d4462 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -19,10 +19,11 @@ from sensai.util.logging import LogTime, datetime_tag from sensai.util.string import ToStringMixin from serena.constants import ( - DEFAULT_ENCODING, + DEFAULT_SOURCE_FILE_ENCODING, PROJECT_TEMPLATE_FILE, REPO_ROOT, SERENA_CONFIG_TEMPLATE_FILE, + SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME, ) @@ -166,7 +167,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): read_only: bool = False ignore_all_files_in_gitignore: bool = True initial_prompt: str = "" - encoding: str = DEFAULT_ENCODING + encoding: str = DEFAULT_SOURCE_FILE_ENCODING SERENA_DEFAULT_PROJECT_FILE = "project.yml" @@ -244,7 +245,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): read_only=data.get("read_only", False), ignore_all_files_in_gitignore=data.get("ignore_all_files_in_gitignore", True), initial_prompt=data.get("initial_prompt", ""), - encoding=data.get("encoding", DEFAULT_ENCODING), + encoding=data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING), ) @classmethod @@ -259,7 +260,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): return cls.autogenerate(project_root) else: raise FileNotFoundError(f"Project configuration file not found: {yaml_path}") - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: yaml_data = yaml.safe_load(f) if "project_name" not in yaml_data: yaml_data["project_name"] = project_root.name @@ -483,11 +484,11 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): """ log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.") try: - with open(path, encoding="utf-8") as f: + with open(path, encoding=SERENA_FILE_ENCODING) as f: project_config_data = yaml.safe_load(f) if "project_name" not in project_config_data: project_name = path.stem - with open(path, "a", encoding="utf-8") as f: + with open(path, "a", encoding=SERENA_FILE_ENCODING) as f: f.write(f"\nproject_name: {project_name}") project_root = project_config_data["project_root"] shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml())) diff --git a/src/serena/constants.py b/src/serena/constants.py index 1e5d98a1..e0ed29cf 100644 --- a/src/serena/constants.py +++ b/src/serena/constants.py @@ -25,10 +25,14 @@ INTERNAL_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "inter SERENA_DASHBOARD_DIR = str(_serena_pkg_path / "resources" / "dashboard") SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons") -DEFAULT_ENCODING = "utf-8" +DEFAULT_SOURCE_FILE_ENCODING = "utf-8" +"""The default encoding assumed for project source files.""" DEFAULT_CONTEXT = "desktop-app" DEFAULT_MODES = ("interactive", "editing") +SERENA_FILE_ENCODING = "utf-8" +"""The encoding used for Serena's own files, such as configuration files and memories.""" + PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.yml") SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml") diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 3eea8ba6..d87e9ad6 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -9,7 +9,7 @@ from typing import Any, Self from joblib import Parallel, delayed -from serena.constants import DEFAULT_ENCODING +from serena.constants import DEFAULT_SOURCE_FILE_ENCODING log = logging.getLogger(__name__) @@ -239,7 +239,7 @@ def search_text( def default_file_reader(file_path: str) -> str: """Reads using the default encoding.""" - with open(file_path, encoding=DEFAULT_ENCODING) as f: + with open(file_path, encoding=DEFAULT_SOURCE_FILE_ENCODING) as f: return f.read() diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index da6d7cb3..900a04a0 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -13,6 +13,7 @@ class WriteMemoryTool(Tool): Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format. The memory name should be meaningful. """ + # NOTE: utf-8 encoding is configured in the MemoriesManager if max_answer_chars == -1: max_answer_chars = self.agent.serena_config.default_max_tool_answer_chars if len(content) > max_answer_chars: diff --git a/src/serena/util/general.py b/src/serena/util/general.py index b350ae18..4a1397e2 100644 --- a/src/serena/util/general.py +++ b/src/serena/util/general.py @@ -4,6 +4,8 @@ from typing import Literal, overload from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap +from serena.constants import SERENA_FILE_ENCODING + def _create_YAML(preserve_comments: bool = False) -> YAML: """ @@ -20,7 +22,7 @@ def load_yaml(path: str, preserve_comments: Literal[False]) -> dict: ... @overload def load_yaml(path: str, preserve_comments: Literal[True]) -> CommentedMap: ... def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap: - with open(path, encoding="utf-8") as f: + with open(path, encoding=SERENA_FILE_ENCODING) as f: yaml = _create_YAML(preserve_comments) return yaml.load(f) @@ -28,5 +30,5 @@ def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = False) -> None: yaml = _create_YAML(preserve_comments) os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w", encoding="utf-8") as f: + with open(path, "w", encoding=SERENA_FILE_ENCODING) as f: yaml.dump(data, f)