Introduce constant for Serena file encoding

* SERENA_FILE_ENCODING for Serena-specific files (config files, memories)
 * DEFAULT_SOURCE_FILE_ENCODING for source files (renamed from DEFAULT_ENCODING)

src/serena/agent.py

@@ -22,6 +22,7 @@ from serena import serena_version
 from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats
 from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode
 from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, ToolSet, get_serena_managed_in_project_dir
+from serena.constants import SERENA_FILE_ENCODING
 from serena.dashboard import SerenaDashboardAPI
 from serena.project import Project
 from serena.prompt_factory import SerenaPromptFactory
@@ -47,7 +48,7 @@ class MemoriesManager:
     def __init__(self, project_root: str):
         self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories"
         self._memory_dir.mkdir(parents=True, exist_ok=True)
-        self._encoding = "utf-8"
+        self._encoding = SERENA_FILE_ENCODING
 
     def _get_memory_file_path(self, name: str) -> Path:
         # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not.

src/serena/code_editor.py

@@ -12,7 +12,7 @@ from solidlsp.ls import LSPFileBuffer
 from solidlsp.ls_types import extract_text_edits
 from solidlsp.ls_utils import PathUtils, TextUtils
 
-from .constants import DEFAULT_ENCODING
+from .constants import DEFAULT_SOURCE_FILE_ENCODING
 from .project import Project
 from .tools.jetbrains_plugin_client import JetBrainsPluginClient
 
@@ -30,7 +30,7 @@ class CodeEditor(Generic[TSymbol], ABC):
         self.agent = agent
 
         # set encoding based on active project, if available
-        encoding = DEFAULT_ENCODING
+        encoding = DEFAULT_SOURCE_FILE_ENCODING
         if agent is not None:
             project = agent.get_active_project()
             if project is not None:

src/serena/config/context_mode.py

@@ -17,6 +17,7 @@ from serena.constants import (
     DEFAULT_CONTEXT,
     DEFAULT_MODES,
     INTERNAL_MODE_YAMLS_DIR,
+    SERENA_FILE_ENCODING,
     SERENAS_OWN_CONTEXT_YAMLS_DIR,
     SERENAS_OWN_MODE_YAMLS_DIR,
     USER_CONTEXT_YAMLS_DIR,
@@ -56,7 +57,7 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin):
     @classmethod
     def from_yaml(cls, yaml_path: str | Path) -> Self:
         """Load a mode from a YAML file."""
-        with open(yaml_path, encoding="utf-8") as f:
+        with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f:
             data = yaml.safe_load(f)
         name = data.pop("name", Path(yaml_path).stem)
         return cls(name=name, **data)
@@ -139,7 +140,7 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin):
     @classmethod
     def from_yaml(cls, yaml_path: str | Path) -> Self:
         """Load a context from a YAML file."""
-        with open(yaml_path, encoding="utf-8") as f:
+        with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f:
             data = yaml.safe_load(f)
         name = data.pop("name", Path(yaml_path).stem)
         # Ensure backwards compatibility for tool_description_overrides

src/serena/config/serena_config.py

@@ -19,10 +19,11 @@ from sensai.util.logging import LogTime, datetime_tag
 from sensai.util.string import ToStringMixin
 
 from serena.constants import (
-    DEFAULT_ENCODING,
+    DEFAULT_SOURCE_FILE_ENCODING,
     PROJECT_TEMPLATE_FILE,
     REPO_ROOT,
     SERENA_CONFIG_TEMPLATE_FILE,
+    SERENA_FILE_ENCODING,
     SERENA_MANAGED_DIR_IN_HOME,
     SERENA_MANAGED_DIR_NAME,
 )
@@ -166,7 +167,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
     read_only: bool = False
     ignore_all_files_in_gitignore: bool = True
     initial_prompt: str = ""
-    encoding: str = DEFAULT_ENCODING
+    encoding: str = DEFAULT_SOURCE_FILE_ENCODING
 
     SERENA_DEFAULT_PROJECT_FILE = "project.yml"
 
@@ -244,7 +245,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
             read_only=data.get("read_only", False),
             ignore_all_files_in_gitignore=data.get("ignore_all_files_in_gitignore", True),
             initial_prompt=data.get("initial_prompt", ""),
-            encoding=data.get("encoding", DEFAULT_ENCODING),
+            encoding=data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING),
         )
 
     @classmethod
@@ -259,7 +260,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
                 return cls.autogenerate(project_root)
             else:
                 raise FileNotFoundError(f"Project configuration file not found: {yaml_path}")
-        with open(yaml_path, encoding="utf-8") as f:
+        with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f:
             yaml_data = yaml.safe_load(f)
         if "project_name" not in yaml_data:
             yaml_data["project_name"] = project_root.name
@@ -483,11 +484,11 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
         """
         log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.")
         try:
-            with open(path, encoding="utf-8") as f:
+            with open(path, encoding=SERENA_FILE_ENCODING) as f:
                 project_config_data = yaml.safe_load(f)
             if "project_name" not in project_config_data:
                 project_name = path.stem
-                with open(path, "a", encoding="utf-8") as f:
+                with open(path, "a", encoding=SERENA_FILE_ENCODING) as f:
                     f.write(f"\nproject_name: {project_name}")
             project_root = project_config_data["project_root"]
             shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml()))

src/serena/constants.py

@@ -25,10 +25,14 @@ INTERNAL_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "inter
 SERENA_DASHBOARD_DIR = str(_serena_pkg_path / "resources" / "dashboard")
 SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons")
 
-DEFAULT_ENCODING = "utf-8"
+DEFAULT_SOURCE_FILE_ENCODING = "utf-8"
+"""The default encoding assumed for project source files."""
 DEFAULT_CONTEXT = "desktop-app"
 DEFAULT_MODES = ("interactive", "editing")
 
+SERENA_FILE_ENCODING = "utf-8"
+"""The encoding used for Serena's own files, such as configuration files and memories."""
+
 PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.yml")
 SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml")
 

src/serena/text_utils.py

@@ -9,7 +9,7 @@ from typing import Any, Self
 
 from joblib import Parallel, delayed
 
-from serena.constants import DEFAULT_ENCODING
+from serena.constants import DEFAULT_SOURCE_FILE_ENCODING
 
 log = logging.getLogger(__name__)
 
@@ -239,7 +239,7 @@ def search_text(
 
 def default_file_reader(file_path: str) -> str:
     """Reads using the default encoding."""
-    with open(file_path, encoding=DEFAULT_ENCODING) as f:
+    with open(file_path, encoding=DEFAULT_SOURCE_FILE_ENCODING) as f:
         return f.read()
 
 

src/serena/tools/memory_tools.py

@@ -13,6 +13,7 @@ class WriteMemoryTool(Tool):
         Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format.
         The memory name should be meaningful.
         """
+        # NOTE: utf-8 encoding is configured in the MemoriesManager
         if max_answer_chars == -1:
             max_answer_chars = self.agent.serena_config.default_max_tool_answer_chars
         if len(content) > max_answer_chars:

src/serena/util/general.py

@@ -4,6 +4,8 @@ from typing import Literal, overload
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
 
+from serena.constants import SERENA_FILE_ENCODING
+
 
 def _create_YAML(preserve_comments: bool = False) -> YAML:
     """
@@ -20,7 +22,7 @@ def load_yaml(path: str, preserve_comments: Literal[False]) -> dict: ...
 @overload
 def load_yaml(path: str, preserve_comments: Literal[True]) -> CommentedMap: ...
 def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap:
-    with open(path, encoding="utf-8") as f:
+    with open(path, encoding=SERENA_FILE_ENCODING) as f:
         yaml = _create_YAML(preserve_comments)
         return yaml.load(f)
 
@@ -28,5 +30,5 @@ def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap
 def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = False) -> None:
     yaml = _create_YAML(preserve_comments)
     os.makedirs(os.path.dirname(path), exist_ok=True)
-    with open(path, "w", encoding="utf-8") as f:
+    with open(path, "w", encoding=SERENA_FILE_ENCODING) as f:
         yaml.dump(data, f)