Merge pull request #674 from oraios/use-project-encoding

src/serena/agent.py

@@ -22,6 +22,7 @@ from serena import serena_version
 from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats
 from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode
 from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, ToolSet, get_serena_managed_in_project_dir
+from serena.constants import SERENA_FILE_ENCODING
 from serena.dashboard import SerenaDashboardAPI
 from serena.project import Project
 from serena.prompt_factory import SerenaPromptFactory
@@ -47,6 +48,7 @@ class MemoriesManager:
     def __init__(self, project_root: str):
         self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories"
         self._memory_dir.mkdir(parents=True, exist_ok=True)
+        self._encoding = SERENA_FILE_ENCODING
 
     def _get_memory_file_path(self, name: str) -> Path:
         # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not.
@@ -58,12 +60,12 @@ class MemoriesManager:
         memory_file_path = self._get_memory_file_path(name)
         if not memory_file_path.exists():
             return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it."
-        with open(memory_file_path, encoding="utf-8") as f:
+        with open(memory_file_path, encoding=self._encoding) as f:
             return f.read()
 
     def save_memory(self, name: str, content: str) -> str:
         memory_file_path = self._get_memory_file_path(name)
-        with open(memory_file_path, "w", encoding="utf-8") as f:
+        with open(memory_file_path, "w", encoding=self._encoding) as f:
             f.write(content)
         return f"Memory {name} written."
 

src/serena/code_editor.py

@@ -12,6 +12,7 @@ from solidlsp.ls import LSPFileBuffer
 from solidlsp.ls_types import extract_text_edits
 from solidlsp.ls_utils import PathUtils, TextUtils
 
+from .constants import DEFAULT_SOURCE_FILE_ENCODING
 from .project import Project
 from .tools.jetbrains_plugin_client import JetBrainsPluginClient
 
@@ -28,6 +29,14 @@ class CodeEditor(Generic[TSymbol], ABC):
         self.project_root = project_root
         self.agent = agent
 
+        # set encoding based on active project, if available
+        encoding = DEFAULT_SOURCE_FILE_ENCODING
+        if agent is not None:
+            project = agent.get_active_project()
+            if project is not None:
+                encoding = agent.get_active_project().project_config.encoding
+        self.encoding = encoding
+
     class EditedFile(ABC):
         @abstractmethod
         def get_contents(self) -> str:
@@ -59,7 +68,7 @@ class CodeEditor(Generic[TSymbol], ABC):
             yield edited_file
             # save the file
             abs_path = os.path.join(self.project_root, relative_path)
-            with open(abs_path, "w", encoding="utf-8") as f:
+            with open(abs_path, "w", encoding=self.encoding) as f:
                 f.write(edited_file.get_contents())
 
     @abstractmethod

src/serena/config/context_mode.py

@@ -17,6 +17,7 @@ from serena.constants import (
     DEFAULT_CONTEXT,
     DEFAULT_MODES,
     INTERNAL_MODE_YAMLS_DIR,
+    SERENA_FILE_ENCODING,
     SERENAS_OWN_CONTEXT_YAMLS_DIR,
     SERENAS_OWN_MODE_YAMLS_DIR,
     USER_CONTEXT_YAMLS_DIR,
@@ -56,7 +57,7 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin):
     @classmethod
     def from_yaml(cls, yaml_path: str | Path) -> Self:
         """Load a mode from a YAML file."""
-        with open(yaml_path, encoding="utf-8") as f:
+        with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f:
             data = yaml.safe_load(f)
         name = data.pop("name", Path(yaml_path).stem)
         return cls(name=name, **data)
@@ -139,7 +140,7 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin):
     @classmethod
     def from_yaml(cls, yaml_path: str | Path) -> Self:
         """Load a context from a YAML file."""
-        with open(yaml_path, encoding="utf-8") as f:
+        with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f:
             data = yaml.safe_load(f)
         name = data.pop("name", Path(yaml_path).stem)
         # Ensure backwards compatibility for tool_description_overrides

src/serena/config/serena_config.py

@@ -19,10 +19,11 @@ from sensai.util.logging import LogTime, datetime_tag
 from sensai.util.string import ToStringMixin
 
 from serena.constants import (
-    DEFAULT_ENCODING,
+    DEFAULT_SOURCE_FILE_ENCODING,
     PROJECT_TEMPLATE_FILE,
     REPO_ROOT,
     SERENA_CONFIG_TEMPLATE_FILE,
+    SERENA_FILE_ENCODING,
     SERENA_MANAGED_DIR_IN_HOME,
     SERENA_MANAGED_DIR_NAME,
 )
@@ -166,7 +167,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
     read_only: bool = False
     ignore_all_files_in_gitignore: bool = True
     initial_prompt: str = ""
-    encoding: str = DEFAULT_ENCODING
+    encoding: str = DEFAULT_SOURCE_FILE_ENCODING
 
     SERENA_DEFAULT_PROJECT_FILE = "project.yml"
 
@@ -244,7 +245,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
             read_only=data.get("read_only", False),
             ignore_all_files_in_gitignore=data.get("ignore_all_files_in_gitignore", True),
             initial_prompt=data.get("initial_prompt", ""),
-            encoding=data.get("encoding", DEFAULT_ENCODING),
+            encoding=data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING),
         )
 
     @classmethod
@@ -259,7 +260,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
                 return cls.autogenerate(project_root)
             else:
                 raise FileNotFoundError(f"Project configuration file not found: {yaml_path}")
-        with open(yaml_path, encoding="utf-8") as f:
+        with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f:
             yaml_data = yaml.safe_load(f)
         if "project_name" not in yaml_data:
             yaml_data["project_name"] = project_root.name
@@ -483,11 +484,11 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
         """
         log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.")
         try:
-            with open(path, encoding="utf-8") as f:
+            with open(path, encoding=SERENA_FILE_ENCODING) as f:
                 project_config_data = yaml.safe_load(f)
             if "project_name" not in project_config_data:
                 project_name = path.stem
-                with open(path, "a", encoding="utf-8") as f:
+                with open(path, "a", encoding=SERENA_FILE_ENCODING) as f:
                     f.write(f"\nproject_name: {project_name}")
             project_root = project_config_data["project_root"]
             shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml()))

src/serena/constants.py

@@ -25,10 +25,14 @@ INTERNAL_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "inter
 SERENA_DASHBOARD_DIR = str(_serena_pkg_path / "resources" / "dashboard")
 SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons")
 
-DEFAULT_ENCODING = "utf-8"
+DEFAULT_SOURCE_FILE_ENCODING = "utf-8"
+"""The default encoding assumed for project source files."""
 DEFAULT_CONTEXT = "desktop-app"
 DEFAULT_MODES = ("interactive", "editing")
 
+SERENA_FILE_ENCODING = "utf-8"
+"""The encoding used for Serena's own files, such as configuration files and memories."""
+
 PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.yml")
 SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml")
 

src/serena/project.py

@@ -255,6 +255,7 @@ class Project:
             relative_file_paths,
             pattern,
             root_path=self.project_root,
+            file_reader=self.read_file,
             context_lines_before=context_lines_before,
             context_lines_after=context_lines_after,
             paths_include_glob=paths_include_glob,
@@ -307,6 +308,7 @@ class Project:
             code_language=self.language,
             ignored_paths=self._ignored_patterns,
             trace_lsp_communication=trace_lsp_communication,
+            encoding=self.project_config.encoding,
         )
         ls_logger = LanguageServerLogger(log_level=log_level)
 

src/serena/text_utils.py

@@ -9,6 +9,8 @@ from typing import Any, Self
 
 from joblib import Parallel, delayed
 
+from serena.constants import DEFAULT_SOURCE_FILE_ENCODING
+
 log = logging.getLogger(__name__)
 
 
@@ -236,8 +238,8 @@ def search_text(
 
 
 def default_file_reader(file_path: str) -> str:
-    """Reads using utf-8 encoding."""
-    with open(file_path, encoding="utf-8") as f:
+    """Reads using the default encoding."""
+    with open(file_path, encoding=DEFAULT_SOURCE_FILE_ENCODING) as f:
         return f.read()
 
 

src/serena/tools/file_tools.py

@@ -59,7 +59,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit):
         Write a new file or overwrite an existing file.
 
         :param relative_path: the relative path to the file to create
-        :param content: the (utf-8-encoded) content to write to the file
+        :param content: the (appropriately encoded) content to write to the file
         :return: a message indicating success or failure
         """
         project_root = self.get_project_root()
@@ -74,7 +74,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit):
             ), f"Cannot create file outside of the project directory, got {relative_path=}"
 
         abs_path.parent.mkdir(parents=True, exist_ok=True)
-        abs_path.write_text(content, encoding="utf-8")
+        abs_path.write_text(content, encoding=self.project.project_config.encoding)
         answer = f"File created: {relative_path}."
         if will_overwrite_existing:
             answer += " Overwrote existing file."
@@ -390,6 +390,7 @@ class SearchForPatternTool(Tool):
             matches = search_files(
                 rel_paths_to_search,
                 substring_pattern,
+                file_reader=self.project.read_file,
                 root_path=self.get_project_root(),
                 paths_include_glob=paths_include_glob,
                 paths_exclude_glob=paths_exclude_glob,

src/serena/tools/memory_tools.py

@@ -10,9 +10,10 @@ class WriteMemoryTool(Tool):
 
     def apply(self, memory_name: str, content: str, max_answer_chars: int = -1) -> str:
         """
-        Write some information about this project that can be useful for future tasks to a memory in md format.
+        Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format.
         The memory name should be meaningful.
         """
+        # NOTE: utf-8 encoding is configured in the MemoriesManager
         if max_answer_chars == -1:
             max_answer_chars = self.agent.serena_config.default_max_tool_answer_chars
         if len(content) > max_answer_chars:

src/serena/util/general.py

@@ -4,6 +4,8 @@ from typing import Literal, overload
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
 
+from serena.constants import SERENA_FILE_ENCODING
+
 
 def _create_YAML(preserve_comments: bool = False) -> YAML:
     """
@@ -20,7 +22,7 @@ def load_yaml(path: str, preserve_comments: Literal[False]) -> dict: ...
 @overload
 def load_yaml(path: str, preserve_comments: Literal[True]) -> CommentedMap: ...
 def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap:
-    with open(path, encoding="utf-8") as f:
+    with open(path, encoding=SERENA_FILE_ENCODING) as f:
         yaml = _create_YAML(preserve_comments)
         return yaml.load(f)
 
@@ -28,5 +30,5 @@ def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap
 def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = False) -> None:
     yaml = _create_YAML(preserve_comments)
     os.makedirs(os.path.dirname(path), exist_ok=True)
-    with open(path, "w", encoding="utf-8") as f:
+    with open(path, "w", encoding=SERENA_FILE_ENCODING) as f:
         yaml.dump(data, f)

src/solidlsp/ls.py

@@ -176,6 +176,7 @@ class SolidLanguageServer(ABC):
                     as opposed to HTTP, TCP modes supported by some language servers.
         """
         self._solidlsp_settings = solidlsp_settings
+        self._encoding = config.encoding
         self.logger = logger
         self.repository_root_path: str = repository_root_path
         self.logger.log(
@@ -391,7 +392,7 @@ class SolidLanguageServer(ABC):
             yield self.open_file_buffers[uri]
             self.open_file_buffers[uri].ref_count -= 1
         else:
-            contents = FileUtils.read_file(self.logger, absolute_file_path)
+            contents = FileUtils.read_file(self.logger, absolute_file_path, self._encoding)
 
             version = 0
             self.open_file_buffers[uri] = LSPFileBuffer(uri, contents, version, self.language_id, 1)
@@ -1408,7 +1409,7 @@ class SolidLanguageServer(ABC):
         # checking if the line is empty, unfortunately ugly and duplicating code, but I don't want to refactor
         with self.open_file(relative_file_path):
             absolute_file_path = str(PurePath(self.repository_root_path, relative_file_path))
-            content = FileUtils.read_file(self.logger, absolute_file_path)
+            content = FileUtils.read_file(self.logger, absolute_file_path, self._encoding)
             if content.split("\n")[line].strip() == "":
                 self.logger.log(
                     f"Passing empty lines to request_container_symbol is currently not supported, {relative_file_path=}, {line=}",

src/solidlsp/ls_config.py

@@ -306,6 +306,8 @@ class LanguageServerConfig:
     start_independent_lsp_process: bool = True
     ignored_paths: list[str] = field(default_factory=list)
     """Paths, dirs or glob-like patterns. The matching will follow the same logic as for .gitignore entries"""
+    encoding: str = "utf-8"
+    """File encoding to use when reading source files"""
 
     @classmethod
     def from_dict(cls, env: dict):

src/solidlsp/ls_utils.py

@@ -165,7 +165,7 @@ class FileUtils:
     """
 
     @staticmethod
-    def read_file(logger: LanguageServerLogger, file_path: str) -> str:
+    def read_file(logger: LanguageServerLogger, file_path: str, encoding: str) -> str:
         """
         Reads the file at the given path and returns the contents as a string.
         """
@@ -173,10 +173,10 @@ class FileUtils:
             logger.log(f"File read '{file_path}' failed: File does not exist.", logging.ERROR)
             raise SolidLSPException(f"File read '{file_path}' failed: File does not exist.")
         try:
-            with open(file_path, encoding="utf-8") as inp_file:
+            with open(file_path, encoding=encoding) as inp_file:
                 return inp_file.read()
         except Exception as exc:
-            logger.log(f"File read '{file_path}' failed to read with encoding 'utf-8': {exc}", logging.ERROR)
+            logger.log(f"File read '{file_path}' failed to read with encoding '{encoding}': {exc}", logging.ERROR)
             raise SolidLSPException("File read failed.") from None
 
     @staticmethod