Consistently use the project's encoding setting, also passing it on the LS

src/serena/agent.py

@@ -47,6 +47,7 @@ class MemoriesManager:
     def __init__(self, project_root: str):
         self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories"
         self._memory_dir.mkdir(parents=True, exist_ok=True)
+        self._encoding = "utf-8"
 
     def _get_memory_file_path(self, name: str) -> Path:
         # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not.
@@ -58,12 +59,12 @@ class MemoriesManager:
         memory_file_path = self._get_memory_file_path(name)
         if not memory_file_path.exists():
             return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it."
-        with open(memory_file_path, encoding="utf-8") as f:
+        with open(memory_file_path, encoding=self._encoding) as f:
             return f.read()
 
     def save_memory(self, name: str, content: str) -> str:
         memory_file_path = self._get_memory_file_path(name)
-        with open(memory_file_path, "w", encoding="utf-8") as f:
+        with open(memory_file_path, "w", encoding=self._encoding) as f:
             f.write(content)
         return f"Memory {name} written."
 

src/serena/code_editor.py

@@ -12,6 +12,7 @@ from solidlsp.ls import LSPFileBuffer
 from solidlsp.ls_types import extract_text_edits
 from solidlsp.ls_utils import PathUtils, TextUtils
 
+from .constants import DEFAULT_ENCODING
 from .project import Project
 from .tools.jetbrains_plugin_client import JetBrainsPluginClient
 
@@ -28,6 +29,14 @@ class CodeEditor(Generic[TSymbol], ABC):
         self.project_root = project_root
         self.agent = agent
 
+        # set encoding based on active project, if available
+        encoding = DEFAULT_ENCODING
+        if agent is not None:
+            project = agent.get_active_project()
+            if project is not None:
+                encoding = agent.get_active_project().project_config.encoding
+        self.encoding = encoding
+
     class EditedFile(ABC):
         @abstractmethod
         def get_contents(self) -> str:
@@ -59,7 +68,7 @@ class CodeEditor(Generic[TSymbol], ABC):
             yield edited_file
             # save the file
             abs_path = os.path.join(self.project_root, relative_path)
-            with open(abs_path, "w", encoding="utf-8") as f:
+            with open(abs_path, "w", encoding=self.encoding) as f:
                 f.write(edited_file.get_contents())
 
     @abstractmethod

src/serena/project.py

@@ -249,6 +249,7 @@ class Project:
             relative_file_paths,
             pattern,
             root_path=self.project_root,
+            file_reader=self.read_file,
             context_lines_before=context_lines_before,
             context_lines_after=context_lines_after,
             paths_include_glob=paths_include_glob,
@@ -301,6 +302,7 @@ class Project:
             code_language=self.language,
             ignored_paths=self._ignored_patterns,
             trace_lsp_communication=trace_lsp_communication,
+            encoding=self.project_config.encoding,
         )
         ls_logger = LanguageServerLogger(log_level=log_level)
 

src/serena/text_utils.py

@@ -9,6 +9,8 @@ from typing import Any, Self
 
 from joblib import Parallel, delayed
 
+from serena.constants import DEFAULT_ENCODING
+
 log = logging.getLogger(__name__)
 
 
@@ -236,8 +238,8 @@ def search_text(
 
 
 def default_file_reader(file_path: str) -> str:
-    """Reads using utf-8 encoding."""
-    with open(file_path, encoding="utf-8") as f:
+    """Reads using the default encoding."""
+    with open(file_path, encoding=DEFAULT_ENCODING) as f:
         return f.read()
 
 

src/serena/tools/file_tools.py

@@ -59,7 +59,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit):
         Write a new file or overwrite an existing file.
 
         :param relative_path: the relative path to the file to create
-        :param content: the (utf-8-encoded) content to write to the file
+        :param content: the (appropriately encoded) content to write to the file
         :return: a message indicating success or failure
         """
         project_root = self.get_project_root()
@@ -74,7 +74,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit):
             ), f"Cannot create file outside of the project directory, got {relative_path=}"
 
         abs_path.parent.mkdir(parents=True, exist_ok=True)
-        abs_path.write_text(content, encoding="utf-8")
+        abs_path.write_text(content, encoding=self.project.project_config.encoding)
         answer = f"File created: {relative_path}."
         if will_overwrite_existing:
             answer += " Overwrote existing file."
@@ -390,6 +390,7 @@ class SearchForPatternTool(Tool):
             matches = search_files(
                 rel_paths_to_search,
                 substring_pattern,
+                file_reader=self.project.read_file,
                 root_path=self.get_project_root(),
                 paths_include_glob=paths_include_glob,
                 paths_exclude_glob=paths_exclude_glob,

src/serena/tools/memory_tools.py

@@ -10,7 +10,7 @@ class WriteMemoryTool(Tool):
 
     def apply(self, memory_name: str, content: str, max_answer_chars: int = -1) -> str:
         """
-        Write some information about this project that can be useful for future tasks to a memory in md format.
+        Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format.
         The memory name should be meaningful.
         """
         if max_answer_chars == -1:

src/solidlsp/ls.py

@@ -176,6 +176,7 @@ class SolidLanguageServer(ABC):
                     as opposed to HTTP, TCP modes supported by some language servers.
         """
         self._solidlsp_settings = solidlsp_settings
+        self._encoding = config.encoding
         self.logger = logger
         self.repository_root_path: str = repository_root_path
         self.logger.log(
@@ -391,7 +392,7 @@ class SolidLanguageServer(ABC):
             yield self.open_file_buffers[uri]
             self.open_file_buffers[uri].ref_count -= 1
         else:
-            contents = FileUtils.read_file(self.logger, absolute_file_path)
+            contents = FileUtils.read_file(self.logger, absolute_file_path, self._encoding)
 
             version = 0
             self.open_file_buffers[uri] = LSPFileBuffer(uri, contents, version, self.language_id, 1)
@@ -1408,7 +1409,7 @@ class SolidLanguageServer(ABC):
         # checking if the line is empty, unfortunately ugly and duplicating code, but I don't want to refactor
         with self.open_file(relative_file_path):
             absolute_file_path = str(PurePath(self.repository_root_path, relative_file_path))
-            content = FileUtils.read_file(self.logger, absolute_file_path)
+            content = FileUtils.read_file(self.logger, absolute_file_path, self._encoding)
             if content.split("\n")[line].strip() == "":
                 self.logger.log(
                     f"Passing empty lines to request_container_symbol is currently not supported, {relative_file_path=}, {line=}",

src/solidlsp/ls_config.py

@@ -306,6 +306,8 @@ class LanguageServerConfig:
     start_independent_lsp_process: bool = True
     ignored_paths: list[str] = field(default_factory=list)
     """Paths, dirs or glob-like patterns. The matching will follow the same logic as for .gitignore entries"""
+    encoding: str = "utf-8"
+    """File encoding to use when reading source files"""
 
     @classmethod
     def from_dict(cls, env: dict):

src/solidlsp/ls_utils.py

@@ -165,7 +165,7 @@ class FileUtils:
     """
 
     @staticmethod
-    def read_file(logger: LanguageServerLogger, file_path: str) -> str:
+    def read_file(logger: LanguageServerLogger, file_path: str, encoding: str) -> str:
         """
         Reads the file at the given path and returns the contents as a string.
         """
@@ -173,10 +173,10 @@ class FileUtils:
             logger.log(f"File read '{file_path}' failed: File does not exist.", logging.ERROR)
             raise SolidLSPException(f"File read '{file_path}' failed: File does not exist.")
         try:
-            with open(file_path, encoding="utf-8") as inp_file:
+            with open(file_path, encoding=encoding) as inp_file:
                 return inp_file.read()
         except Exception as exc:
-            logger.log(f"File read '{file_path}' failed to read with encoding 'utf-8': {exc}", logging.ERROR)
+            logger.log(f"File read '{file_path}' failed to read with encoding '{encoding}': {exc}", logging.ERROR)
             raise SolidLSPException("File read failed.") from None
 
     @staticmethod