From 4e3db0bd0938621a0b12b3c4108263f4c86c656e Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 16 Oct 2025 14:56:58 +0200 Subject: [PATCH] Consistently use the project's encoding setting, also passing it on the LS --- src/serena/agent.py | 5 +++-- src/serena/code_editor.py | 11 ++++++++++- src/serena/project.py | 2 ++ src/serena/text_utils.py | 6 ++++-- src/serena/tools/file_tools.py | 5 +++-- src/serena/tools/memory_tools.py | 2 +- src/solidlsp/ls.py | 5 +++-- src/solidlsp/ls_config.py | 2 ++ src/solidlsp/ls_utils.py | 6 +++--- 9 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index d8a491dd..1c95b5fb 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -47,6 +47,7 @@ class MemoriesManager: def __init__(self, project_root: str): self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" self._memory_dir.mkdir(parents=True, exist_ok=True) + self._encoding = "utf-8" def _get_memory_file_path(self, name: str) -> Path: # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not. @@ -58,12 +59,12 @@ class MemoriesManager: memory_file_path = self._get_memory_file_path(name) if not memory_file_path.exists(): return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it." - with open(memory_file_path, encoding="utf-8") as f: + with open(memory_file_path, encoding=self._encoding) as f: return f.read() def save_memory(self, name: str, content: str) -> str: memory_file_path = self._get_memory_file_path(name) - with open(memory_file_path, "w", encoding="utf-8") as f: + with open(memory_file_path, "w", encoding=self._encoding) as f: f.write(content) return f"Memory {name} written." diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index ef00970b..5dc3fe65 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -12,6 +12,7 @@ from solidlsp.ls import LSPFileBuffer from solidlsp.ls_types import extract_text_edits from solidlsp.ls_utils import PathUtils, TextUtils +from .constants import DEFAULT_ENCODING from .project import Project from .tools.jetbrains_plugin_client import JetBrainsPluginClient @@ -28,6 +29,14 @@ class CodeEditor(Generic[TSymbol], ABC): self.project_root = project_root self.agent = agent + # set encoding based on active project, if available + encoding = DEFAULT_ENCODING + if agent is not None: + project = agent.get_active_project() + if project is not None: + encoding = agent.get_active_project().project_config.encoding + self.encoding = encoding + class EditedFile(ABC): @abstractmethod def get_contents(self) -> str: @@ -59,7 +68,7 @@ class CodeEditor(Generic[TSymbol], ABC): yield edited_file # save the file abs_path = os.path.join(self.project_root, relative_path) - with open(abs_path, "w", encoding="utf-8") as f: + with open(abs_path, "w", encoding=self.encoding) as f: f.write(edited_file.get_contents()) @abstractmethod diff --git a/src/serena/project.py b/src/serena/project.py index 966fc8c7..e1d8c743 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -249,6 +249,7 @@ class Project: relative_file_paths, pattern, root_path=self.project_root, + file_reader=self.read_file, context_lines_before=context_lines_before, context_lines_after=context_lines_after, paths_include_glob=paths_include_glob, @@ -301,6 +302,7 @@ class Project: code_language=self.language, ignored_paths=self._ignored_patterns, trace_lsp_communication=trace_lsp_communication, + encoding=self.project_config.encoding, ) ls_logger = LanguageServerLogger(log_level=log_level) diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 19987cee..3eea8ba6 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -9,6 +9,8 @@ from typing import Any, Self from joblib import Parallel, delayed +from serena.constants import DEFAULT_ENCODING + log = logging.getLogger(__name__) @@ -236,8 +238,8 @@ def search_text( def default_file_reader(file_path: str) -> str: - """Reads using utf-8 encoding.""" - with open(file_path, encoding="utf-8") as f: + """Reads using the default encoding.""" + with open(file_path, encoding=DEFAULT_ENCODING) as f: return f.read() diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index d7ec5a41..38cfd2f7 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -59,7 +59,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit): Write a new file or overwrite an existing file. :param relative_path: the relative path to the file to create - :param content: the (utf-8-encoded) content to write to the file + :param content: the (appropriately encoded) content to write to the file :return: a message indicating success or failure """ project_root = self.get_project_root() @@ -74,7 +74,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit): ), f"Cannot create file outside of the project directory, got {relative_path=}" abs_path.parent.mkdir(parents=True, exist_ok=True) - abs_path.write_text(content, encoding="utf-8") + abs_path.write_text(content, encoding=self.project.project_config.encoding) answer = f"File created: {relative_path}." if will_overwrite_existing: answer += " Overwrote existing file." @@ -390,6 +390,7 @@ class SearchForPatternTool(Tool): matches = search_files( rel_paths_to_search, substring_pattern, + file_reader=self.project.read_file, root_path=self.get_project_root(), paths_include_glob=paths_include_glob, paths_exclude_glob=paths_exclude_glob, diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index f52ae3f4..da6d7cb3 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -10,7 +10,7 @@ class WriteMemoryTool(Tool): def apply(self, memory_name: str, content: str, max_answer_chars: int = -1) -> str: """ - Write some information about this project that can be useful for future tasks to a memory in md format. + Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format. The memory name should be meaningful. """ if max_answer_chars == -1: diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 29df1ee4..d4a9c111 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -176,6 +176,7 @@ class SolidLanguageServer(ABC): as opposed to HTTP, TCP modes supported by some language servers. """ self._solidlsp_settings = solidlsp_settings + self._encoding = config.encoding self.logger = logger self.repository_root_path: str = repository_root_path self.logger.log( @@ -391,7 +392,7 @@ class SolidLanguageServer(ABC): yield self.open_file_buffers[uri] self.open_file_buffers[uri].ref_count -= 1 else: - contents = FileUtils.read_file(self.logger, absolute_file_path) + contents = FileUtils.read_file(self.logger, absolute_file_path, self._encoding) version = 0 self.open_file_buffers[uri] = LSPFileBuffer(uri, contents, version, self.language_id, 1) @@ -1408,7 +1409,7 @@ class SolidLanguageServer(ABC): # checking if the line is empty, unfortunately ugly and duplicating code, but I don't want to refactor with self.open_file(relative_file_path): absolute_file_path = str(PurePath(self.repository_root_path, relative_file_path)) - content = FileUtils.read_file(self.logger, absolute_file_path) + content = FileUtils.read_file(self.logger, absolute_file_path, self._encoding) if content.split("\n")[line].strip() == "": self.logger.log( f"Passing empty lines to request_container_symbol is currently not supported, {relative_file_path=}, {line=}", diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index d70c1dce..cc023e3b 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -306,6 +306,8 @@ class LanguageServerConfig: start_independent_lsp_process: bool = True ignored_paths: list[str] = field(default_factory=list) """Paths, dirs or glob-like patterns. The matching will follow the same logic as for .gitignore entries""" + encoding: str = "utf-8" + """File encoding to use when reading source files""" @classmethod def from_dict(cls, env: dict): diff --git a/src/solidlsp/ls_utils.py b/src/solidlsp/ls_utils.py index 11fe88f3..3cc124f9 100644 --- a/src/solidlsp/ls_utils.py +++ b/src/solidlsp/ls_utils.py @@ -165,7 +165,7 @@ class FileUtils: """ @staticmethod - def read_file(logger: LanguageServerLogger, file_path: str) -> str: + def read_file(logger: LanguageServerLogger, file_path: str, encoding: str) -> str: """ Reads the file at the given path and returns the contents as a string. """ @@ -173,10 +173,10 @@ class FileUtils: logger.log(f"File read '{file_path}' failed: File does not exist.", logging.ERROR) raise SolidLSPException(f"File read '{file_path}' failed: File does not exist.") try: - with open(file_path, encoding="utf-8") as inp_file: + with open(file_path, encoding=encoding) as inp_file: return inp_file.read() except Exception as exc: - logger.log(f"File read '{file_path}' failed to read with encoding 'utf-8': {exc}", logging.ERROR) + logger.log(f"File read '{file_path}' failed to read with encoding '{encoding}': {exc}", logging.ERROR) raise SolidLSPException("File read failed.") from None @staticmethod