From 4e3db0bd0938621a0b12b3c4108263f4c86c656e Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 16 Oct 2025 14:56:58 +0200 Subject: [PATCH 1/2] Consistently use the project's encoding setting, also passing it on the LS --- src/serena/agent.py | 5 +++-- src/serena/code_editor.py | 11 ++++++++++- src/serena/project.py | 2 ++ src/serena/text_utils.py | 6 ++++-- src/serena/tools/file_tools.py | 5 +++-- src/serena/tools/memory_tools.py | 2 +- src/solidlsp/ls.py | 5 +++-- src/solidlsp/ls_config.py | 2 ++ src/solidlsp/ls_utils.py | 6 +++--- 9 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index d8a491dd..1c95b5fb 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -47,6 +47,7 @@ class MemoriesManager: def __init__(self, project_root: str): self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" self._memory_dir.mkdir(parents=True, exist_ok=True) + self._encoding = "utf-8" def _get_memory_file_path(self, name: str) -> Path: # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not. @@ -58,12 +59,12 @@ class MemoriesManager: memory_file_path = self._get_memory_file_path(name) if not memory_file_path.exists(): return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it." - with open(memory_file_path, encoding="utf-8") as f: + with open(memory_file_path, encoding=self._encoding) as f: return f.read() def save_memory(self, name: str, content: str) -> str: memory_file_path = self._get_memory_file_path(name) - with open(memory_file_path, "w", encoding="utf-8") as f: + with open(memory_file_path, "w", encoding=self._encoding) as f: f.write(content) return f"Memory {name} written." diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index ef00970b..5dc3fe65 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -12,6 +12,7 @@ from solidlsp.ls import LSPFileBuffer from solidlsp.ls_types import extract_text_edits from solidlsp.ls_utils import PathUtils, TextUtils +from .constants import DEFAULT_ENCODING from .project import Project from .tools.jetbrains_plugin_client import JetBrainsPluginClient @@ -28,6 +29,14 @@ class CodeEditor(Generic[TSymbol], ABC): self.project_root = project_root self.agent = agent + # set encoding based on active project, if available + encoding = DEFAULT_ENCODING + if agent is not None: + project = agent.get_active_project() + if project is not None: + encoding = agent.get_active_project().project_config.encoding + self.encoding = encoding + class EditedFile(ABC): @abstractmethod def get_contents(self) -> str: @@ -59,7 +68,7 @@ class CodeEditor(Generic[TSymbol], ABC): yield edited_file # save the file abs_path = os.path.join(self.project_root, relative_path) - with open(abs_path, "w", encoding="utf-8") as f: + with open(abs_path, "w", encoding=self.encoding) as f: f.write(edited_file.get_contents()) @abstractmethod diff --git a/src/serena/project.py b/src/serena/project.py index 966fc8c7..e1d8c743 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -249,6 +249,7 @@ class Project: relative_file_paths, pattern, root_path=self.project_root, + file_reader=self.read_file, context_lines_before=context_lines_before, context_lines_after=context_lines_after, paths_include_glob=paths_include_glob, @@ -301,6 +302,7 @@ class Project: code_language=self.language, ignored_paths=self._ignored_patterns, trace_lsp_communication=trace_lsp_communication, + encoding=self.project_config.encoding, ) ls_logger = LanguageServerLogger(log_level=log_level) diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 19987cee..3eea8ba6 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -9,6 +9,8 @@ from typing import Any, Self from joblib import Parallel, delayed +from serena.constants import DEFAULT_ENCODING + log = logging.getLogger(__name__) @@ -236,8 +238,8 @@ def search_text( def default_file_reader(file_path: str) -> str: - """Reads using utf-8 encoding.""" - with open(file_path, encoding="utf-8") as f: + """Reads using the default encoding.""" + with open(file_path, encoding=DEFAULT_ENCODING) as f: return f.read() diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index d7ec5a41..38cfd2f7 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -59,7 +59,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit): Write a new file or overwrite an existing file. :param relative_path: the relative path to the file to create - :param content: the (utf-8-encoded) content to write to the file + :param content: the (appropriately encoded) content to write to the file :return: a message indicating success or failure """ project_root = self.get_project_root() @@ -74,7 +74,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit): ), f"Cannot create file outside of the project directory, got {relative_path=}" abs_path.parent.mkdir(parents=True, exist_ok=True) - abs_path.write_text(content, encoding="utf-8") + abs_path.write_text(content, encoding=self.project.project_config.encoding) answer = f"File created: {relative_path}." if will_overwrite_existing: answer += " Overwrote existing file." @@ -390,6 +390,7 @@ class SearchForPatternTool(Tool): matches = search_files( rel_paths_to_search, substring_pattern, + file_reader=self.project.read_file, root_path=self.get_project_root(), paths_include_glob=paths_include_glob, paths_exclude_glob=paths_exclude_glob, diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index f52ae3f4..da6d7cb3 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -10,7 +10,7 @@ class WriteMemoryTool(Tool): def apply(self, memory_name: str, content: str, max_answer_chars: int = -1) -> str: """ - Write some information about this project that can be useful for future tasks to a memory in md format. + Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format. The memory name should be meaningful. """ if max_answer_chars == -1: diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 29df1ee4..d4a9c111 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -176,6 +176,7 @@ class SolidLanguageServer(ABC): as opposed to HTTP, TCP modes supported by some language servers. """ self._solidlsp_settings = solidlsp_settings + self._encoding = config.encoding self.logger = logger self.repository_root_path: str = repository_root_path self.logger.log( @@ -391,7 +392,7 @@ class SolidLanguageServer(ABC): yield self.open_file_buffers[uri] self.open_file_buffers[uri].ref_count -= 1 else: - contents = FileUtils.read_file(self.logger, absolute_file_path) + contents = FileUtils.read_file(self.logger, absolute_file_path, self._encoding) version = 0 self.open_file_buffers[uri] = LSPFileBuffer(uri, contents, version, self.language_id, 1) @@ -1408,7 +1409,7 @@ class SolidLanguageServer(ABC): # checking if the line is empty, unfortunately ugly and duplicating code, but I don't want to refactor with self.open_file(relative_file_path): absolute_file_path = str(PurePath(self.repository_root_path, relative_file_path)) - content = FileUtils.read_file(self.logger, absolute_file_path) + content = FileUtils.read_file(self.logger, absolute_file_path, self._encoding) if content.split("\n")[line].strip() == "": self.logger.log( f"Passing empty lines to request_container_symbol is currently not supported, {relative_file_path=}, {line=}", diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index d70c1dce..cc023e3b 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -306,6 +306,8 @@ class LanguageServerConfig: start_independent_lsp_process: bool = True ignored_paths: list[str] = field(default_factory=list) """Paths, dirs or glob-like patterns. The matching will follow the same logic as for .gitignore entries""" + encoding: str = "utf-8" + """File encoding to use when reading source files""" @classmethod def from_dict(cls, env: dict): diff --git a/src/solidlsp/ls_utils.py b/src/solidlsp/ls_utils.py index 11fe88f3..3cc124f9 100644 --- a/src/solidlsp/ls_utils.py +++ b/src/solidlsp/ls_utils.py @@ -165,7 +165,7 @@ class FileUtils: """ @staticmethod - def read_file(logger: LanguageServerLogger, file_path: str) -> str: + def read_file(logger: LanguageServerLogger, file_path: str, encoding: str) -> str: """ Reads the file at the given path and returns the contents as a string. """ @@ -173,10 +173,10 @@ class FileUtils: logger.log(f"File read '{file_path}' failed: File does not exist.", logging.ERROR) raise SolidLSPException(f"File read '{file_path}' failed: File does not exist.") try: - with open(file_path, encoding="utf-8") as inp_file: + with open(file_path, encoding=encoding) as inp_file: return inp_file.read() except Exception as exc: - logger.log(f"File read '{file_path}' failed to read with encoding 'utf-8': {exc}", logging.ERROR) + logger.log(f"File read '{file_path}' failed to read with encoding '{encoding}': {exc}", logging.ERROR) raise SolidLSPException("File read failed.") from None @staticmethod From 3cde03a8fb85b47921d3de135b2c9b3e03dc29c9 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 16 Oct 2025 22:09:39 +0200 Subject: [PATCH 2/2] Introduce constant for Serena file encoding * SERENA_FILE_ENCODING for Serena-specific files (config files, memories) * DEFAULT_SOURCE_FILE_ENCODING for source files (renamed from DEFAULT_ENCODING) --- src/serena/agent.py | 3 ++- src/serena/code_editor.py | 4 ++-- src/serena/config/context_mode.py | 5 +++-- src/serena/config/serena_config.py | 13 +++++++------ src/serena/constants.py | 6 +++++- src/serena/text_utils.py | 4 ++-- src/serena/tools/memory_tools.py | 1 + src/serena/util/general.py | 6 ++++-- 8 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 1c95b5fb..9358db1d 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -22,6 +22,7 @@ from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, ToolSet, get_serena_managed_in_project_dir +from serena.constants import SERENA_FILE_ENCODING from serena.dashboard import SerenaDashboardAPI from serena.project import Project from serena.prompt_factory import SerenaPromptFactory @@ -47,7 +48,7 @@ class MemoriesManager: def __init__(self, project_root: str): self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" self._memory_dir.mkdir(parents=True, exist_ok=True) - self._encoding = "utf-8" + self._encoding = SERENA_FILE_ENCODING def _get_memory_file_path(self, name: str) -> Path: # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not. diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index 5dc3fe65..3d99d2bb 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -12,7 +12,7 @@ from solidlsp.ls import LSPFileBuffer from solidlsp.ls_types import extract_text_edits from solidlsp.ls_utils import PathUtils, TextUtils -from .constants import DEFAULT_ENCODING +from .constants import DEFAULT_SOURCE_FILE_ENCODING from .project import Project from .tools.jetbrains_plugin_client import JetBrainsPluginClient @@ -30,7 +30,7 @@ class CodeEditor(Generic[TSymbol], ABC): self.agent = agent # set encoding based on active project, if available - encoding = DEFAULT_ENCODING + encoding = DEFAULT_SOURCE_FILE_ENCODING if agent is not None: project = agent.get_active_project() if project is not None: diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index 44cd42fb..ed2eb11c 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -17,6 +17,7 @@ from serena.constants import ( DEFAULT_CONTEXT, DEFAULT_MODES, INTERNAL_MODE_YAMLS_DIR, + SERENA_FILE_ENCODING, SERENAS_OWN_CONTEXT_YAMLS_DIR, SERENAS_OWN_MODE_YAMLS_DIR, USER_CONTEXT_YAMLS_DIR, @@ -56,7 +57,7 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a mode from a YAML file.""" - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) return cls(name=name, **data) @@ -139,7 +140,7 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a context from a YAML file.""" - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) # Ensure backwards compatibility for tool_description_overrides diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 00bbc901..5b9d4462 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -19,10 +19,11 @@ from sensai.util.logging import LogTime, datetime_tag from sensai.util.string import ToStringMixin from serena.constants import ( - DEFAULT_ENCODING, + DEFAULT_SOURCE_FILE_ENCODING, PROJECT_TEMPLATE_FILE, REPO_ROOT, SERENA_CONFIG_TEMPLATE_FILE, + SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME, ) @@ -166,7 +167,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): read_only: bool = False ignore_all_files_in_gitignore: bool = True initial_prompt: str = "" - encoding: str = DEFAULT_ENCODING + encoding: str = DEFAULT_SOURCE_FILE_ENCODING SERENA_DEFAULT_PROJECT_FILE = "project.yml" @@ -244,7 +245,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): read_only=data.get("read_only", False), ignore_all_files_in_gitignore=data.get("ignore_all_files_in_gitignore", True), initial_prompt=data.get("initial_prompt", ""), - encoding=data.get("encoding", DEFAULT_ENCODING), + encoding=data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING), ) @classmethod @@ -259,7 +260,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): return cls.autogenerate(project_root) else: raise FileNotFoundError(f"Project configuration file not found: {yaml_path}") - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: yaml_data = yaml.safe_load(f) if "project_name" not in yaml_data: yaml_data["project_name"] = project_root.name @@ -483,11 +484,11 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): """ log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.") try: - with open(path, encoding="utf-8") as f: + with open(path, encoding=SERENA_FILE_ENCODING) as f: project_config_data = yaml.safe_load(f) if "project_name" not in project_config_data: project_name = path.stem - with open(path, "a", encoding="utf-8") as f: + with open(path, "a", encoding=SERENA_FILE_ENCODING) as f: f.write(f"\nproject_name: {project_name}") project_root = project_config_data["project_root"] shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml())) diff --git a/src/serena/constants.py b/src/serena/constants.py index 1e5d98a1..e0ed29cf 100644 --- a/src/serena/constants.py +++ b/src/serena/constants.py @@ -25,10 +25,14 @@ INTERNAL_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "inter SERENA_DASHBOARD_DIR = str(_serena_pkg_path / "resources" / "dashboard") SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons") -DEFAULT_ENCODING = "utf-8" +DEFAULT_SOURCE_FILE_ENCODING = "utf-8" +"""The default encoding assumed for project source files.""" DEFAULT_CONTEXT = "desktop-app" DEFAULT_MODES = ("interactive", "editing") +SERENA_FILE_ENCODING = "utf-8" +"""The encoding used for Serena's own files, such as configuration files and memories.""" + PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.yml") SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml") diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 3eea8ba6..d87e9ad6 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -9,7 +9,7 @@ from typing import Any, Self from joblib import Parallel, delayed -from serena.constants import DEFAULT_ENCODING +from serena.constants import DEFAULT_SOURCE_FILE_ENCODING log = logging.getLogger(__name__) @@ -239,7 +239,7 @@ def search_text( def default_file_reader(file_path: str) -> str: """Reads using the default encoding.""" - with open(file_path, encoding=DEFAULT_ENCODING) as f: + with open(file_path, encoding=DEFAULT_SOURCE_FILE_ENCODING) as f: return f.read() diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index da6d7cb3..900a04a0 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -13,6 +13,7 @@ class WriteMemoryTool(Tool): Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format. The memory name should be meaningful. """ + # NOTE: utf-8 encoding is configured in the MemoriesManager if max_answer_chars == -1: max_answer_chars = self.agent.serena_config.default_max_tool_answer_chars if len(content) > max_answer_chars: diff --git a/src/serena/util/general.py b/src/serena/util/general.py index b350ae18..4a1397e2 100644 --- a/src/serena/util/general.py +++ b/src/serena/util/general.py @@ -4,6 +4,8 @@ from typing import Literal, overload from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap +from serena.constants import SERENA_FILE_ENCODING + def _create_YAML(preserve_comments: bool = False) -> YAML: """ @@ -20,7 +22,7 @@ def load_yaml(path: str, preserve_comments: Literal[False]) -> dict: ... @overload def load_yaml(path: str, preserve_comments: Literal[True]) -> CommentedMap: ... def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap: - with open(path, encoding="utf-8") as f: + with open(path, encoding=SERENA_FILE_ENCODING) as f: yaml = _create_YAML(preserve_comments) return yaml.load(f) @@ -28,5 +30,5 @@ def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = False) -> None: yaml = _create_YAML(preserve_comments) os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w", encoding="utf-8") as f: + with open(path, "w", encoding=SERENA_FILE_ENCODING) as f: yaml.dump(data, f)