Allow non-ASCII characters in JSON responses

escaping only required characters

Introduce method Tool._to_json to unify JSON handling

Resolves #785

src/serena/tools/file_tools.py

@@ -6,7 +6,6 @@ File and file system-related tools, specifically for
   * editing at the file level
 """
 
-import json
 import os
 import re
 from collections import defaultdict
@@ -79,7 +78,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit):
         answer = f"File created: {relative_path}."
         if will_overwrite_existing:
             answer += " Overwrote existing file."
-        return json.dumps(answer)
+        return answer
 
 
 class ListDirTool(Tool):
@@ -106,7 +105,7 @@ class ListDirTool(Tool):
                 "project_root": self.get_project_root(),
                 "hint": "Check if the path is correct relative to the project root",
             }
-            return json.dumps(error_info)
+            return self._to_json(error_info)
 
         self.project.validate_relative_path(relative_path, require_not_ignored=skip_ignored_files)
 
@@ -118,7 +117,7 @@ class ListDirTool(Tool):
             is_ignored_file=self.project.is_ignored_path if skip_ignored_files else None,
         )
 
-        result = json.dumps({"dirs": dirs, "files": files})
+        result = self._to_json({"dirs": dirs, "files": files})
         return self._limit_length(result, max_answer_chars)
 
 
@@ -154,7 +153,7 @@ class FindFileTool(Tool):
             relative_to=self.get_project_root(),
         )
 
-        result = json.dumps({"files": files})
+        result = self._to_json({"files": files})
         return result
 
 
@@ -440,5 +439,5 @@ class SearchForPatternTool(Tool):
         for match in matches:
             assert match.source_file_path is not None
             file_to_matches[match.source_file_path].append(match.to_display_string())
-        result = json.dumps(file_to_matches)
+        result = self._to_json(file_to_matches)
         return self._limit_length(result, max_answer_chars)

src/serena/tools/jetbrains_tools.py

@@ -1,5 +1,3 @@
-import json
-
 from serena.tools import Tool, ToolMarkerOptional, ToolMarkerSymbolicRead
 from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient
 
@@ -55,7 +53,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
                 depth=depth,
                 include_body=include_body,
             )
-            result = json.dumps(response_dict)
+            result = self._to_json(response_dict)
         return self._limit_length(result, max_answer_chars)
 
 
@@ -86,7 +84,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark
                 name_path=name_path,
                 relative_path=relative_path,
             )
-            result = json.dumps(response_dict)
+            result = self._to_json(response_dict)
         return self._limit_length(result, max_answer_chars)
 
 
@@ -116,5 +114,5 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp
             response_dict = client.get_symbols_overview(
                 relative_path=relative_path,
             )
-            result = json.dumps(response_dict)
+            result = self._to_json(response_dict)
         return self._limit_length(result, max_answer_chars)

src/serena/tools/memory_tools.py

@@ -1,4 +1,3 @@
-import json
 from typing import Literal
 
 from serena.tools import ReplaceContentTool, Tool
@@ -50,7 +49,7 @@ class ListMemoriesTool(Tool):
         """
         List available memories. Any memory can be read using the `read_memory` tool.
         """
-        return json.dumps(self.memories_manager.list_memories())
+        return self._to_json(self.memories_manager.list_memories())
 
 
 class DeleteMemoryTool(Tool):

src/serena/tools/symbol_tools.py

@@ -3,7 +3,6 @@ Language server-related tools
 """
 
 import dataclasses
-import json
 import os
 from collections.abc import Sequence
 from copy import copy
@@ -73,7 +72,7 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead):
         if os.path.isdir(file_path):
             raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ")
         result = symbol_retriever.get_symbol_overview(relative_path)[relative_path]
-        result_json_str = json.dumps([dataclasses.asdict(i) for i in result])
+        result_json_str = self._to_json([dataclasses.asdict(i) for i in result])
         return self._limit_length(result_json_str, max_answer_chars)
 
 
@@ -145,7 +144,7 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead):
             within_relative_path=relative_path,
         )
         symbol_dicts = [_sanitize_symbol_dict(s.to_dict(kind=True, location=True, depth=depth, include_body=include_body)) for s in symbols]
-        result = json.dumps(symbol_dicts)
+        result = self._to_json(symbol_dicts)
         return self._limit_length(result, max_answer_chars)
 
 
@@ -198,7 +197,7 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead):
                 )
                 ref_dict["content_around_reference"] = content_around_ref.to_display_string()
             reference_dicts.append(ref_dict)
-        result = json.dumps(reference_dicts)
+        result = self._to_json(reference_dicts)
         return self._limit_length(result, max_answer_chars)
 
 

src/serena/tools/tools_base.py

@@ -1,4 +1,5 @@
 import inspect
+import json
 import os
 from abc import ABC
 from collections.abc import Iterable
@@ -295,6 +296,10 @@ class Tool(Component):
             log.error(msg)
             return msg
 
+    @staticmethod
+    def _to_json(x: Any) -> str:
+        return json.dumps(x, ensure_ascii=False)
+
 
 class EditedFileContext:
     """