Enhancing file read robustness (#682)

If a file cannot be read using the project's defined encoding, 
use charset-normalizer to detect the likely encoding

src/serena/project.py

@@ -13,6 +13,7 @@ from serena.util.file_system import GitignoreParser, match_path
 from solidlsp import SolidLanguageServer
 from solidlsp.ls_config import Language, LanguageServerConfig
 from solidlsp.ls_logger import LanguageServerLogger
+from solidlsp.ls_utils import FileUtils
 from solidlsp.settings import SolidLSPSettings
 
 log = logging.getLogger(__name__)
@@ -138,9 +139,8 @@ class Project:
         :return: the content of the file
         """
         abs_path = Path(self.project_root) / relative_path
-        if not abs_path.exists():
-            raise FileNotFoundError(f"File not found: {abs_path}")
-        return abs_path.read_text(encoding=self.project_config.encoding)
+        ls_logger = LanguageServerLogger(log_level=log.level)
+        return FileUtils.read_file(ls_logger, str(abs_path), self.project_config.encoding)
 
     def get_ignore_spec(self) -> pathspec.PathSpec:
         """

src/solidlsp/ls_utils.py

@@ -13,6 +13,7 @@ import zipfile
 from enum import Enum
 from pathlib import Path, PurePath
 
+import charset_normalizer
 import requests
 
 from solidlsp.ls_exceptions import SolidLSPException
@@ -173,8 +174,19 @@ class FileUtils:
             logger.log(f"File read '{file_path}' failed: File does not exist.", logging.ERROR)
             raise SolidLSPException(f"File read '{file_path}' failed: File does not exist.")
         try:
-            with open(file_path, encoding=encoding) as inp_file:
-                return inp_file.read()
+            try:
+                with open(file_path, encoding=encoding) as inp_file:
+                    return inp_file.read()
+            except UnicodeDecodeError as ude:
+                results = charset_normalizer.from_path(file_path)
+                match = results.best()
+                if match:
+                    logger.log(
+                        f"Could not decode {file_path} with encoding='{encoding}'; using best match '{match.encoding}' instead",
+                        logging.WARNING,
+                    )
+                    return match.raw.decode(match.encoding)
+                raise ude
         except Exception as exc:
             logger.log(f"File read '{file_path}' failed to read with encoding '{encoding}': {exc}", logging.ERROR)
             raise SolidLSPException("File read failed.") from None