fix: normalize line endings in read_file's charset_normalizer fallback (#1706)

FileUtils.read_file has two return paths with different newline semantics.
The primary path, open(file_path, encoding=encoding).read(), uses the default
newline=None and therefore applies universal newlines, so CR never reaches
callers. The charset_normalizer fallback decodes the raw bytes instead
(match.raw.decode(match.encoding)), which performs no newline translation, so
any file that fails to decode with the project's configured encoding re-enters
Serena carrying its CR characters.

Normalize to LF at the fallback's return, matching the primary path.

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 Status of the `main` branch. Changes prior to the next official version change will appear here.
 
+* General:
+  - Fix: `FileUtils.read_file`'s `charset_normalizer` fallback (used when a file cannot be decoded with
+    the project's configured `encoding`) decoded the raw bytes directly and therefore skipped the
+    universal-newline translation that the primary read path applies. CR characters from disk thus
+    reached Serena's in-memory file contents, where the rest of the code assumes LF-normalized text and
+    the `line_ending` setting is meant to be the single point of line-ending translation on write. The
+    fallback now normalizes line endings to LF, consistently with the primary path.
+
 # v1.6.0 (2026-07-16)
 
 * General:

src/solidlsp/ls_utils.py

@@ -387,6 +387,9 @@ class FileUtils:
         Reads the file at the given path using the given encoding and returns the contents as a string.
         If decoding fails, tries to detect the encoding using charset_normalizer.
 
+        Line endings are normalized to LF (universal newlines), irrespective of the encoding
+        used to decode the file.
+
         Raises FileNotFoundError if the file does not exist.
         """
         if not os.path.exists(file_path):
@@ -403,7 +406,10 @@ class FileUtils:
                     log.warning(
                         f"Could not decode {file_path} with encoding='{encoding}'; using best match '{match.encoding}' instead",
                     )
-                    return match.raw.decode(match.encoding)
+                    # Decoding the raw bytes bypasses the universal-newline translation that the
+                    # open() call above applies, so normalize explicitly to keep both paths equivalent.
+                    decoded = match.raw.decode(match.encoding)
+                    return decoded.replace("\r\n", "\n").replace("\r", "\n")
                 raise ude
         except Exception as exc:
             log.error(f"Failed to read '{file_path}' with encoding '{encoding}': {exc}")

test/solidlsp/util/test_ls_utils.py

@@ -4,6 +4,8 @@ import hashlib
 from pathlib import Path
 from unittest.mock import patch
 
+import pytest
+
 from solidlsp.ls_utils import FileUtils
 
 
@@ -40,3 +42,65 @@ def test_download_file_verified_writes_decoded_response_body(tmp_path: Path) ->
         )
 
     assert target_path.read_bytes() == payload
+
+
+# A file that cannot be decoded with the project encoding, forcing read_file's
+# charset_normalizer fallback. The accented characters make the bytes invalid UTF-8,
+# and the content is long enough for encoding detection to be reliable.
+_CP1252_LINES = [
+    "# -*- coding: cp1252 -*-",
+    "# Author: José Fernández",
+    "# Copyright (c) 2019 Müller & Söhne GmbH.",
+    "",
+    "import os",
+    "",
+    "",
+    "class ConfiguracionBasica:",
+    '    """Clase de configuración para el módulo de facturación."""',
+    "",
+    "    def __init__(self, nombre, valor=None):",
+    "        self.nombre = nombre",
+    "        self.valor = valor",
+    "",
+    "    def describir(self):",
+    '        return f"{self.nombre}: {self.valor}"',
+]
+
+
+def test_read_file_fallback_normalizes_crlf(tmp_path: Path) -> None:
+    """The charset_normalizer fallback should apply universal newlines, just like the primary path."""
+    file_path = tmp_path / "config_cp1252.py"
+    file_path.write_bytes(("\r\n".join(_CP1252_LINES) + "\r\n").encode("cp1252"))
+
+    # guard against a vacuous test: the fixture must actually force the fallback
+    with pytest.raises(UnicodeDecodeError):
+        file_path.read_text(encoding="utf-8")
+
+    content = FileUtils.read_file(str(file_path), "utf-8")
+
+    assert "José Fernández" in content, "fallback should decode the file as cp1252"
+    assert "\r" not in content
+    assert content.splitlines() == _CP1252_LINES
+
+
+def test_read_file_fallback_normalizes_lone_cr(tmp_path: Path) -> None:
+    """Old-style lone CR separators should be normalized by the fallback as well."""
+    file_path = tmp_path / "lone_cr_cp1252.py"
+    file_path.write_bytes(("\r".join(_CP1252_LINES) + "\r").encode("cp1252"))
+
+    content = FileUtils.read_file(str(file_path), "utf-8")
+
+    assert "\r" not in content
+    assert content.splitlines() == _CP1252_LINES
+
+
+def test_read_file_primary_path_normalizes_crlf(tmp_path: Path) -> None:
+    """Control: the primary open() path already normalizes; both paths must agree."""
+    lines = ["import os", "", "def f():", "    return 1"]
+    file_path = tmp_path / "config_utf8.py"
+    file_path.write_bytes(("\r\n".join(lines) + "\r\n").encode("utf-8"))
+
+    content = FileUtils.read_file(str(file_path), "utf-8")
+
+    assert "\r" not in content
+    assert content.splitlines() == lines