Read and write text files as explicit UTF-8

FileManager and LspService opened files with the locale default
encoding. On Windows that is cp1252, so a source file containing
characters outside it (e.g. U+201D or an emoji) raised
UnicodeDecodeError on read, and writes produced non-UTF-8 bytes. Linux
and macOS hide this behind UTF-8 locales, which is why #46 and #47
showed up only on the Windows CI.

- Pass encoding="utf-8" in FileManager.get_content/save_file and in the
  LspService file-open read, and document the UTF-8 contract on
  IFileManager.
- Add regression tests using text outside cp1252, so a locale-decoded
  read raises rather than silently producing mojibake.
- Record the convention in developing-finecode.md, with the env vars
  that reproduce a non-UTF-8 locale. PLW1514 is preview-only and misses
  method references passed as callables, so enforcement stays manual.

Author: Aksem

docs/guides/developing-finecode.md

@@ -580,6 +580,14 @@ The cap also has no remaining job now that the interpreter matrix exists. The se
 
 If a genuinely newer Python breaks a package, fix it when that version exists — do not pre-emptively cap. New packages must follow this: declare `requires-python = ">=<min>"` with no upper component.
 
+## Text file IO: explicit UTF-8
+
+Every text-mode `open`, `Path.read_text`, `Path.write_text` and `NamedTemporaryFile` passes `encoding="utf-8"`. This includes callables passed by reference rather than called inline, e.g. `asyncio.to_thread(p.write_text, s, encoding="utf-8")`.
+
+The default is `locale.getencoding()`, which on Windows is the ANSI code page (cp1252) and stays that way until Python 3.15 (PEP 686). FineCode supports 3.11+, so the locale default cannot be assumed to be UTF-8. Linux and macOS dev machines never see the failure — their UTF-8 locales hide it — but #46 and #47 were exactly that: Windows-only `UnicodeDecodeError`s on the source-file read path.
+
+To reproduce a non-UTF-8 locale locally, run with `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0`. Enforcement is not automated yet: `PLW1514` (`unspecified-encoding`) is preview-only and misses method references passed as callables, so it cannot guard the convention on its own.
+
 ## JSON-RPC key naming convention
 
 All JSON-RPC channels in FineCode use **camelCase** for message keys:

finecode_extension_api/src/finecode_extension_api/contrib/lsp_service.py

@@ -1199,7 +1199,7 @@ async def _forward_file_event(self, event: ifileeditor.FileEvent) -> None:
             async with self._get_uri_lock(uri):
                 if uri not in self._open_documents:
                     try:
-                        content = event.file_path.read_text()
+                        content = event.file_path.read_text(encoding="utf-8")
                     except OSError:
                         return
                     lsp_version = self._next_version(uri)

finecode_extension_api/src/finecode_extension_api/interfaces/ifilemanager.py

@@ -44,6 +44,9 @@ class IFileManager(Protocol):
     async def get_content(self, file_path: Path) -> str:
         """Return the content of ``file_path``.
 
+        The file is decoded as UTF-8; content that is not valid UTF-8 raises
+        ``UnicodeDecodeError``.
+
         Raises:
             FileNotFound: ``file_path`` does not exist.
         """
@@ -88,7 +91,9 @@ async def rename_file(
         """
         ...
 
-    async def save_file(self, file_path: Path, file_content: str) -> None: ...
+    async def save_file(self, file_path: Path, file_content: str) -> None:
+        """Write ``file_content`` encoded as UTF-8."""
+        ...
 
     async def create_dir(
         self, dir_path: Path, create_parents: bool = True, exist_ok: bool = True

finecode_extension_api/tests/test_lsp_service.py

@@ -12,6 +12,11 @@
 from finecode_extension_api.contrib.lsp_service import LspService, _FileChangeType
 from finecode_extension_api.interfaces import ifileeditor, ilspclient
 
+# Em dash, right double quote, é and 😀 — the right double quote (U+201D) and
+# the emoji are outside cp1252, so a locale-decoded (cp1252 on Windows) read
+# raises instead of rounding them into mojibake.
+_NON_ASCII = "x = '— ” é \U0001f600'"
+
 
 @dataclasses.dataclass
 class _SentNotification:
@@ -406,6 +411,26 @@ async def test_file_open_event_and_hover_race_do_not_double_sync(
         assert session.sync_notification_count(file_path.as_uri()) == 1
 
 
+async def test_file_open_event_sends_utf8_file_content_in_did_open(
+    tmp_path: Path,
+) -> None:
+    """UTF-8 file content must reach the server's `textDocument/didOpen` even
+    where the default text encoding is the locale's, cp1252 on Windows."""
+    file_path = tmp_path / "subject.py"
+    file_path.write_bytes(_NON_ASCII.encode("utf-8"))
+
+    async with _running_service(file_path, _NON_ASCII) as (service, session, _):
+        await service._handle_file_event(
+            ifileeditor.FileOpenEvent(file_path=file_path)
+        )
+
+    did_open = [
+        n for n in session.notifications if n.method == "textDocument/didOpen"
+    ]
+    assert len(did_open) == 1
+    assert did_open[0].params["textDocument"]["text"] == _NON_ASCII
+
+
 class _FakeTransportError(Exception):
     """Duck-typed shape of a transport-level JSON-RPC error carrying a code,
     without depending on finecode_jsonrpc (which this package must not

finecode_extension_runner/src/finecode_extension_runner/impls/file_manager.py

@@ -68,7 +68,7 @@ async def rename_file(
 
     async def save_file(self, file_path: Path, file_content: str) -> None:
         self.logger.debug(f"Save file {file_path}")
-        await asyncio.to_thread(file_path.write_text, file_content)
+        await asyncio.to_thread(file_path.write_text, file_content, encoding="utf-8")
 
     async def create_dir(
         self, dir_path: Path, create_parents: bool = True, exist_ok: bool = True
@@ -137,7 +137,7 @@ def _chmod_writable(self, path: Path) -> None:
     def read_content_file_from_fs(self, file_path: Path) -> str:
         # don't use this method directly, use `get_content` instead
         self.logger.debug(f"Read file: {file_path}")
-        with open(file_path) as f:
+        with open(file_path, encoding="utf-8") as f:
             file_content = f.read()
 
         return file_content

finecode_extension_runner/tests/test_file_manager.py

@@ -10,6 +10,13 @@
 from finecode_extension_runner.impls.file_manager import FileManager
 
 
+# Em dash, right double quote, é and 😀 — no newline, so the byte assertions
+# below do not depend on Windows \n→\r\n translation. The right double quote
+# (U+201D) and 😀 are outside cp1252, so a cp1252 read raises instead of
+# mojibake, while — and é would round-trip as wrong-but-non-raising bytes.
+_NON_ASCII = "x = '— ” é \U0001f600'"
+
+
 def _make_dir_tree(root: pathlib.Path) -> pathlib.Path:
     nested = root / "sub"
     nested.mkdir(parents=True)
@@ -191,3 +198,26 @@ async def test_delete_file_on_a_directory_raises(
         await FileManager(logger=logger).delete_file(directory)
 
     assert directory.exists()
+
+
+async def test_get_content_decodes_utf8_regardless_of_locale(
+    tmp_path: pathlib.Path,
+) -> None:
+    """UTF-8 content must decode even where the default text encoding is the
+    locale's, cp1252 on Windows."""
+    path = tmp_path / "subject.py"
+    path.write_bytes(_NON_ASCII.encode("utf-8"))
+
+    assert await FileManager(logger=logger).get_content(path) == _NON_ASCII
+
+
+async def test_save_file_encodes_utf8_regardless_of_locale(
+    tmp_path: pathlib.Path,
+) -> None:
+    """Content must be written as UTF-8 even where the default text encoding
+    is the locale's, cp1252 on Windows."""
+    path = tmp_path / "subject.py"
+
+    await FileManager(logger=logger).save_file(path, _NON_ASCII)
+
+    assert path.read_bytes() == _NON_ASCII.encode("utf-8")