feat: add CPython-parity Unicode codecs and locale

crates/weavepy-cli/src/main.rs

@@ -527,7 +527,11 @@ fn run_on_large_stack(entry: fn() -> ExitCode) -> ExitCode {
 fn main_dispatch() -> ExitCode {
     init_tracing();
 
-    let raw: Vec<String> = env::args().collect();
+    // `env::args()` panics on non-UTF-8 argv (bpo-35883's exact repro);
+    // decode PEP 383-style instead, carrying undecodable bytes in the
+    // PUA bridge window that `Interpreter::set_argv` maps back to
+    // lone surrogates (RFC 0050).
+    let raw: Vec<String> = weavepy::vm::os_args_bridged();
 
     // Multiprocessing spawn-child entry point. The parent passes
     // `--multiprocessing-fork` and an optional payload fd via
@@ -676,7 +680,7 @@ fn split_argv(raw: Vec<String>) -> (Vec<String>, Option<(&'static str, String)>,
 }
 
 fn real_main() -> Result<ExitCode> {
-    let raw: Vec<String> = env::args().collect();
+    let raw: Vec<String> = weavepy::vm::os_args_bridged();
     let (wp_argv, mode, child_argv) = split_argv(raw);
     // Re-parse the WeavePy-only slice with clap.
     let mut cli = Cli::parse_from(wp_argv);
@@ -685,7 +689,10 @@ fn real_main() -> Result<ExitCode> {
     match &mode {
         Some(("c", cmd)) => cli.command = Some(cmd.clone()),
         Some(("m", m)) => cli.module = Some(m.clone()),
-        Some(("s", path)) => cli.script = Some(PathBuf::from(path)),
+        // A script path may carry PEP 383-escaped bytes (PUA-bridged by
+        // `os_args_bridged`); recover the OS-level bytes so the file
+        // actually opens (RFC 0050).
+        Some(("s", path)) => cli.script = Some(bridged_arg_to_pathbuf(path)),
         Some(("-", _)) => cli.script = Some(PathBuf::from("-")),
         _ => {}
     }
@@ -821,6 +828,7 @@ fn build_flags(cli: &Cli, env: &EnvOverrides) -> InterpreterFlags {
         hash_seed: env.hash_seed,
         io_encoding: env.io_encoding.clone(),
         io_errors: env.io_errors.clone(),
+        utf8_mode: env.utf8_mode,
     };
     if cli.optimize == 0 && env.optimize > 0 {
         flags.optimize = env.optimize;
@@ -848,6 +856,9 @@ struct EnvOverrides {
     /// part may be empty (`:errors` sets only the handler).
     io_encoding: Option<String>,
     io_errors: Option<String>,
+    /// `PYTHONUTF8=0|1` (PEP 540). `None` when unset/empty; an invalid
+    /// value is a startup fatal error (CPython `config_init_utf8_mode`).
+    utf8_mode: Option<u8>,
 }
 
 impl EnvOverrides {
@@ -912,6 +923,23 @@ impl EnvOverrides {
                 }
             }
         }
+        // `PYTHONUTF8` (PEP 540): "1" enables UTF-8 mode, "0" disables it,
+        // empty means unset; anything else is a startup fatal error
+        // (CPython's `config_init_utf8_mode`).
+        if let Ok(v) = env::var("PYTHONUTF8") {
+            match v.as_str() {
+                "" => {}
+                "1" => o.utf8_mode = Some(1),
+                "0" => o.utf8_mode = Some(0),
+                other => {
+                    eprintln!(
+                        "Fatal Python error: init_utf8_mode: invalid PYTHONUTF8 environment \
+                         variable value '{other}'"
+                    );
+                    std::process::exit(1);
+                }
+            }
+        }
         o
     }
 
@@ -920,6 +948,22 @@ impl EnvOverrides {
     }
 }
 
+/// Rebuild a filesystem path from a (possibly PUA-bridged) argv string,
+/// recovering the original OS bytes for PEP 383-escaped names.
+fn bridged_arg_to_pathbuf(arg: &str) -> PathBuf {
+    #[cfg(unix)]
+    {
+        use std::os::unix::ffi::OsStringExt;
+        PathBuf::from(std::ffi::OsString::from_vec(
+            weavepy::vm::bridged_arg_bytes(arg),
+        ))
+    }
+    #[cfg(not(unix))]
+    {
+        PathBuf::from(arg)
+    }
+}
+
 /// Escape a string into a Python single-quoted string literal.
 fn quote_py_string(s: &str) -> String {
     let mut out = String::with_capacity(s.len() + 2);

crates/weavepy-compiler/src/cpython_code.rs

@@ -399,6 +399,53 @@ pub struct CpythonCode {
     pub inst_offsets: Vec<u32>,
 }
 
+/// Memoised [`CodeObject::to_cpython`] output. The encoding is pure —
+/// it depends only on the (immutable-after-compile) code object — but
+/// hot paths (`f_lasti`, `co_lines()` in trace functions) call it per
+/// event, and a full re-encode of a large code object costs
+/// milliseconds. Interior-mutable under the same GIL invariant as
+/// [`crate::bytecode::CacheSlot`] so the fill can happen through a
+/// shared `&CodeObject` (`Arc<CodeObject>` crosses thread boundaries).
+#[derive(Default)]
+pub struct CpCache {
+    inner: std::cell::UnsafeCell<Option<std::sync::Arc<CpythonCode>>>,
+}
+
+// SAFETY: all reads/writes happen under the VM's GIL invariant — see
+// `CacheSlot` in bytecode.rs for the full justification.
+unsafe impl Send for CpCache {}
+unsafe impl Sync for CpCache {}
+
+impl CpCache {
+    fn get_or_init(&self, init: impl FnOnce() -> CpythonCode) -> std::sync::Arc<CpythonCode> {
+        // SAFETY: the GIL invariant guarantees no concurrent access.
+        let slot = unsafe { &mut *self.inner.get() };
+        slot.get_or_insert_with(|| std::sync::Arc::new(init()))
+            .clone()
+    }
+}
+
+impl std::fmt::Debug for CpCache {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("CpCache")
+    }
+}
+
+/// A clone starts cold: the copy may be mutated (e.g. `code.replace`)
+/// before it is next encoded.
+impl Clone for CpCache {
+    fn clone(&self) -> Self {
+        Self::default()
+    }
+}
+
+/// The cache never affects code-object identity.
+impl PartialEq for CpCache {
+    fn eq(&self, _: &Self) -> bool {
+        true
+    }
+}
+
 const CO_FAST_LOCAL: u8 = 0x20;
 const CO_FAST_CELL: u8 = 0x40;
 const CO_FAST_FREE: u8 = 0x80;
@@ -1127,9 +1174,11 @@ fn map_from_cpython(cp_op: u8, arg: u32, nlocals: u32) -> Option<(OpCode, u32)>
 
 impl CodeObject {
     /// The CPython-3.13 wire view of this code object (RFC 0033).
+    /// Encoded once per code object and memoised (the encoding is hot:
+    /// trace functions hit `f_lasti` / `co_lines()` per event).
     #[must_use]
-    pub fn to_cpython(&self) -> CpythonCode {
-        encode(self)
+    pub fn to_cpython(&self) -> std::sync::Arc<CpythonCode> {
+        self.cp_cache.get_or_init(|| encode(self))
     }
 
     /// Translate a WeavePy instruction index into the `co_code` byte offset

crates/weavepy-compiler/src/lib.rs

@@ -175,6 +175,9 @@ pub struct CodeObject {
     /// coroutine. Never set by the compiler — only by the runtime
     /// marking helper and marshal round-trips.
     pub is_iterable_coroutine: bool,
+    /// Memoised [`Self::to_cpython`] encoding (never compared, resets
+    /// on clone).
+    pub cp_cache: cpython_code::CpCache,
 }
 
 /// A per-instruction source-column span (PEP-657). `col`/`end_col` are
@@ -2956,6 +2959,11 @@ impl Compiler {
             }
             return Ok(());
         }
+        // PEP 626: the `try:` line is "executed" and must fire a line
+        // event even though it compiles to nothing (CPython emits a NOP
+        // carrying the statement's location; trace consumers — and
+        // test_sys_settrace's relative-line bookkeeping — count on it).
+        self.emit(OpCode::Nop, 0);
         // PEP 654 static check, before anything else compiles so the
         // `except*` jump error wins over e.g. a `return` in a module-
         // level `finally` (matching CPython's reporting order).

crates/weavepy-parser/src/lib.rs

@@ -19,6 +19,7 @@ mod parser;
 
 pub use ast::{dump_module, Module};
 pub use error::ParseError;
+pub use parser::{set_unicode_name_resolver, UnicodeNameResolution};
 pub use weavepy_lexer::EscapeWarning;
 
 /// Parse a Python source buffer into a [`Module`].

crates/weavepy-parser/src/parser.rs

@@ -4915,6 +4915,40 @@ fn cps_to_constant(cps: Vec<u32>) -> Constant {
     }
 }
 
+/// `\N{NAME}` resolution outcome supplied by the embedding VM (RFC 0050
+/// WS4): the runtime resolves names against its generated UCD tables and
+/// honours a poisoned `sys.modules['unicodedata']` the way CPython's
+/// tokenizer does.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum UnicodeNameResolution {
+    /// Resolved to this code point.
+    Code(u32),
+    /// No character has this name.
+    Unknown,
+    /// The unicodedata machinery can't be loaded (CPython: the import of
+    /// `unicodedata` failed, e.g. `sys.modules['unicodedata'] = None`).
+    Unavailable,
+}
+
+static UNICODE_NAME_RESOLVER: std::sync::OnceLock<fn(&str) -> UnicodeNameResolution> =
+    std::sync::OnceLock::new();
+
+/// Install the VM-side `\N{NAME}` resolver. First installation wins;
+/// standalone parser users fall back to the `unicode_names2` table.
+pub fn set_unicode_name_resolver(f: fn(&str) -> UnicodeNameResolution) {
+    let _ = UNICODE_NAME_RESOLVER.set(f);
+}
+
+fn resolve_unicode_name(name: &str) -> UnicodeNameResolution {
+    match UNICODE_NAME_RESOLVER.get() {
+        Some(f) => f(name),
+        None => match unicode_names2::character(name) {
+            Some(ch) => UnicodeNameResolution::Code(ch as u32),
+            None => UnicodeNameResolution::Unknown,
+        },
+    }
+}
+
 fn decode_str_body(s: &str, raw: bool) -> Result<Constant, String> {
     if raw {
         return Ok(Constant::Str(s.to_owned()));
@@ -5032,10 +5066,24 @@ fn decode_str_body(s: &str, raw: bool) -> Result<Constant, String> {
                 loop {
                     match chars.next() {
                         Some((i, '}')) => {
-                            let ch = unicode_names2::character(&name).ok_or_else(|| {
-                                unicode_err(bs, i + 1, "unknown Unicode character name")
-                            })?;
-                            out.push(ch as u32);
+                            match resolve_unicode_name(&name) {
+                                UnicodeNameResolution::Code(cp) => out.push(cp),
+                                UnicodeNameResolution::Unknown => {
+                                    return Err(unicode_err(
+                                        bs,
+                                        i + 1,
+                                        "unknown Unicode character name",
+                                    ))
+                                }
+                                // CPython's tokenizer message when importing
+                                // `unicodedata` fails — *not* wrapped in the
+                                // unicodeescape-codec position text.
+                                UnicodeNameResolution::Unavailable => {
+                                    return Err("(unicode error) \\N escapes not supported \
+                                                (can't load unicodedata module)"
+                                        .to_owned())
+                                }
+                            }
                             break;
                         }
                         Some((_, c)) => name.push(c),

crates/weavepy-vm/Cargo.toml

@@ -53,9 +53,6 @@ rust_decimal     = { workspace = true }
 quick-xml        = { workspace = true }
 
 # RFC 0023 — drop-in stdlib parity + HTTPS.
-unicode-normalization = { workspace = true }
-unicode-properties    = { workspace = true }
-unicode_names2        = { workspace = true }
 memmap2               = { workspace = true }
 memchr                = { workspace = true }
 libc                  = { workspace = true }

crates/weavepy-vm/src/builtin_types.rs

@@ -2535,6 +2535,53 @@ fn install_unicode_error_dunders(ty: &Rc<TypeObject>, kind: UnicodeErrorKind) {
                 rest.len()
             )));
         }
+        // CPython parses with `PyArg_ParseTuple("UUnnU" / "UOnnU" / "UnnU")`:
+        // wrong-typed arguments raise TypeError at construction.
+        let is_str = |o: &Object| {
+            matches!(o, Object::Str(_) | Object::WStr(_))
+                || matches!(o, Object::Instance(i)
+                    if i.cls().mro.borrow().iter().any(|t| t.name == "str"))
+        };
+        let is_index = |o: &Object| matches!(o, Object::Int(_) | Object::Bool(_));
+        let is_buffer = |o: &Object| {
+            matches!(
+                o,
+                Object::Bytes(_) | Object::ByteArray(_) | Object::MemoryView(_)
+            )
+        };
+        let check = |ok: bool, pos: usize, expect: &str, got: &Object| {
+            if ok {
+                Ok(())
+            } else {
+                Err(crate::error::type_error(format!(
+                    "argument {} must be {expect}, not {}",
+                    pos + 1,
+                    got.type_name_owned()
+                )))
+            }
+        };
+        match kind {
+            UnicodeErrorKind::Encode => {
+                check(is_str(&rest[0]), 0, "str", &rest[0])?;
+                check(is_str(&rest[1]), 1, "str", &rest[1])?;
+                check(is_index(&rest[2]), 2, "int", &rest[2])?;
+                check(is_index(&rest[3]), 3, "int", &rest[3])?;
+                check(is_str(&rest[4]), 4, "str", &rest[4])?;
+            }
+            UnicodeErrorKind::Decode => {
+                check(is_str(&rest[0]), 0, "str", &rest[0])?;
+                check(is_buffer(&rest[1]), 1, "a bytes-like object", &rest[1])?;
+                check(is_index(&rest[2]), 2, "int", &rest[2])?;
+                check(is_index(&rest[3]), 3, "int", &rest[3])?;
+                check(is_str(&rest[4]), 4, "str", &rest[4])?;
+            }
+            UnicodeErrorKind::Translate => {
+                check(is_str(&rest[0]), 0, "str", &rest[0])?;
+                check(is_index(&rest[1]), 1, "int", &rest[1])?;
+                check(is_index(&rest[2]), 2, "int", &rest[2])?;
+                check(is_str(&rest[3]), 3, "str", &rest[3])?;
+            }
+        }
         let mut dict = inst_rc.dict.borrow_mut();
         set(&mut dict, "args", Object::new_tuple(rest.to_vec()));
         let mut i = 0;