diff --git a/crates/weavepy-cli/src/repl.rs b/crates/weavepy-cli/src/repl.rs index c36961d..ff96505 100644 --- a/crates/weavepy-cli/src/repl.rs +++ b/crates/weavepy-cli/src/repl.rs @@ -44,6 +44,10 @@ pub(crate) struct Repl { main_module: Rc, history_path: Option, quiet: bool, + /// `CO_FUTURE_*` bits accumulated from executed inputs, so a + /// `from __future__ import …` typed at one prompt affects every + /// later prompt (CPython's `codeop.Compile` behaviour). + future_flags: u32, } impl Repl { @@ -73,6 +77,7 @@ impl Repl { main_module, history_path, quiet, + future_flags: 0, }) } @@ -184,7 +189,9 @@ impl Repl { // A "single expression" candidate is one parse-able as // `Module(body=[Expr(value=…)])`. Anything else (statements, // multiple expressions, blocks) bails to the suite path. - let module = parser::parse_module(trimmed).map_err(|_| ())?; + let module = parser::parse_module_with_warnings_flags(trimmed, self.flufl_active()) + .0 + .map_err(|_| ())?; if module.body.len() != 1 { return Err(()); } @@ -192,8 +199,14 @@ impl Repl { if !is_expr { return Err(()); } - let code = - compiler::compile_module_with_source(&module, trimmed, "").map_err(|_| ())?; + let code = compiler::compile_module_with_options( + &module, + trimmed, + "", + self.compile_options(), + ) + .map_err(|_| ())?; + self.future_flags |= code.future_flags; let globals = self.main_module.dict.clone(); let result = self .interpreter @@ -216,16 +229,34 @@ impl Repl { } fn execute_once(&mut self, source: &str, filename: String) -> Result<(), String> { - let module = parser::parse_module(source) + let module = parser::parse_module_with_warnings_flags(source, self.flufl_active()) + .0 .map_err(|e| weavepy::Error::Parse(e).format(source, &filename))?; - let code = compiler::compile_module_with_source(&module, source, &filename) - .map_err(|e| weavepy::Error::Compile(e).format(source, &filename))?; + let code = compiler::compile_module_with_options( + &module, + source, + &filename, + self.compile_options(), + ) + .map_err(|e| weavepy::Error::Compile(e).format(source, &filename))?; + self.future_flags |= code.future_flags; let globals = self.main_module.dict.clone(); self.interpreter .exec_module_in(&code, globals) .map(|_| ()) .map_err(|e| weavepy::Error::Runtime(e).format(source, &filename)) } + + fn flufl_active(&self) -> bool { + self.future_flags & compiler::flags::CO_FUTURE_BARRY_AS_BDFL != 0 + } + + fn compile_options(&self) -> compiler::CompileOptions { + compiler::CompileOptions { + flags: self.future_flags, + ..Default::default() + } + } } fn build_main_module(interpreter: &Interpreter) -> Rc { @@ -277,6 +308,7 @@ fn needs_continuation(source: &str) -> bool { span.end.0 as usize >= source.len().saturating_sub(1) } Err(parser::ParseError::Lex(lexer::LexError::UnterminatedString { .. })) => true, + Err(parser::ParseError::Lex(lexer::LexError::UnterminatedTripleString { .. })) => true, // An unterminated (possibly triple-quoted) f-string literal is the // multi-line-continuation case too — the user is still typing it. // (`FstringExpectingBrace`/`...OrSpec` are real errors, not these.) diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index 98650de..615bd99 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -175,6 +175,12 @@ pub struct CodeObject { /// coroutine. Never set by the compiler — only by the runtime /// marking helper and marshal round-trips. pub is_iterable_coroutine: bool, + /// `CO_FUTURE_*` bits active for this code object (the module's + /// own `__future__` imports merged with any inherited/compile-flag + /// bits). Reported through `co_flags` so `compile(..., + /// dont_inherit=False)` can inherit the caller's futures the way + /// CPython does (RFC 0052). + pub future_flags: u32, /// Memoised [`Self::to_cpython`] encoding (never compared, resets /// on clone). pub cp_cache: cpython_code::CpCache, @@ -428,6 +434,98 @@ impl From for Constant { } } +// ---------- compile flags (CPython `Include/cpython/compile.h`) ---------- + +/// CPython compiler-flag constants. The `CO_FUTURE_*` values are the +/// exact bits `__future__.CO_*` exposes and `co_flags` carries; the +/// `PyCF_*` values are the `compile()` control bits `ast` re-exports. +pub mod flags { + pub const CO_FUTURE_DIVISION: u32 = 0x0002_0000; + pub const CO_FUTURE_ABSOLUTE_IMPORT: u32 = 0x0004_0000; + pub const CO_FUTURE_WITH_STATEMENT: u32 = 0x0008_0000; + pub const CO_FUTURE_PRINT_FUNCTION: u32 = 0x0010_0000; + pub const CO_FUTURE_UNICODE_LITERALS: u32 = 0x0020_0000; + pub const CO_FUTURE_BARRY_AS_BDFL: u32 = 0x0040_0000; + pub const CO_FUTURE_GENERATOR_STOP: u32 = 0x0080_0000; + pub const CO_FUTURE_ANNOTATIONS: u32 = 0x0100_0000; + + /// All future-statement bits (CPython `PyCF_MASK`). + pub const PYCF_MASK: u32 = CO_FUTURE_DIVISION + | CO_FUTURE_ABSOLUTE_IMPORT + | CO_FUTURE_WITH_STATEMENT + | CO_FUTURE_PRINT_FUNCTION + | CO_FUTURE_UNICODE_LITERALS + | CO_FUTURE_BARRY_AS_BDFL + | CO_FUTURE_GENERATOR_STOP + | CO_FUTURE_ANNOTATIONS; + /// Formerly-meaningful bits accepted and ignored (CPython + /// `PyCF_MASK_OBSOLETE` — `CO_NESTED`). + pub const PYCF_MASK_OBSOLETE: u32 = 0x0010; + + pub const PYCF_SOURCE_IS_UTF8: u32 = 0x0100; + pub const PYCF_DONT_IMPLY_DEDENT: u32 = 0x0200; + pub const PYCF_ONLY_AST: u32 = 0x0400; + pub const PYCF_IGNORE_COOKIE: u32 = 0x0800; + pub const PYCF_TYPE_COMMENTS: u32 = 0x1000; + pub const PYCF_ALLOW_TOP_LEVEL_AWAIT: u32 = 0x2000; + pub const PYCF_ALLOW_INCOMPLETE_INPUT: u32 = 0x4000; + pub const PYCF_OPTIMIZED_AST: u32 = 0x8000 | PYCF_ONLY_AST; + + /// All non-future bits `compile()` accepts (CPython + /// `PyCF_COMPILE_MASK`). + pub const PYCF_COMPILE_MASK: u32 = PYCF_ONLY_AST + | PYCF_ALLOW_TOP_LEVEL_AWAIT + | PYCF_TYPE_COMMENTS + | PYCF_DONT_IMPLY_DEDENT + | PYCF_ALLOW_INCOMPLETE_INPUT + | PYCF_OPTIMIZED_AST; + + /// Map a `__future__` feature name to its `CO_FUTURE_*` bit. + /// Returns 0 for features that predate the flag scheme entirely + /// (there are none — every known feature has a bit). + pub fn future_feature_bit(name: &str) -> Option { + Some(match name { + "division" => CO_FUTURE_DIVISION, + "absolute_import" => CO_FUTURE_ABSOLUTE_IMPORT, + "with_statement" => CO_FUTURE_WITH_STATEMENT, + "print_function" => CO_FUTURE_PRINT_FUNCTION, + "unicode_literals" => CO_FUTURE_UNICODE_LITERALS, + "barry_as_FLUFL" => CO_FUTURE_BARRY_AS_BDFL, + "generator_stop" => CO_FUTURE_GENERATOR_STOP, + "annotations" => CO_FUTURE_ANNOTATIONS, + // `nested_scopes` / `generators` are always-on features with + // no live bit in 3.x. + "nested_scopes" | "generators" => 0, + _ => return None, + }) + } +} + +/// Options threaded from `compile()` into the compiler (RFC 0052). +#[derive(Debug, Clone, Copy, Default)] +pub struct CompileOptions { + /// `CO_FUTURE_*` bits active *before* the module's own + /// `__future__` imports are folded in (inherited from the calling + /// code or passed via `compile(flags=...)`), plus any `PyCF_*` + /// control bits (`PyCF_ALLOW_TOP_LEVEL_AWAIT` is honoured here). + pub flags: u32, + /// Resolved optimization level (0, 1 or 2 — the caller resolves + /// `-1` against the interpreter default before compiling). + pub optimize: u8, +} + +/// The per-compilation parameters shared by a top-level compiler and +/// every nested scope it spawns. +#[derive(Debug, Clone, Copy, Default)] +struct CompileParams { + future_annotations: bool, + optimize: u8, + /// Merged `CO_FUTURE_*` bits (inherited + module's own imports), + /// stamped onto every produced code object. + future_flags: u32, + allow_top_level_await: bool, +} + // ---------- public entry point ---------- /// PEP 563: does this module open with `from __future__ import annotations`? @@ -440,12 +538,178 @@ fn has_future_annotations(module: &Module) -> bool { module.body.iter().any(|stmt| { matches!( &stmt.kind, - StmtKind::ImportFrom { module: Some(m), names, .. } + StmtKind::ImportFrom { module: Some(m), names, level: 0 } if m == "__future__" && names.iter().any(|a| a.name == "annotations") ) }) } +/// The module's own `__future__` feature bits (`CO_FUTURE_*`). The +/// validator has already rejected unknown features and misplaced +/// imports by the time this runs, so a plain scan suffices. +fn module_future_flags(module: &Module) -> u32 { + let mut bits = 0u32; + for stmt in &module.body { + if let StmtKind::ImportFrom { + module: Some(m), + names, + level: 0, + } = &stmt.kind + { + if m == "__future__" { + for a in names { + bits |= flags::future_feature_bit(&a.name).unwrap_or(0); + } + } + } + } + bits +} + +thread_local! { + /// PEP 563 active for the compilation currently running on this + /// thread. Consulted by the free-variable scans + /// ([`collect_reads_stmt`]) so stringified annotations don't + /// contribute reads to scope analysis (no spurious cells for + /// `def bar(): inner: outer = 1` — test_future's + /// `test_annotations_symbol_table_pass`). + static PEP563_ACTIVE: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +fn pep563_active() -> bool { + PEP563_ACTIVE.with(std::cell::Cell::get) +} + +/// RAII guard installing the PEP 563 flag for the current compilation. +struct Pep563Guard(bool); + +impl Pep563Guard { + fn install(active: bool) -> Self { + Pep563Guard(PEP563_ACTIVE.with(|c| c.replace(active))) + } +} + +impl Drop for Pep563Guard { + fn drop(&mut self) { + PEP563_ACTIVE.with(|c| c.set(self.0)); + } +} + +/// Build the shared per-compilation parameters from caller options + +/// the module's own `__future__` imports. +fn make_params(module: &Module, opts: CompileOptions) -> CompileParams { + let future_flags = (opts.flags & flags::PYCF_MASK) | module_future_flags(module); + CompileParams { + future_annotations: has_future_annotations(module) + || future_flags & flags::CO_FUTURE_ANNOTATIONS != 0, + optimize: opts.optimize, + future_flags, + allow_top_level_await: opts.flags & flags::PYCF_ALLOW_TOP_LEVEL_AWAIT != 0, + } +} + +/// PEP 695 `type` statements arrive first-class from the parser (so +/// `ast.parse` and `symtable` observe the real node); rewrite each to +/// its lazy `__weavepy_type_alias__` assignment form *before* any +/// compiler pass (validation, mangling, scope analysis, codegen) +/// runs, so every later pass sees the same shape the parser used to +/// emit directly. Returns the module untouched (borrowed) when no +/// `type` statement exists — the common case — to avoid cloning the +/// AST. +fn lower_type_aliases(module: &Module) -> std::borrow::Cow<'_, Module> { + fn block_lists(kind: &StmtKind) -> Vec<&[Stmt]> { + match kind { + StmtKind::FunctionDef { body, .. } + | StmtKind::AsyncFunctionDef { body, .. } + | StmtKind::ClassDef { body, .. } + | StmtKind::With { body, .. } + | StmtKind::AsyncWith { body, .. } => vec![body], + StmtKind::If { body, orelse, .. } + | StmtKind::While { body, orelse, .. } + | StmtKind::For { body, orelse, .. } + | StmtKind::AsyncFor { body, orelse, .. } => vec![body, orelse], + StmtKind::Try { + body, + handlers, + orelse, + finalbody, + } => { + let mut out: Vec<&[Stmt]> = vec![body, orelse, finalbody]; + out.extend(handlers.iter().map(|h| h.body.as_slice())); + out + } + StmtKind::Match { cases, .. } => cases.iter().map(|c| c.body.as_slice()).collect(), + _ => Vec::new(), + } + } + fn contains(stmts: &[Stmt]) -> bool { + stmts.iter().any(|s| { + matches!(s.kind, StmtKind::TypeAlias { .. }) + || block_lists(&s.kind).into_iter().any(contains) + }) + } + fn rewrite(stmts: &mut [Stmt]) { + for s in stmts { + if matches!(s.kind, StmtKind::TypeAlias { .. }) { + *s = weavepy_parser::lower_type_alias_stmt(s); + continue; + } + match &mut s.kind { + StmtKind::FunctionDef { body, .. } + | StmtKind::AsyncFunctionDef { body, .. } + | StmtKind::ClassDef { body, .. } + | StmtKind::With { body, .. } + | StmtKind::AsyncWith { body, .. } => rewrite(body), + StmtKind::If { body, orelse, .. } + | StmtKind::While { body, orelse, .. } + | StmtKind::For { body, orelse, .. } + | StmtKind::AsyncFor { body, orelse, .. } => { + rewrite(body); + rewrite(orelse); + } + StmtKind::Try { + body, + handlers, + orelse, + finalbody, + } => { + rewrite(body); + rewrite(orelse); + rewrite(finalbody); + for h in handlers { + rewrite(&mut h.body); + } + } + StmtKind::Match { cases, .. } => { + for c in cases { + rewrite(&mut c.body); + } + } + _ => {} + } + } + } + if !contains(&module.body) { + return std::borrow::Cow::Borrowed(module); + } + let mut lowered = module.clone(); + rewrite(&mut lowered.body); + std::borrow::Cow::Owned(lowered) +} + +/// Run only the parse-adjacent validation pass (the symtable-stage +/// checks CPython performs while *building* the symbol table: +/// `global`/`nonlocal` directive conflicts, `__future__` placement, +/// annotation-scope restrictions, …) without generating code. +/// `_symtable.symtable()` uses this so symtable-build-time +/// `SyntaxError`s surface with CPython's messages and locations. +pub fn validate_module_only(module: &Module, source: &str) -> Result<(), CompileError> { + let module = lower_type_aliases(module); + let module = &*module; + let params = make_params(module, CompileOptions::default()); + validate::validate_module(module, source, params.future_annotations) +} + /// Compile a parsed module into a top-level [`CodeObject`]. pub fn compile_module(module: &Module) -> Result { compile_module_with_filename(module, "") @@ -467,7 +731,22 @@ pub fn compile_module_with_source( source: &str, filename: &str, ) -> Result { - validate::validate_module(module, source)?; + compile_module_with_options(module, source, filename, CompileOptions::default()) +} + +/// As [`compile_module_with_source`] with explicit `compile()` options +/// (future/`PyCF_*` flags + optimization level) — RFC 0052. +pub fn compile_module_with_options( + module: &Module, + source: &str, + filename: &str, + opts: CompileOptions, +) -> Result { + let module = lower_type_aliases(module); + let module = &*module; + let params = make_params(module, opts); + let _pep563 = Pep563Guard::install(params.future_annotations); + validate::validate_module(module, source, params.future_annotations)?; let line_index = LineIndex::new(source); let mut top = Compiler::new( "".to_owned(), @@ -475,7 +754,7 @@ pub fn compile_module_with_source( CodeKind::Module, Rc::new(line_index), Rc::from(source), - has_future_annotations(module), + params, ); top.compile_module_body(module)?; Ok(top.finish()) @@ -491,7 +770,21 @@ pub fn compile_interactive_with_source( source: &str, filename: &str, ) -> Result { - validate::validate_module(module, source)?; + compile_interactive_with_options(module, source, filename, CompileOptions::default()) +} + +/// As [`compile_interactive_with_source`] with explicit options. +pub fn compile_interactive_with_options( + module: &Module, + source: &str, + filename: &str, + opts: CompileOptions, +) -> Result { + let module = lower_type_aliases(module); + let module = &*module; + let params = make_params(module, opts); + let _pep563 = Pep563Guard::install(params.future_annotations); + validate::validate_module(module, source, params.future_annotations)?; let line_index = LineIndex::new(source); let mut top = Compiler::new( "".to_owned(), @@ -499,7 +792,7 @@ pub fn compile_interactive_with_source( CodeKind::Module, Rc::new(line_index), Rc::from(source), - has_future_annotations(module), + params, ); top.interactive = true; top.compile_module_body(module)?; @@ -514,6 +807,16 @@ pub fn compile_eval_with_source( module: &Module, source: &str, filename: &str, +) -> Result { + compile_eval_with_options(module, source, filename, CompileOptions::default()) +} + +/// As [`compile_eval_with_source`] with explicit options. +pub fn compile_eval_with_options( + module: &Module, + source: &str, + filename: &str, + opts: CompileOptions, ) -> Result { // CPython's `eval` grammar only admits a single expression; any // statement syntax (`del x`, `x = 1`, a second statement, …) is a @@ -546,7 +849,11 @@ pub fn compile_eval_with_source( span, )); } - validate::validate_module(module, source)?; + let module = lower_type_aliases(module); + let module = &*module; + let params = make_params(module, opts); + let _pep563 = Pep563Guard::install(params.future_annotations); + validate::validate_module(module, source, params.future_annotations)?; let line_index = LineIndex::new(source); let mut top = Compiler::new( "".to_owned(), @@ -554,7 +861,7 @@ pub fn compile_eval_with_source( CodeKind::Module, Rc::new(line_index), Rc::from(source), - has_future_annotations(module), + params, ); top.eval_mode = true; top.compile_module_body(module)?; @@ -765,6 +1072,10 @@ struct Compiler { /// strings rather than being evaluated at definition time. Propagated /// to every nested function/class scope. future_annotations: bool, + /// Per-compilation options shared with every nested scope + /// (optimize level, merged `CO_FUTURE_*` bits, top-level-await + /// permission) — RFC 0052. + params: CompileParams, /// CPython `ste_private`: the name of the innermost enclosing class, /// inherited by every scope textually inside it. Used to *demangle* /// def/class binding names back to their source spelling for @@ -830,6 +1141,14 @@ struct LoopFrame { /// blocks; without this the handled exception leaks until frame /// exit — test_exceptions.testExceptionCleanupState). handler_depth_at_entry: u32, + /// `exc_on_stack` when the loop was entered. `break`/`continue` + /// from inside a `finally` body running on the *exception path* + /// must discard both the propagating exception object (still on + /// the value stack, awaiting the RERAISE we're now skipping) and + /// its PUSH_EXC_INFO handler state — CPython's unwind of the + /// EXCEPTION_HANDLER fblock (`for … try: 1/0 finally: continue`, + /// test_grammar.test_continue_in_finally). + exc_on_stack_at_entry: u32, } /// One pending `finally` clause. We hold the AST so `return`, @@ -884,8 +1203,9 @@ impl Compiler { kind: CodeKind, line_index: Rc, source: Rc, - future_annotations: bool, + params: CompileParams, ) -> Self { + let future_annotations = params.future_annotations; let mut co = CodeObject::default(); // Default qualname == name; nested scopes overwrite this via // `compute_child_qualname` once the parent context is known. @@ -893,6 +1213,7 @@ impl Compiler { co.name = name; co.filename = filename; co.is_class_body = matches!(kind, CodeKind::Class); + co.future_flags = params.future_flags; Self { co, kind, @@ -921,6 +1242,7 @@ impl Compiler { eval_mode: false, source, future_annotations, + params, private: None, pep695_qualname: None, pending_pep695_qualname: None, @@ -1279,6 +1601,14 @@ impl Compiler { fn compile_module_body(&mut self, module: &Module) -> Result<(), CompileError> { self.analyze_scope_module(module); + // PyCF_ALLOW_TOP_LEVEL_AWAIT: a module body that awaits is a + // coroutine code object, and (like generator functions) the VM's + // bootstrap requires RETURN_GENERATOR as the first instruction — + // so this must be decided before emission starts. + if self.allows_top_level_await() && body_has_top_level_await(&module.body) { + self.co.is_coroutine = true; + self.emit(OpCode::ReturnGenerator, 0); + } self.emit(OpCode::Resume, 0); // CPython's symtable marks a module block containing any annotated // statement (at the block's own level) and the compiler emits @@ -1289,7 +1619,26 @@ impl Compiler { self.emit(OpCode::SetupAnnotations, 0); self.annotations_initialized = true; } - for stmt in &module.body { + // CPython's compiler_body stores a module's leading string + // literal as `__doc__` (exec mode only — the REPL echoes it and + // eval mode can't contain it) and skips re-evaluating it as an + // expression statement. Under `-OO` (optimize >= 2) the + // docstring is dropped entirely. + let mut body: &[Stmt] = &module.body; + if !self.interactive && !self.eval_mode { + if let Some(doc) = first_stmt_docstring(&module.body) { + if self.params.optimize < 2 { + let doc_const = self.co.intern_constant(Constant::Str(doc.to_owned())); + let doc_name = self.co.intern_name("__doc__"); + self.set_line_from(module.body[0].span.start.0); + self.set_span(module.body[0].span); + self.emit(OpCode::LoadConst, doc_const); + self.emit(OpCode::StoreName, doc_name); + } + body = &module.body[1..]; + } + } + for stmt in body { self.compile_stmt(stmt)?; } Ok(()) @@ -1374,6 +1723,13 @@ impl Compiler { } let mut free_candidates = reads.clone(); free_candidates.extend(needed_in_inner.iter().cloned()); + // Iterate in sorted order: `free_candidates` is a `HashSet`, and + // its iteration order would otherwise leak into `free_order` (→ + // `co_freevars`), making two compiles of the same source disagree + // (`test_compile_ast` asserts source-vs-AST code equality). CPython + // sorts these names too (`dictbytype`). + let mut free_candidates: Vec = free_candidates.into_iter().collect(); + free_candidates.sort_unstable(); for name in free_candidates { if self.bindings.contains_key(&name) { continue; @@ -1399,6 +1755,10 @@ impl Compiler { // reads or declares them as free / nonlocal. We do this // BEFORE emission so the very first `STORE_*` for each // promoted name routes through the cell. + // Sorted for the same determinism reason as above — the promotion + // order becomes the `co_cellvars` order. + let mut needed_in_inner: Vec = needed_in_inner.into_iter().collect(); + needed_in_inner.sort_unstable(); for name in needed_in_inner { if matches!(self.bindings.get(&name), Some(Binding::Local)) { self.bindings.insert(name.clone(), Binding::Cell); @@ -1432,6 +1792,13 @@ impl Compiler { self.emit(OpCode::PopTop, 0); } } + StmtKind::TypeAlias { .. } => { + // Normally rewritten at the compiler entry + // (`lower_type_aliases`); handled here too so a caller + // compiling a raw parse AST still works. + let lowered = weavepy_parser::lower_type_alias_stmt(stmt); + self.compile_stmt(&lowered)?; + } StmtKind::Pass => {} StmtKind::Delete(targets) => { for target in targets { @@ -1446,9 +1813,11 @@ impl Compiler { // RAISE_VARARGS 1 // end: // - // We don't yet strip assertions under `-O`; the VM - // checks `sys.flags.optimize` at runtime if it wants - // to elide the AssertionError raise. + // Under `-O`/`-OO` (optimize >= 1) assertions compile + // to nothing, exactly like CPython's compiler_assert. + if self.params.optimize >= 1 { + return Ok(()); + } self.compile_expr(test)?; // The raise sequence carries the *test expression's* // location (CPython compiler_assert): the traceback's @@ -1596,6 +1965,7 @@ impl Compiler { break_sites: Vec::new(), is_for_loop: false, handler_depth_at_entry: self.handler_depth, + exc_on_stack_at_entry: self.exc_on_stack, }); for s in body { self.compile_stmt(s)?; @@ -1660,6 +2030,7 @@ impl Compiler { break_sites: Vec::new(), is_for_loop: true, handler_depth_at_entry: self.handler_depth, + exc_on_stack_at_entry: self.exc_on_stack, }); for s in body { self.compile_stmt(s)?; @@ -1691,10 +2062,14 @@ impl Compiler { orelse, } => { if !self.in_async_context() { - return Err(CompileError::spanned( - "'async for' outside async function", - stmt.span, - )); + if self.allows_top_level_await() { + self.co.is_coroutine = true; + } else { + return Err(CompileError::spanned( + "'async for' outside async function", + stmt.span, + )); + } } self.compile_async_for(target, iter, body, orelse)?; } @@ -1779,10 +2154,14 @@ impl Compiler { } StmtKind::AsyncWith { items, body } => { if !self.in_async_context() { - return Err(CompileError::spanned( - "'async with' outside async function", - stmt.span, - )); + if self.allows_top_level_await() { + self.co.is_coroutine = true; + } else { + return Err(CompileError::spanned( + "'async with' outside async function", + stmt.span, + )); + } } self.compile_async_with(items, body)?; } @@ -1873,12 +2252,22 @@ impl Compiler { let exc_to_pop = self .handler_depth .saturating_sub(frame_top.handler_depth_at_entry); + let exc_vals = self + .exc_on_stack + .saturating_sub(frame_top.exc_on_stack_at_entry); // Leaving `except` handler bodies on the way out: discard // their handled-exception state (CPython POP_EXCEPT // during block unwind). for _ in 0..exc_to_pop { self.emit(OpCode::PopExcept, 0); } + // Leaving a `finally` running on the exception path: + // drop the propagating exception (value stack) and its + // handler state — the RERAISE it awaited is skipped. + for _ in 0..exc_vals { + self.emit(OpCode::PopExcept, 0); + self.emit(OpCode::PopTop, 0); + } // Run any `finally` clauses that lie between us and // the enclosing loop, in innermost-out order. self.inline_finally_for_loop_exit()?; @@ -1902,9 +2291,17 @@ impl Compiler { let exc_to_pop = self .handler_depth .saturating_sub(frame_top.handler_depth_at_entry); + let exc_vals = self + .exc_on_stack + .saturating_sub(frame_top.exc_on_stack_at_entry); for _ in 0..exc_to_pop { self.emit(OpCode::PopExcept, 0); } + // See Break: unwind exception-path `finally` state. + for _ in 0..exc_vals { + self.emit(OpCode::PopExcept, 0); + self.emit(OpCode::PopTop, 0); + } self.inline_finally_for_loop_exit()?; let site = self.emit(OpCode::JumpBackward, 0); self.patch_jump(site, target); @@ -2802,7 +3199,7 @@ impl Compiler { CodeKind::Function, self.line_index.clone(), self.source.clone(), - self.future_annotations, + self.params, ); inner.private = self.private.clone(); // PEP 695: a hidden `` scope being @@ -2843,6 +3240,53 @@ impl Compiler { inner.bindings.insert("__class__".to_owned(), Binding::Free); inner.free_order.push("__class__".to_owned()); } + // CPython symtable: every `nonlocal` must resolve to a binding + // in some enclosing *function* scope. Our scope analysis is + // chained — each scope only consults its parent — but that's + // sufficient: any name an outer function chain provides is + // already forwarded into `self.bindings` (as Local / Cell / + // Free) by the time this child compiles. The module scope + // never satisfies a nonlocal, and a class-body Local is a class + // attribute, not a nonlocal binding target. + { + let mut ng = HashSet::new(); + let mut nl = HashSet::new(); + let mut na = HashSet::new(); + for s in body { + collect_decls(s, &mut ng, &mut nl, &mut na); + } + let mut nl: Vec = nl.into_iter().collect(); + nl.sort_unstable(); + for n in nl { + let ok = match self.kind { + CodeKind::Module => false, + CodeKind::Class => matches!( + self.bindings.get(&n), + Some(Binding::Free | Binding::Nonlocal | Binding::ClassPassthrough) + ), + CodeKind::Function | CodeKind::Comprehension => matches!( + self.bindings.get(&n), + Some( + Binding::Local + | Binding::Cell + | Binding::Free + | Binding::Nonlocal + | Binding::ClassPassthrough + ) + ), + }; + if !ok { + let span = find_nonlocal_decl_span(body, &n).unwrap_or_else(|| { + body.first() + .map_or(weavepy_lexer::Span::new(0, 0), |s| s.span) + }); + return Err(CompileError::spanned( + format!("no binding for nonlocal '{n}' found"), + span, + )); + } + } + } inner.analyze_scope_function(¶m_names, body, &[&self.bindings]); for free in &inner.free_order { if matches!(self.bindings.get(free), Some(Binding::Local)) { @@ -2876,8 +3320,10 @@ impl Compiler { // `LoadConst`, and a `None` slot is reused by the implicit // `return None`. let doc_slot = match first_stmt_docstring(body) { - Some(doc) => Constant::Str(doc.to_owned()), - None => Constant::None, + // `-OO` (optimize >= 2) strips docstrings; the slot decays + // to the shared `None` constant like CPython's. + Some(doc) if self.params.optimize < 2 => Constant::Str(doc.to_owned()), + _ => Constant::None, }; inner.co.intern_constant(doc_slot); for s in body { @@ -3072,7 +3518,7 @@ impl Compiler { CodeKind::Class, self.line_index.clone(), self.source.clone(), - self.future_annotations, + self.params, ); inner.private = Some(Rc::from(name)); inner.co.qualname = self.compute_child_qualname(name); @@ -3175,6 +3621,10 @@ impl Compiler { let mut free_candidates = reads; free_candidates.extend(needed_in_inner.iter().cloned()); free_candidates.remove("__class__"); + // Sorted so `free_order` (→ `co_freevars`) is deterministic across + // compiles — see the function-scope analogue above. + let mut free_candidates: Vec = free_candidates.into_iter().collect(); + free_candidates.sort_unstable(); for name in free_candidates { if inner.bindings.contains_key(&name) { continue; @@ -3200,6 +3650,8 @@ impl Compiler { // forwards it (the name joins `co_freevars`) while its own // loads/stores keep namespace semantics — see // [`Binding::ClassPassthrough`]. + let mut needed_in_inner: Vec = needed_in_inner.into_iter().collect(); + needed_in_inner.sort_unstable(); for name in &needed_in_inner { if name == "__class__" || name == "__classdict__" { continue; @@ -3287,10 +3739,13 @@ impl Compiler { // body reserves that slot for the qualname, so it must be an // explicit store rather than a constant-slot convention. if let Some(doc) = first_stmt_docstring(body) { - let doc_const = inner.co.intern_constant(Constant::Str(doc.to_owned())); - let doc_name = inner.co.intern_name("__doc__"); - inner.emit(OpCode::LoadConst, doc_const); - inner.emit(OpCode::StoreName, doc_name); + // `-OO` (optimize >= 2) strips class docstrings too. + if self.params.optimize < 2 { + let doc_const = inner.co.intern_constant(Constant::Str(doc.to_owned())); + let doc_name = inner.co.intern_name("__doc__"); + inner.emit(OpCode::LoadConst, doc_const); + inner.emit(OpCode::StoreName, doc_name); + } } // SETUP_ANNOTATIONS before the first body statement when the class @@ -4396,8 +4851,22 @@ impl Compiler { // which normalises quoting and whitespace — `List[list["C2"]]` // annotates as "List[list['C2']]". Fall back to the raw source // slice for nodes the unparser doesn't cover. - let text = weavepy_parser::unparse::unparse_expr(annotation) + let mut text = weavepy_parser::unparse::unparse_expr(annotation) .or_else(|| self.annotation_source(annotation)); + // The Rust AST doesn't carry `Constant.kind`, so a legacy + // `u'…'` prefix (which CPython's unparser preserves) is + // recovered from the source text. + if let (Some(t), ExprKind::Constant(AstConstant::Str(_))) = + (text.as_deref(), &annotation.kind) + { + if t.starts_with(['\'', '"']) + && self + .annotation_source(annotation) + .is_some_and(|src| src.starts_with(['u', 'U'])) + { + text = Some(format!("u{t}")); + } + } if let Some(text) = text { let idx = self.co.intern_constant(Constant::Str(text)); self.emit(OpCode::LoadConst, idx); @@ -4490,6 +4959,10 @@ impl Compiler { fn compile_assign(&mut self, target: &Expr) -> Result<(), CompileError> { match &target.kind { + ExprKind::Name(n) if n == "__debug__" => Err(CompileError::spanned( + "cannot assign to __debug__", + target.span, + )), ExprKind::Name(n) => { // CPython attributes the STORE to the Name node itself, // not the enclosing statement. Only observable when the @@ -4507,6 +4980,12 @@ impl Compiler { self.current_span = saved_span; Ok(()) } + // `obj.__debug__ = 1` — CPython's `forbidden_name` check + // applies to attribute targets too. + ExprKind::Attribute { attr, .. } if attr == "__debug__" => Err(CompileError::spanned( + "cannot assign to __debug__", + target.span, + )), ExprKind::Attribute { value, attr } => { self.compile_expr(value)?; let idx = self.co.intern_name(attr); @@ -4696,6 +5175,10 @@ impl Compiler { fn compile_delete(&mut self, target: &Expr) -> Result<(), CompileError> { match &target.kind { + ExprKind::Name(n) if n == "__debug__" => Err(CompileError::spanned( + "cannot delete __debug__", + target.span, + )), ExprKind::Name(n) => { self.emit_delete_name(n); Ok(()) @@ -4838,6 +5321,15 @@ impl Compiler { } fn emit_load_name(&mut self, name: &str) { + // `__debug__` is a compile-time constant in CPython: `True` + // at optimize 0, `False` under `-O`/`-OO` (RFC 0052). + if name == "__debug__" { + let idx = self + .co + .intern_constant(Constant::Bool(self.params.optimize == 0)); + self.emit(OpCode::LoadConst, idx); + return; + } let binding = self.bindings.get(name).copied(); // PEP 695 annotation scope inside a class body: free and // (implicit-)global loads consult the `__classdict__` mapping @@ -5290,14 +5782,21 @@ impl Compiler { } ExprKind::Await(value) => { if !self.in_async_context() { - return Err(CompileError::spanned( - if self.kind == CodeKind::Function { - "'await' outside async function" - } else { - "'await' outside function" - }, - e.span, - )); + if self.allows_top_level_await() { + // PyCF_ALLOW_TOP_LEVEL_AWAIT: the module code + // becomes a coroutine (CPython marks it + // CO_COROUTINE) — the asyncio REPL contract. + self.co.is_coroutine = true; + } else { + return Err(CompileError::spanned( + if self.kind == CodeKind::Function { + "'await' outside async function" + } else { + "'await' outside function" + }, + e.span, + )); + } } self.compile_expr(value)?; self.compile_await_dance(0); @@ -5310,11 +5809,14 @@ impl Compiler { /// inside a comprehension scope the message names the comprehension /// form, otherwise it's "outside function". fn yield_placement_error(&self, kw: &str, span: weavepy_lexer::Span) -> CompileError { + // CPython's symtable always says "'yield' inside …" for the + // comprehension case, even for `yield from`; only the + // "outside function" form names `yield from` distinctly. let msg = match self.comp_kind { - Some(CompKind::List) => format!("'{kw}' inside list comprehension"), - Some(CompKind::Set) => format!("'{kw}' inside set comprehension"), - Some(CompKind::Dict) => format!("'{kw}' inside dict comprehension"), - Some(CompKind::Generator) => format!("'{kw}' inside generator expression"), + Some(CompKind::List) => "'yield' inside list comprehension".to_owned(), + Some(CompKind::Set) => "'yield' inside set comprehension".to_owned(), + Some(CompKind::Dict) => "'yield' inside dict comprehension".to_owned(), + Some(CompKind::Generator) => "'yield' inside generator expression".to_owned(), None => format!("'{kw}' outside function"), }; CompileError::spanned(msg, span) @@ -5351,6 +5853,13 @@ impl Compiler { self.co.is_coroutine || self.co.is_async_generator } + /// PyCF_ALLOW_TOP_LEVEL_AWAIT (RFC 0052): `await` / `async for` / + /// `async with` are legal at module top level; using one turns the + /// module code object into a coroutine. + fn allows_top_level_await(&self) -> bool { + self.params.allow_top_level_await && matches!(self.kind, CodeKind::Module) + } + fn compile_async_for( &mut self, target: &Expr, @@ -5380,6 +5889,7 @@ impl Compiler { break_sites: Vec::new(), is_for_loop: true, handler_depth_at_entry: self.handler_depth, + exc_on_stack_at_entry: self.exc_on_stack, }); for s in body { self.compile_stmt(s)?; @@ -5712,7 +6222,7 @@ impl Compiler { CodeKind::Comprehension, self.line_index.clone(), self.source.clone(), - self.future_annotations, + self.params, ); inner.current_line = self.current_line; inner.comp_kind = Some(kind); @@ -5829,6 +6339,8 @@ impl Compiler { collect_inner_free_expr(cond, &inner.bindings, &mut needed_in_inner); } } + let mut needed_in_inner: Vec = needed_in_inner.into_iter().collect(); + needed_in_inner.sort_unstable(); for name in needed_in_inner { if matches!(inner.bindings.get(&name), Some(Binding::Local)) { inner.bindings.insert(name.clone(), Binding::Cell); @@ -6368,8 +6880,50 @@ fn collect_inner_free( ) { collect_pep695_header_reads(stmt, out); match &stmt.kind { - StmtKind::FunctionDef { args, body, .. } - | StmtKind::AsyncFunctionDef { args, body, .. } => { + StmtKind::FunctionDef { + args, + body, + decorator_list, + returns, + .. + } + | StmtKind::AsyncFunctionDef { + args, + body, + decorator_list, + returns, + .. + } => { + // Decorators, default values, and annotations evaluate in the + // *enclosing* scope, but may themselves contain nested scopes + // (`@lambda f: null(f)` — PEP 614) that close over our locals. + for d in decorator_list { + collect_inner_free_expr(d, outer_bindings, out); + } + for d in args + .defaults + .iter() + .chain(args.kw_defaults.iter().flatten()) + { + collect_inner_free_expr(d, outer_bindings, out); + } + if !pep563_active() { + for a in args + .posonlyargs + .iter() + .chain(&args.args) + .chain(&args.kwonlyargs) + .chain(&args.vararg) + .chain(&args.kwarg) + { + if let Some(ann) = &a.annotation { + collect_inner_free_expr(ann, outer_bindings, out); + } + } + if let Some(r) = returns { + collect_inner_free_expr(r, outer_bindings, out); + } + } let mut inner_locals: HashSet = HashSet::new(); for a in &args.posonlyargs { inner_locals.insert(a.name.clone()); @@ -6634,6 +7188,155 @@ fn body_is_generator(body: &[Stmt]) -> bool { body.iter().any(stmt_contains_yield) } +/// Pre-scan for PyCF_ALLOW_TOP_LEVEL_AWAIT (RFC 0052): does the module +/// body use `await` / `async for` / `async with` / an inline-awaited +/// async comprehension at its own scope level? When it does, the module +/// code object must be a coroutine, and — like generator functions — +/// that has to be known *before* emission so `RETURN_GENERATOR` can be +/// the first instruction (the VM's generator bootstrap stops there). +/// Nested `def`/`class` bodies don't count; their awaits are their own. +fn body_has_top_level_await(body: &[Stmt]) -> bool { + fn expr_hit(e: &Expr) -> bool { + match &e.kind { + ExprKind::Await(_) => true, + // Lambda bodies are their own scope, but default values + // evaluate here. + ExprKind::Lambda { args, .. } | ExprKind::TypeParamFn { args, .. } => { + args.defaults.iter().any(expr_hit) + || args.kw_defaults.iter().flatten().any(expr_hit) + } + // An inline-awaited async list/set/dict comprehension awaits + // *in this scope*; so does anything in the first `for` + // clause's iterable (evaluated here, passed in as `.0`) — + // including a nested async comprehension + // (`[1 for x in {y async for y in a}]`). + ExprKind::ListComp { elt, generators } | ExprKind::SetComp { elt, generators } => { + comp_clause_is_async(generators, elt, None) + || generators.first().is_some_and(|g| expr_hit(&g.iter)) + } + ExprKind::DictComp { + key, + value, + generators, + } => { + comp_clause_is_async(generators, key, Some(value)) + || generators.first().is_some_and(|g| expr_hit(&g.iter)) + } + // An async genexp is just an async-generator object — only + // its first iterable evaluates here. + ExprKind::GeneratorExp { generators, .. } => { + generators.first().is_some_and(|g| expr_hit(&g.iter)) + } + ExprKind::Yield(v) => v.as_deref().is_some_and(expr_hit), + ExprKind::YieldFrom(v) => expr_hit(v), + ExprKind::JoinedStr(parts) => parts.iter().any(expr_hit), + ExprKind::FormattedValue { + value, format_spec, .. + } => expr_hit(value) || format_spec.as_deref().is_some_and(expr_hit), + ExprKind::BinOp { left, right, .. } => expr_hit(left) || expr_hit(right), + ExprKind::BoolOp { values, .. } => values.iter().any(expr_hit), + ExprKind::UnaryOp { operand, .. } => expr_hit(operand), + ExprKind::Compare { + left, comparators, .. + } => expr_hit(left) || comparators.iter().any(expr_hit), + ExprKind::IfExp { test, body, orelse } => { + expr_hit(test) || expr_hit(body) || expr_hit(orelse) + } + ExprKind::NamedExpr { target, value } => expr_hit(target) || expr_hit(value), + ExprKind::Call { + func, + args, + keywords, + } => { + expr_hit(func) + || args.iter().any(expr_hit) + || keywords.iter().any(|k| expr_hit(&k.value)) + } + ExprKind::Attribute { value, .. } => expr_hit(value), + ExprKind::Subscript { value, slice } => expr_hit(value) || expr_hit(slice), + ExprKind::Slice { lower, upper, step } => { + lower.as_deref().is_some_and(expr_hit) + || upper.as_deref().is_some_and(expr_hit) + || step.as_deref().is_some_and(expr_hit) + } + ExprKind::Tuple(items) | ExprKind::List(items) | ExprKind::Set(items) => { + items.iter().any(expr_hit) + } + ExprKind::Dict { keys, values } => { + keys.iter().any(|k| k.as_ref().is_some_and(expr_hit)) || values.iter().any(expr_hit) + } + ExprKind::Starred(inner) => expr_hit(inner), + ExprKind::Constant(_) | ExprKind::Name(_) => false, + } + } + fn stmt_hit(stmt: &Stmt) -> bool { + match &stmt.kind { + StmtKind::AsyncFor { .. } | StmtKind::AsyncWith { .. } => true, + StmtKind::FunctionDef { .. } + | StmtKind::AsyncFunctionDef { .. } + | StmtKind::ClassDef { .. } => false, + StmtKind::Expr(e) => expr_hit(e), + StmtKind::Assign { targets, value } => expr_hit(value) || targets.iter().any(expr_hit), + StmtKind::AugAssign { target, value, .. } => expr_hit(target) || expr_hit(value), + StmtKind::AnnAssign { target, value, .. } => { + expr_hit(target) || value.as_ref().is_some_and(expr_hit) + } + StmtKind::Return(v) => v.as_ref().is_some_and(expr_hit), + StmtKind::If { test, body, orelse } | StmtKind::While { test, body, orelse } => { + expr_hit(test) || body.iter().any(stmt_hit) || orelse.iter().any(stmt_hit) + } + StmtKind::For { + target, + iter, + body, + orelse, + } => { + expr_hit(target) + || expr_hit(iter) + || body.iter().any(stmt_hit) + || orelse.iter().any(stmt_hit) + } + StmtKind::With { items, body } => { + items.iter().any(|w| { + expr_hit(&w.context_expr) || w.optional_vars.as_ref().is_some_and(expr_hit) + }) || body.iter().any(stmt_hit) + } + StmtKind::Try { + body, + handlers, + orelse, + finalbody, + } => { + body.iter().any(stmt_hit) + || handlers.iter().any(|h| h.body.iter().any(stmt_hit)) + || orelse.iter().any(stmt_hit) + || finalbody.iter().any(stmt_hit) + } + StmtKind::Raise { exc, cause } => { + exc.as_ref().is_some_and(expr_hit) || cause.as_ref().is_some_and(expr_hit) + } + StmtKind::Match { subject, cases } => { + expr_hit(subject) + || cases.iter().any(|c| { + c.guard.as_ref().is_some_and(expr_hit) || c.body.iter().any(stmt_hit) + }) + } + StmtKind::Global(_) + | StmtKind::Nonlocal(_) + | StmtKind::Import(_) + | StmtKind::ImportFrom { .. } + | StmtKind::Pass + | StmtKind::Break + | StmtKind::Continue => false, + StmtKind::Delete(targets) => targets.iter().any(expr_hit), + StmtKind::Assert { test, msg } => expr_hit(test) || msg.as_ref().is_some_and(expr_hit), + // `await` is rejected inside type-alias values at parse time. + StmtKind::TypeAlias { .. } => false, + } + } + body.iter().any(stmt_hit) +} + fn stmt_contains_yield(stmt: &Stmt) -> bool { match &stmt.kind { StmtKind::FunctionDef { .. } @@ -6713,6 +7416,8 @@ fn stmt_contains_yield(stmt: &Stmt) -> bool { StmtKind::Assert { test, msg } => { expr_contains_yield(test) || msg.as_ref().is_some_and(expr_contains_yield) } + // `yield` is rejected inside type-alias values at parse time. + StmtKind::TypeAlias { .. } => false, } } @@ -7850,9 +8555,23 @@ fn collect_decls( collect_target_names(t, assigned); } } - StmtKind::AugAssign { target, .. } | StmtKind::AnnAssign { target, .. } => { + StmtKind::AugAssign { target, .. } => { collect_target_names(target, assigned); } + // CPython symtable (AnnAssign): a *simple* annotated name is + // DEF_LOCAL even without a value (`x: int` → UnboundLocalError + // on read); a parenthesized one only binds when it has a value + // (`(x): int` alone leaves `x` resolving globally → NameError). + StmtKind::AnnAssign { + target, + value, + simple, + .. + } => { + if *simple || value.is_some() { + collect_target_names(target, assigned); + } + } // `del NAME` is a binding operation in CPython (`DEF_LOCAL`): the // name is local to this scope, and — crucially — a nested scope // declaring it `nonlocal` resolves to (and cells) it here. Bare @@ -7962,6 +8681,60 @@ fn collect_decls( } } +/// Locate the `nonlocal NAME` statement declaring `name` within this +/// scope's body (recursing into compound statements but not into nested +/// scopes), for error anchoring. +fn find_nonlocal_decl_span(body: &[Stmt], name: &str) -> Option { + for s in body { + match &s.kind { + StmtKind::Nonlocal(ns) if ns.iter().any(|n| n == name) => return Some(s.span), + StmtKind::For { body, orelse, .. } + | StmtKind::AsyncFor { body, orelse, .. } + | StmtKind::While { body, orelse, .. } + | StmtKind::If { body, orelse, .. } => { + if let Some(sp) = find_nonlocal_decl_span(body, name) + .or_else(|| find_nonlocal_decl_span(orelse, name)) + { + return Some(sp); + } + } + StmtKind::Try { + body, + handlers, + orelse, + finalbody, + } => { + if let Some(sp) = find_nonlocal_decl_span(body, name) + .or_else(|| { + handlers + .iter() + .find_map(|h| find_nonlocal_decl_span(&h.body, name)) + }) + .or_else(|| find_nonlocal_decl_span(orelse, name)) + .or_else(|| find_nonlocal_decl_span(finalbody, name)) + { + return Some(sp); + } + } + StmtKind::With { body, .. } | StmtKind::AsyncWith { body, .. } => { + if let Some(sp) = find_nonlocal_decl_span(body, name) { + return Some(sp); + } + } + StmtKind::Match { cases, .. } => { + if let Some(sp) = cases + .iter() + .find_map(|c| find_nonlocal_decl_span(&c.body, name)) + { + return Some(sp); + } + } + _ => {} + } + } + None +} + fn collect_target_names(expr: &Expr, out: &mut HashSet) { match &expr.kind { ExprKind::Name(n) => { @@ -8076,7 +8849,11 @@ fn collect_reads_stmt(stmt: &Stmt, out: &mut HashSet) { .. } => { collect_reads_expr(target, out); - collect_reads_expr(annotation, out); + // PEP 563: stringified annotations are never evaluated and + // must not participate in scope analysis. + if !pep563_active() { + collect_reads_expr(annotation, out); + } if let Some(v) = value { collect_reads_expr(v, out); } @@ -8138,20 +8915,22 @@ fn collect_reads_stmt(stmt: &Stmt, out: &mut HashSet) { for d in args.kw_defaults.iter().flatten() { collect_reads_expr(d, out); } - for a in args - .posonlyargs - .iter() - .chain(&args.args) - .chain(&args.kwonlyargs) - .chain(&args.vararg) - .chain(&args.kwarg) - { - if let Some(ann) = &a.annotation { - collect_reads_expr(ann, out); + if !pep563_active() { + for a in args + .posonlyargs + .iter() + .chain(&args.args) + .chain(&args.kwonlyargs) + .chain(&args.vararg) + .chain(&args.kwarg) + { + if let Some(ann) = &a.annotation { + collect_reads_expr(ann, out); + } + } + if let Some(r) = returns { + collect_reads_expr(r, out); } - } - if let Some(r) = returns { - collect_reads_expr(r, out); } for s in body { collect_reads_stmt(s, out); diff --git a/crates/weavepy-compiler/src/mangle.rs b/crates/weavepy-compiler/src/mangle.rs index a1e5b92..60dd044 100644 --- a/crates/weavepy-compiler/src/mangle.rs +++ b/crates/weavepy-compiler/src/mangle.rs @@ -219,6 +219,29 @@ impl Mangler { sub.expr(&mut k.value); } } + StmtKind::TypeAlias { + name, + type_params, + value, + .. + } => { + // Normally dead — the compiler lowers `type` statements + // to their assignment form before mangling — but kept + // faithful for safety: the binding and the type-parameter + // names mangle; bounds/defaults/value are ordinary + // expressions (matching what the lowered form produces). + self.name(name); + for tp in type_params { + self.name(&mut tp.name); + if let TypeParamKind::TypeVar { bound: Some(b) } = &mut tp.kind { + self.expr(b); + } + if let Some(d) = &mut tp.default { + self.expr(d); + } + } + self.expr(value); + } StmtKind::Return(v) => { if let Some(v) = v { self.expr(v); diff --git a/crates/weavepy-compiler/src/validate.rs b/crates/weavepy-compiler/src/validate.rs index c3d40e2..d01bc94 100644 --- a/crates/weavepy-compiler/src/validate.rs +++ b/crates/weavepy-compiler/src/validate.rs @@ -43,6 +43,11 @@ enum ScopeKind { /// (CPython symtable: "nonlocal binding not allowed for type /// parameter"). TypeParams, + /// A comprehension's implicit function scope. Names read inside it + /// don't count as uses of the enclosing scope (the outermost + /// iterable is visited in the enclosing scope before this is + /// pushed), and walrus targets bind through it. + Comprehension, } /// One declaration recorded by a `global`/`nonlocal` statement — @@ -64,27 +69,51 @@ struct Scope { /// [`ScopeKind::TypeParams`]. Consulted when resolving `nonlocal` /// declarations from nested scopes. bound: std::collections::HashSet, + /// Names *read* so far, in source order (CPython's `USE` flag) — + /// a later `global`/`nonlocal` for one of these is "used prior to + /// … declaration". + used: std::collections::HashSet, + /// Names assigned/deleted/bound so far (`DEF_LOCAL`) — a later + /// declaration is "assigned to before … declaration". + assigned: std::collections::HashSet, + /// Names annotated so far (`DEF_ANNOT`) — can never be declared + /// global/nonlocal in this scope, before *or* after. + annotated: std::collections::HashSet, } impl Scope { + fn new(kind: ScopeKind) -> Scope { + Scope { + kind, + params: Vec::new(), + directives: Vec::new(), + bound: std::collections::HashSet::new(), + used: std::collections::HashSet::new(), + assigned: std::collections::HashSet::new(), + annotated: std::collections::HashSet::new(), + } + } + fn directive_for(&self, name: &str) -> Option<&Directive> { self.directives.iter().find(|d| d.name == name) } } -pub(crate) fn validate_module(module: &Module, source: &str) -> Result<(), CompileError> { +pub(crate) fn validate_module( + module: &Module, + source: &str, + future_annotations: bool, +) -> Result<(), CompileError> { let mut v = Validator { source, - scopes: vec![Scope { - kind: ScopeKind::Module, - params: Vec::new(), - directives: Vec::new(), - bound: std::collections::HashSet::new(), - }], + scopes: vec![Scope::new(ScopeKind::Module)], + future_annotations, }; // `from __future__ import …` placement / feature validation // (CPython `future.c`). Only a docstring, comments, and other - // future imports may precede one. + // future imports may precede one. Relative imports + // (`from .__future__ import x`) are ordinary imports, not future + // statements. let mut prologue = true; for (i, stmt) in module.body.iter().enumerate() { match &stmt.kind { @@ -97,7 +126,11 @@ pub(crate) fn validate_module(module: &Module, source: &str) -> Result<(), Compi { // Module docstring keeps the prologue open. } - StmtKind::ImportFrom { module: m, .. } if m.as_deref() == Some("__future__") => { + StmtKind::ImportFrom { + module: m, + level: 0, + .. + } if m.as_deref() == Some("__future__") => { if !prologue { return Err(CompileError::spanned( "from __future__ imports must occur at the beginning of the file", @@ -117,6 +150,11 @@ pub(crate) fn validate_module(module: &Module, source: &str) -> Result<(), Compi struct Validator<'src> { source: &'src str, scopes: Vec, + /// PEP 563 active (module has `from __future__ import annotations` + /// or the caller passed `CO_FUTURE_ANNOTATIONS`): annotations are + /// never evaluated, so their names don't participate in scope + /// analysis, but yield/await/walrus inside them become errors. + future_annotations: bool, } impl Validator<'_> { @@ -128,6 +166,56 @@ impl Validator<'_> { self.scopes.last_mut().expect("scope stack never empty") } + /// Record a *read* of `name` in the current scope (CPython `USE`). + fn mark_use(&mut self, name: &str) { + self.scope_mut().used.insert(name.to_owned()); + } + + /// Record a binding of `name` (CPython `DEF_LOCAL`). Comprehension + /// scopes are transparent to bindings: a walrus inside one binds + /// in the enclosing function/class/module scope. + fn mark_assigned(&mut self, name: &str) { + let idx = self + .scopes + .iter() + .rposition(|s| s.kind != ScopeKind::Comprehension) + .expect("scope stack always has a non-comprehension scope"); + self.scopes[idx].assigned.insert(name.to_owned()); + } + + /// Visit an annotation expression. Under PEP 563 the annotation is + /// never evaluated: its names don't count as uses for scope + /// analysis, but yield/await/named expressions inside it are + /// compile-time errors (CPython symtable). + fn visit_annotation(&mut self, ann: &Expr) -> Result<(), CompileError> { + if self.future_annotations { + check_annotation_expr(ann) + } else { + self.visit_expr(ann) + } + } + + /// Visit an assignment target: bare names (and names inside + /// tuple/list/starred unpacking) are bindings, while + /// attribute/subscript targets *read* their base expression + /// (CPython marks `x` as `USE` in `x[0] = 1`). + fn visit_target(&mut self, expr: &Expr) -> Result<(), CompileError> { + match &expr.kind { + ExprKind::Name(n) => { + let n = n.clone(); + self.mark_assigned(&n); + } + ExprKind::Tuple(items) | ExprKind::List(items) => { + for i in items { + self.visit_target(i)?; + } + } + ExprKind::Starred(inner) => self.visit_target(inner)?, + _ => self.visit_expr(expr)?, + } + Ok(()) + } + fn visit_body(&mut self, body: &[Stmt]) -> Result<(), CompileError> { for s in body { self.visit_stmt(s)?; @@ -140,9 +228,8 @@ impl Validator<'_> { args: &Arguments, body: &[Stmt], decorators: &[Expr], - defaults_scope_ok: bool, + returns: Option<&Expr>, ) -> Result<(), CompileError> { - let _ = defaults_scope_ok; for d in decorators { self.visit_expr(d)?; } @@ -153,6 +240,9 @@ impl Validator<'_> { for d in args.kw_defaults.iter().flatten() { self.visit_expr(d)?; } + if let Some(r) = returns { + self.visit_annotation(r)?; + } let mut params: Vec<(&str, Span)> = Vec::new(); for a in args .posonlyargs @@ -163,7 +253,7 @@ impl Validator<'_> { .chain(&args.kwarg) { if let Some(ann) = &a.annotation { - self.visit_expr(ann)?; + self.visit_annotation(ann)?; } if params.iter().any(|(n, _)| *n == a.name) { return Err(CompileError::spanned( @@ -189,10 +279,9 @@ impl Validator<'_> { bound.extend(assigned); } self.scopes.push(Scope { - kind: ScopeKind::Function, params: params.iter().map(|(n, _)| (*n).to_owned()).collect(), - directives: Vec::new(), bound, + ..Scope::new(ScopeKind::Function) }); let result = self.visit_body(body); self.scopes.pop(); @@ -207,10 +296,8 @@ impl Validator<'_> { return false; } self.scopes.push(Scope { - kind: ScopeKind::TypeParams, - params: Vec::new(), - directives: Vec::new(), bound: type_params.iter().map(|tp| tp.name.clone()).collect(), + ..Scope::new(ScopeKind::TypeParams) }); true } @@ -218,27 +305,33 @@ impl Validator<'_> { fn visit_stmt(&mut self, stmt: &Stmt) -> Result<(), CompileError> { match &stmt.kind { StmtKind::FunctionDef { + name, args, body, decorator_list, type_params, - .. + returns, } | StmtKind::AsyncFunctionDef { + name, args, body, decorator_list, type_params, - .. + returns, } => { + // The def's name binds in the enclosing scope. + let name = name.clone(); + self.mark_assigned(&name); let pushed = self.push_type_params_scope(type_params); - let result = self.visit_function(args, body, decorator_list, true); + let result = self.visit_function(args, body, decorator_list, returns.as_deref()); if pushed { self.scopes.pop(); } result?; } StmtKind::ClassDef { + name, body, decorator_list, bases, @@ -246,6 +339,8 @@ impl Validator<'_> { type_params, .. } => { + let name = name.clone(); + self.mark_assigned(&name); for d in decorator_list { self.visit_expr(d)?; } @@ -257,12 +352,7 @@ impl Validator<'_> { for k in keywords { self.visit_expr(&k.value)?; } - self.scopes.push(Scope { - kind: ScopeKind::Class, - params: Vec::new(), - directives: Vec::new(), - bound: std::collections::HashSet::new(), - }); + self.scopes.push(Scope::new(ScopeKind::Class)); let result = self.visit_body(body); self.scopes.pop(); result @@ -276,12 +366,34 @@ impl Validator<'_> { let span = stmt.span; for n in names { let scope = self.scope(); + // CPython symtable priority: PARAM, USE, ANNOT, + // ASSIGN — re-checked on every declaration, so a + // *duplicate* `global x` after an intervening + // use/assignment still errors. if scope.params.iter().any(|p| p == n) { return Err(CompileError::spanned( format!("name '{n}' is parameter and global"), span, )); } + if scope.used.contains(n) { + return Err(CompileError::spanned( + format!("name '{n}' is used prior to global declaration"), + span, + )); + } + if scope.annotated.contains(n) { + return Err(CompileError::spanned( + format!("annotated name '{n}' can't be global"), + span, + )); + } + if scope.assigned.contains(n) { + return Err(CompileError::spanned( + format!("name '{n}' is assigned to before global declaration"), + span, + )); + } if let Some(d) = scope.directive_for(n) { if !d.is_global { // Earlier `nonlocal` — anchor at the first @@ -317,6 +429,24 @@ impl Validator<'_> { span, )); } + if scope.used.contains(n) { + return Err(CompileError::spanned( + format!("name '{n}' is used prior to nonlocal declaration"), + span, + )); + } + if scope.annotated.contains(n) { + return Err(CompileError::spanned( + format!("annotated name '{n}' can't be nonlocal"), + span, + )); + } + if scope.assigned.contains(n) { + return Err(CompileError::spanned( + format!("name '{n}' is assigned to before nonlocal declaration"), + span, + )); + } if let Some(d) = scope.directive_for(n) { if d.is_global { let at = d.span; @@ -338,7 +468,7 @@ impl Validator<'_> { // rebinding is rejected (CPython symtable). for s in self.scopes[..self.scopes.len() - 1].iter().rev() { match s.kind { - ScopeKind::Class => {} + ScopeKind::Class | ScopeKind::Comprehension => {} ScopeKind::TypeParams => { if s.bound.contains(n) { return Err(CompileError::spanned( @@ -378,7 +508,7 @@ impl Validator<'_> { target, annotation, value, - .. + simple, } => { match &target.kind { ExprKind::Tuple(_) | ExprKind::List(_) => { @@ -388,7 +518,46 @@ impl Validator<'_> { target.span, )); } - ExprKind::Name(_) | ExprKind::Attribute { .. } | ExprKind::Subscript { .. } => { + ExprKind::Name(n) if n == "__debug__" => { + return Err(CompileError::spanned( + "cannot assign to __debug__", + target.span, + )); + } + ExprKind::Attribute { attr, .. } if attr == "__debug__" => { + return Err(CompileError::spanned( + "cannot assign to __debug__", + target.span, + )); + } + ExprKind::Name(n) => { + // Simple (unparenthesized) targets are + // annotations (`DEF_ANNOT`): incompatible with + // a global/nonlocal directive in either order. + // Parenthesized ones only bind (`DEF_LOCAL`). + if *simple { + // CPython skips this check at module scope + // (`ste_symbols == st_global`): `global x` + + // `x: int` at top level is valid. + if self.scope().kind != ScopeKind::Module { + if let Some(d) = self.scope().directive_for(n) { + let what = if d.is_global { "global" } else { "nonlocal" }; + return Err(CompileError::spanned( + format!("annotated name '{n}' can't be {what}"), + stmt.span, + )); + } + } + let n = n.clone(); + self.scope_mut().annotated.insert(n.clone()); + self.mark_assigned(&n); + } else { + let n = n.clone(); + self.mark_assigned(&n); + } + } + ExprKind::Attribute { .. } | ExprKind::Subscript { .. } => { + self.visit_expr(target)?; } _ => { return Err(CompileError::spanned( @@ -397,7 +566,7 @@ impl Validator<'_> { )); } } - self.visit_expr(annotation)?; + self.visit_annotation(annotation)?; if let Some(v) = value { self.visit_expr(v)?; } @@ -414,6 +583,10 @@ impl Validator<'_> { if let Some(t) = &h.type_ { self.visit_expr(t)?; } + if let Some(n) = &h.name { + let n = n.clone(); + self.mark_assigned(&n); + } self.visit_body(&h.body)?; } self.visit_body(orelse)?; @@ -421,12 +594,12 @@ impl Validator<'_> { } StmtKind::Assign { targets, value } => { for t in targets { - self.visit_expr(t)?; + self.visit_target(t)?; } self.visit_expr(value)?; } StmtKind::AugAssign { target, value, .. } => { - self.visit_expr(target)?; + self.visit_target(target)?; self.visit_expr(value)?; } StmtKind::Return(v) => { @@ -435,8 +608,9 @@ impl Validator<'_> { } } StmtKind::Delete(targets) => { + // `del x` binds (CPython `DEF_LOCAL`), same as assignment. for t in targets { - self.visit_expr(t)?; + self.visit_target(t)?; } } StmtKind::If { test, body, orelse } => { @@ -461,7 +635,7 @@ impl Validator<'_> { body, orelse, } => { - self.visit_expr(target)?; + self.visit_target(target)?; self.visit_expr(iter)?; self.visit_body(body)?; self.visit_body(orelse)?; @@ -470,7 +644,7 @@ impl Validator<'_> { for it in items { self.visit_expr(&it.context_expr)?; if let Some(v) = &it.optional_vars { - self.visit_expr(v)?; + self.visit_target(v)?; } } self.visit_body(body)?; @@ -504,7 +678,7 @@ impl Validator<'_> { if let StmtKind::ImportFrom { module: Some(m), names, - .. + level: 0, } = &stmt.kind { if m == "__future__" { @@ -515,7 +689,9 @@ impl Validator<'_> { self.alias_span(stmt, &a.name), )); } - if a.name != "*" && !KNOWN_FUTURES.contains(&a.name.as_str()) { + if !KNOWN_FUTURES.contains(&a.name.as_str()) { + // `from __future__ import *` gets the same + // "not defined" diagnostic (CPython future.c). return Err(CompileError::spanned( format!("future feature {} is not defined", a.name), self.alias_span(stmt, &a.name), @@ -550,18 +726,30 @@ impl Validator<'_> { fn visit_pattern(&mut self, pattern: &Pattern) -> Result<(), CompileError> { match pattern { Pattern::Value(e) => self.visit_expr(e)?, + Pattern::Capture(Some(n)) | Pattern::Star(Some(n)) => { + let n = n.clone(); + self.mark_assigned(&n); + } Pattern::Sequence(items) | Pattern::Or(items) => { for p in items { self.visit_pattern(p)?; } } - Pattern::Mapping { keys, patterns, .. } => { + Pattern::Mapping { + keys, + patterns, + rest, + } => { for k in keys { self.visit_expr(k)?; } for p in patterns { self.visit_pattern(p)?; } + if let Some(Some(n)) = rest { + let n = n.clone(); + self.mark_assigned(&n); + } } Pattern::Class { cls, @@ -576,7 +764,11 @@ impl Validator<'_> { self.visit_pattern(p)?; } } - Pattern::As { pattern, .. } => self.visit_pattern(pattern)?, + Pattern::As { pattern, name } => { + self.visit_pattern(pattern)?; + let name = name.clone(); + self.mark_assigned(&name); + } _ => {} } Ok(()) @@ -584,8 +776,12 @@ impl Validator<'_> { fn visit_expr(&mut self, expr: &Expr) -> Result<(), CompileError> { match &expr.kind { + ExprKind::Name(n) => { + let n = n.clone(); + self.mark_use(&n); + } ExprKind::Lambda { args, body } => { - self.visit_function(args, &[], &[], true)?; + self.visit_function(args, &[], &[], None)?; // Lambda bodies are expressions; visit inside a // function scope for nested checks. let mut params: Vec = Vec::new(); @@ -600,10 +796,9 @@ impl Validator<'_> { params.push(a.name.clone()); } self.scopes.push(Scope { - kind: ScopeKind::Function, params: params.clone(), - directives: Vec::new(), bound: params.into_iter().collect(), + ..Scope::new(ScopeKind::Function) }); let result = self.visit_expr(body); self.scopes.pop(); @@ -645,7 +840,12 @@ impl Validator<'_> { self.visit_expr(orelse)?; } ExprKind::NamedExpr { target, value } => { - self.visit_expr(target)?; + if let ExprKind::Name(n) = &target.kind { + let n = n.clone(); + self.mark_assigned(&n); + } else { + self.visit_expr(target)?; + } self.visit_expr(value)?; } ExprKind::Call { @@ -714,10 +914,29 @@ impl Validator<'_> { &mut self, generators: &[Comprehension], elements: &[&Expr], + ) -> Result<(), CompileError> { + // The outermost iterable evaluates in the enclosing scope; + // everything else lives in the comprehension's implicit + // function scope, so reads there don't count as uses of the + // enclosing scope (`[x for y in q]` then `global x` is fine, + // `[1 for y in x]` then `global x` is not). + if let Some(first) = generators.first() { + self.visit_expr(&first.iter)?; + } + self.scopes.push(Scope::new(ScopeKind::Comprehension)); + let result = self.visit_comprehension_inner(generators, elements); + self.scopes.pop(); + result + } + + fn visit_comprehension_inner( + &mut self, + generators: &[Comprehension], + elements: &[&Expr], ) -> Result<(), CompileError> { let mut iter_vars: Vec = Vec::new(); let mut walrus_vars: Vec = Vec::new(); - for g in generators { + for (gi, g) in generators.iter().enumerate() { // Iteration target: reject names already bound by a walrus // earlier in this comprehension. let mut targets: Vec<(&str, Span)> = Vec::new(); @@ -733,8 +952,15 @@ impl Validator<'_> { )); } iter_vars.push((*name).to_owned()); + // Iteration variables bind in the comprehension scope + // itself, not the enclosing one. + let s = self.scope_mut(); + s.assigned.insert((*name).to_owned()); + s.bound.insert((*name).to_owned()); + } + if gi > 0 { + self.visit_expr(&g.iter)?; } - self.visit_expr(&g.iter)?; self.check_walrus(&g.iter, &iter_vars, &mut walrus_vars)?; for cond in &g.ifs { self.visit_expr(cond)?; @@ -752,15 +978,19 @@ impl Validator<'_> { /// comprehension/lambda scopes) and reject rebinds of comprehension /// iteration variables. fn check_walrus( - &self, + &mut self, expr: &Expr, iter_vars: &[String], walrus_vars: &mut Vec, ) -> Result<(), CompileError> { - let mut found: Vec<(&str, Span)> = Vec::new(); - collect_walrus_targets(expr, &mut found); + let mut found: Vec<(String, Span)> = Vec::new(); + { + let mut borrowed: Vec<(&str, Span)> = Vec::new(); + collect_walrus_targets(expr, &mut borrowed); + found.extend(borrowed.into_iter().map(|(n, s)| (n.to_owned(), s))); + } for (name, span) in found { - if iter_vars.iter().any(|v| v == name) { + if iter_vars.iter().any(|v| v == &name) { return Err(CompileError::spanned( format!( "assignment expression cannot rebind comprehension iteration \ @@ -769,7 +999,10 @@ impl Validator<'_> { span, )); } - walrus_vars.push(name.to_owned()); + // Walrus targets bind through the comprehension scope into + // the enclosing function/class/module scope. + self.mark_assigned(&name); + walrus_vars.push(name); } Ok(()) } @@ -800,6 +1033,155 @@ impl Validator<'_> { } } +/// PEP 563: yield / await / named expressions may not appear anywhere +/// inside an annotation once `from __future__ import annotations` is +/// active (CPython symtable's `_check_no_deferred_annotation` rules). +/// Lambdas open a new scope, so their bodies are exempt. +fn check_annotation_expr(expr: &Expr) -> Result<(), CompileError> { + match &expr.kind { + ExprKind::Yield(_) | ExprKind::YieldFrom(_) => { + return Err(CompileError::spanned( + "yield expression cannot be used within an annotation", + expr.span, + )); + } + ExprKind::Await(_) => { + return Err(CompileError::spanned( + "await expression cannot be used within an annotation", + expr.span, + )); + } + ExprKind::NamedExpr { target, .. } => { + return Err(CompileError::spanned( + "named expression cannot be used within an annotation", + target.span, + )); + } + ExprKind::Lambda { args, .. } => { + // The lambda body is a new scope, but its defaults belong + // to the annotation's own scope. + for d in args + .defaults + .iter() + .chain(args.kw_defaults.iter().flatten()) + { + check_annotation_expr(d)?; + } + return Ok(()); + } + _ => {} + } + let mut result = Ok(()); + for_each_child_expr(expr, &mut |child| { + if result.is_ok() { + result = check_annotation_expr(child); + } + }); + result +} + +/// Call `f` on every direct child expression of `expr`. +fn for_each_child_expr<'a>(expr: &'a Expr, f: &mut dyn FnMut(&'a Expr)) { + match &expr.kind { + ExprKind::BoolOp { values, .. } => values.iter().for_each(f), + ExprKind::BinOp { left, right, .. } => { + f(left); + f(right); + } + ExprKind::UnaryOp { operand, .. } => f(operand), + ExprKind::Lambda { args, body } => { + for d in &args.defaults { + f(d); + } + for d in args.kw_defaults.iter().flatten() { + f(d); + } + f(body); + } + ExprKind::IfExp { test, body, orelse } => { + f(test); + f(body); + f(orelse); + } + ExprKind::Dict { keys, values } => { + keys.iter().flatten().for_each(&mut *f); + values.iter().for_each(f); + } + ExprKind::ListComp { elt, generators } + | ExprKind::SetComp { elt, generators } + | ExprKind::GeneratorExp { elt, generators } => { + f(elt); + for g in generators { + f(&g.target); + f(&g.iter); + g.ifs.iter().for_each(&mut *f); + } + } + ExprKind::DictComp { + key, + value, + generators, + } => { + f(key); + f(value); + for g in generators { + f(&g.target); + f(&g.iter); + g.ifs.iter().for_each(&mut *f); + } + } + ExprKind::Compare { + left, comparators, .. + } => { + f(left); + comparators.iter().for_each(f); + } + ExprKind::Call { + func, + args, + keywords, + } => { + f(func); + args.iter().for_each(&mut *f); + for k in keywords { + f(&k.value); + } + } + ExprKind::NamedExpr { target, value } => { + f(target); + f(value); + } + ExprKind::Attribute { value, .. } => f(value), + ExprKind::Subscript { value, slice } => { + f(value); + f(slice); + } + ExprKind::Slice { lower, upper, step } => { + [lower, upper, step] + .into_iter() + .flatten() + .for_each(|e| f(e)); + } + ExprKind::Tuple(items) | ExprKind::List(items) | ExprKind::Set(items) => { + items.iter().for_each(f); + } + ExprKind::Starred(inner) + | ExprKind::Yield(Some(inner)) + | ExprKind::YieldFrom(inner) + | ExprKind::Await(inner) => f(inner), + ExprKind::JoinedStr(parts) => parts.iter().for_each(f), + ExprKind::FormattedValue { + value, format_spec, .. + } => { + f(value); + if let Some(s) = format_spec { + f(s); + } + } + _ => {} + } +} + fn collect_name_targets<'a>(expr: &'a Expr, out: &mut Vec<(&'a str, Span)>) { match &expr.kind { ExprKind::Name(n) => out.push((n, expr.span)), diff --git a/crates/weavepy-lexer/src/error.rs b/crates/weavepy-lexer/src/error.rs index cd053c7..9dc5b5e 100644 --- a/crates/weavepy-lexer/src/error.rs +++ b/crates/weavepy-lexer/src/error.rs @@ -5,8 +5,21 @@ use thiserror::Error; /// Errors produced by [`crate::tokenize`]. #[derive(Debug, Clone, Error, PartialEq, Eq)] pub enum LexError { - #[error("unterminated string literal at byte {pos}")] - UnterminatedString { pos: u32 }, + #[error("unterminated string literal (detected at line {detected_line})")] + UnterminatedString { pos: u32, detected_line: u32 }, + /// The string ran to EOL/EOF right after a backslash-escaped copy of + /// its own quote character (`"blech\"`): CPython appends a hint. + #[error("unterminated string literal (detected at line {detected_line}); perhaps you escaped the end quote?")] + UnterminatedStringEscapedQuote { pos: u32, detected_line: u32 }, + #[error("unterminated triple-quoted string literal (detected at line {detected_line})")] + UnterminatedTripleString { pos: u32, detected_line: u32 }, + /// A `\` line continuation at end of input (bpo-2180). CPython's + /// tokenizer reports this as an EOF error, anchored one column past + /// the backslash. `line_had_tokens` mirrors CPython's file + /// tokenizer, which reports column 0 (suppressing the traceback + /// caret) when no token preceded the continuation. + #[error("unexpected EOF while parsing")] + UnexpectedEofParsing { pos: u32, line_had_tokens: bool }, // PEP 701 f-string diagnostics. CPython distinguishes an unterminated // f-string *literal* from an unterminated *replacement field* and uses // f-string-specific wording, which several `test_fstring` negative @@ -59,6 +72,19 @@ pub enum LexError { FstringUnmatchedParen { close: char, pos: u32 }, #[error("'{open}' was never closed")] BracketNeverClosed { open: char, pos: u32 }, + /// A closer with no bracket open at all (`)1 + 2`). + #[error("unmatched '{close}'")] + UnmatchedClose { close: char, pos: u32 }, + /// A closer that doesn't pair with the innermost opener. CPython + /// appends " on line N" when the opener sits on an earlier line. + #[error("closing parenthesis '{close}' does not match opening parenthesis '{open}'{suffix}", suffix = open_line.map(|l| format!(" on line {l}")).unwrap_or_default())] + MismatchedClose { + close: char, + open: char, + /// `Some(line)` only when the opener is on a different line. + open_line: Option, + pos: u32, + }, #[error("f-string: newlines are not allowed in format specifiers for single quoted f-strings")] FstringNewlineInSpec { pos: u32 }, // CPython renders this as `invalid character '€' (U+20AC)` — the @@ -80,6 +106,10 @@ pub enum LexError { InconsistentIndent { pos: u32 }, #[error("unindent does not match any outer indentation level")] UnknownDedent { pos: u32 }, + /// More than `MAXINDENT` (100) nested indentation levels — CPython's + /// `E_TOODEEP`, an `IndentationError`. + #[error("too many levels of indentation")] + TooDeepIndent { pos: u32 }, /// Malformed numeric literal. `message` carries CPython's exact /// wording ("invalid hexadecimal literal", "invalid digit '9' in /// octal literal", "leading zeros in decimal integer literals…"); @@ -100,7 +130,10 @@ impl LexError { /// compute the `SyntaxError` line/column at the raise site. pub fn byte_offset(&self) -> u32 { match self { - LexError::UnterminatedString { pos } + LexError::UnterminatedString { pos, .. } + | LexError::UnterminatedStringEscapedQuote { pos, .. } + | LexError::UnterminatedTripleString { pos, .. } + | LexError::UnexpectedEofParsing { pos, .. } | LexError::UnterminatedFstring { pos } | LexError::UnterminatedTripleFstring { pos } | LexError::FstringExpectingBrace { pos, .. } @@ -111,12 +144,15 @@ impl LexError { | LexError::FstringParenMismatch { pos, .. } | LexError::FstringUnmatchedParen { pos, .. } | LexError::BracketNeverClosed { pos, .. } + | LexError::UnmatchedClose { pos, .. } + | LexError::MismatchedClose { pos, .. } | LexError::FstringNewlineInSpec { pos } | LexError::InvalidChar { pos, .. } | LexError::InvalidNonPrintable { pos, .. } | LexError::InvalidToken { pos } | LexError::InconsistentIndent { pos } | LexError::UnknownDedent { pos } + | LexError::TooDeepIndent { pos } | LexError::InvalidNumber { pos, .. } | LexError::InvalidStringPrefix { pos, .. } | LexError::StrayBackslash { pos } diff --git a/crates/weavepy-lexer/src/lib.rs b/crates/weavepy-lexer/src/lib.rs index 2cafbf2..2096d79 100644 --- a/crates/weavepy-lexer/src/lib.rs +++ b/crates/weavepy-lexer/src/lib.rs @@ -313,3 +313,13 @@ mod tests { assert!(lex_err_msg("f'{(\"x'").starts_with("unterminated string literal")); } } + +#[cfg(test)] +mod flufl_probe_tests { + #[test] + fn lexes_lessgreater_as_notequal() { + let toks = crate::tokenize("2 <> 3\n").unwrap(); + let kinds: Vec<_> = toks.iter().map(|t| format!("{:?}", t.kind)).collect(); + assert!(kinds.contains(&"NotEqual".to_string()), "{kinds:?}"); + } +} diff --git a/crates/weavepy-lexer/src/scanner.rs b/crates/weavepy-lexer/src/scanner.rs index 7525939..cfcf9a9 100644 --- a/crates/weavepy-lexer/src/scanner.rs +++ b/crates/weavepy-lexer/src/scanner.rs @@ -307,6 +307,22 @@ impl<'src> Scanner<'src> { } if matches!(self.peek(), Some(b'\n')) { self.pos += 1; + // A continuation whose next line never arrives (EOF + // right after the swallowed newline): CPython reports + // an EOF error rather than silently completing the + // statement — unless a bracket is open, which wins. + if self.peek().is_none() { + if let Some(&(bracket, pos)) = self.open_brackets.first() { + return Err(LexError::BracketNeverClosed { + open: bracket as char, + pos: pos as u32, + }); + } + return Err(LexError::UnexpectedEofParsing { + pos: self.pos as u32, + line_had_tokens: self.last_was_content, + }); + } // Skip the newline; do not start a new logical line. return Ok(None); } @@ -320,10 +336,12 @@ impl<'src> Scanner<'src> { pos: pos as u32, }); } - // CPython anchors the error at the backslash itself when - // nothing follows, and at the offending character when - // one does. - return Err(LexError::StrayBackslash { pos: bs_pos as u32 }); + // bpo-2180: a line continuation ending the input is an + // EOF error, anchored one column past the backslash. + return Err(LexError::UnexpectedEofParsing { + pos: (bs_pos + 1) as u32, + line_had_tokens: self.last_was_content, + }); } return Err(LexError::StrayBackslash { pos: (bs_pos + 1) as u32, @@ -414,6 +432,41 @@ impl<'src> Scanner<'src> { self.at_line_start = false; return Ok(()); } + // issue-40847: a line holding only whitespace and `\` + // continuations whose joined logical line is still blank is a + // blank line — no INDENT/DEDENT processing. Probe ahead without + // committing; if content ever appears, fall through to normal + // indent handling (the continuation is consumed by the main + // loop as part of the logical line). + if b == b'\\' { + let mut probe = self.pos; + let blank_joined = loop { + if self.src.get(probe) != Some(&b'\\') { + break false; + } + let mut q = probe + 1; + if self.src.get(q) == Some(&b'\r') { + q += 1; + } + if self.src.get(q) != Some(&b'\n') { + break false; + } + probe = q + 1; + while matches!(self.src.get(probe), Some(b' ' | b'\t' | 0x0C)) { + probe += 1; + } + match self.src.get(probe) { + None | Some(b'\n' | b'\r' | b'#') => break true, + Some(b'\\') => continue, + _ => break false, + } + }; + if blank_joined { + self.pos = probe; + self.at_line_start = false; + return Ok(()); + } + } let tab_error = || LexError::InconsistentIndent { pos: line_start as u32, @@ -421,6 +474,17 @@ impl<'src> Scanner<'src> { let current = *self.indents.last().expect("indent stack non-empty"); let alt_current = *self.alt_indents.last().expect("indent stack non-empty"); if indent > current { + // CPython's MAXINDENT (100): the fixed `indstack` is full + // when a 101st level would be pushed (E_TOODEEP). + if self.indents.len() >= 100 { + let mut line_end = self.pos; + while line_end < self.src.len() && self.src[line_end] != b'\n' { + line_end += 1; + } + return Err(LexError::TooDeepIndent { + pos: line_end as u32, + }); + } if alt <= alt_current { return Err(tab_error()); } @@ -588,6 +652,23 @@ impl<'src> Scanner<'src> { self.pos += 1; match self.peek() { Some(b) if valid(b) => {} + // `0b1_2`: a decimal digit invalid for the + // radix keeps CPython's "invalid digit" + // wording, not the bare-literal one. + Some(b) + if b.is_ascii_digit() + && radix_char != b'x' + && radix_char != b'X' => + { + self.pos += 1; + return Err(LexError::InvalidNumber { + pos: (self.pos - 1) as u32, + message: format!( + "invalid digit '{}' in {radix_name} literal", + b as char + ), + }); + } _ => { return Err(LexError::InvalidNumber { pos: (self.pos - 1) as u32, @@ -674,19 +755,27 @@ impl<'src> Scanner<'src> { } if matches!(self.peek(), Some(b'e' | b'E')) { - // Only a real exponent: `1e3`, `1e+3`. A bare `1e` or `1e+` - // is "invalid decimal literal" at the last consumed byte. - is_float = true; + // Only a real exponent: `1e3`, `1e+3`. A signed exponent + // with no digits (`1e+`) is "invalid decimal literal", but + // a bare `e` not followed by sign/digit is *backed up* like + // CPython's tokenizer — `1.else` is the float `1.` followed + // by the keyword `else` (issue 21642). + let exp_start = self.pos; self.pos += 1; - if matches!(self.peek(), Some(b'+' | b'-')) { + let signed = matches!(self.peek(), Some(b'+' | b'-')); + if signed { self.pos += 1; } - let got = consume_digit_run(self)?; - if !got { + if matches!(self.peek(), Some(b) if b.is_ascii_digit()) { + is_float = true; + consume_digit_run(self)?; + } else if signed { return Err(LexError::InvalidNumber { pos: (self.pos - 1) as u32, message: "invalid decimal literal".to_owned(), }); + } else { + self.pos = exp_start; } } @@ -712,7 +801,12 @@ impl<'src> Scanner<'src> { } } - self.verify_end_of_number("invalid decimal literal")?; + let end_msg = if is_imaginary { + "invalid imaginary literal" + } else { + "invalid decimal literal" + }; + self.verify_end_of_number(end_msg)?; Ok(self.token(TokenKind::Number, start, self.pos)) } @@ -720,16 +814,18 @@ impl<'src> Scanner<'src> { /// by an identifier character is a syntax error ("invalid decimal /// literal" at the number's last byte) — except when the trailing /// identifier is a keyword that may legally follow a number - /// (`1if x else y`, `0in xs`, …). + /// (`1if x else y`, `0in xs`, …), which today only draws a + /// SyntaxWarning with the same message. A non-ASCII follower is + /// left for the main loop (it becomes "invalid character …"). fn verify_end_of_number(&mut self, message: &str) -> Result<(), LexError> { - const ALLOWED: &[&str] = &["and", "else", "for", "if", "in", "is", "not", "or", "while"]; + const ALLOWED: &[&str] = &["and", "else", "for", "if", "in", "is", "not", "or"]; let next = match self.peek() { Some(b) => b, None => return Ok(()), }; let is_ident_byte = |b: u8| b == b'_' || b.is_ascii_alphabetic() || b.is_ascii_digit() || b >= 0x80; - if !(next == b'_' || next.is_ascii_alphabetic() || next >= 0x80) { + if !(next == b'_' || next.is_ascii_alphabetic()) { return Ok(()); } let mut end = self.pos; @@ -738,6 +834,10 @@ impl<'src> Scanner<'src> { } let word = std::str::from_utf8(&self.src[self.pos..end]).unwrap_or(""); if ALLOWED.contains(&word) { + self.escape_warnings.push(EscapeWarning { + offset: self.pos as u32, + message: message.to_owned(), + }); return Ok(()); } Err(LexError::InvalidNumber { @@ -1196,14 +1296,14 @@ impl<'src> Scanner<'src> { // "expecting '}'" at) is the *opening* quote — the field's // expression text ends just before it. let quote_pos = self.pos as u32; - let unterminated = |pos: u32| { + let unterminated = |pos: u32, detected_line: u32| { if quote == outer_quote { LexError::FstringExpectingBrace { pos: quote_pos, field_start, } } else { - LexError::UnterminatedString { pos } + LexError::UnterminatedString { pos, detected_line } } }; // Walk back over the immediately-preceding ASCII-letter run to @@ -1234,7 +1334,7 @@ impl<'src> Scanner<'src> { let _ = prefix.raw; loop { let Some(b) = self.peek() else { - return Err(unterminated(self.pos as u32)); + return Err(unterminated(self.pos as u32, self.line_of(self.pos))); }; if b == b'\\' { // A backslash escapes the next byte for tokenizing in raw @@ -1258,7 +1358,7 @@ impl<'src> Scanner<'src> { return Ok(()); } if (b == b'\n' || b == b'\r') && !triple { - return Err(unterminated(self.pos as u32)); + return Err(unterminated(self.pos as u32, self.line_of(self.pos))); } self.pos += 1; } @@ -1272,9 +1372,30 @@ impl<'src> Scanner<'src> { ) -> Result { let raw = prefix.raw; let mut warned = false; + // When the last thing consumed before EOL/EOF was a + // backslash-escaped copy of the closing quote (`"blech\"`), + // CPython appends "; perhaps you escaped the end quote?". + let escaped_end_quote = |scanner: &Self| { + scanner.pos >= 2 + && scanner.src[scanner.pos - 1] == quote + && scanner.src[scanner.pos - 2] == b'\\' + }; + let unterminated = |scanner: &Self| { + if escaped_end_quote(scanner) { + LexError::UnterminatedStringEscapedQuote { + pos: start as u32, + detected_line: scanner.line_of(scanner.pos), + } + } else { + LexError::UnterminatedString { + pos: start as u32, + detected_line: scanner.line_of(scanner.pos), + } + } + }; while let Some(b) = self.peek() { if b == b'\n' || b == b'\r' { - return Err(LexError::UnterminatedString { pos: start as u32 }); + return Err(unterminated(self)); } if b == b'\\' && !raw { if !warned { @@ -1316,7 +1437,7 @@ impl<'src> Scanner<'src> { } self.pos += 1; } - Err(LexError::UnterminatedString { pos: start as u32 }) + Err(unterminated(self)) } fn scan_triple_string( @@ -1329,7 +1450,10 @@ impl<'src> Scanner<'src> { let mut warned = false; loop { let Some(b) = self.peek() else { - return Err(LexError::UnterminatedString { pos: start as u32 }); + return Err(LexError::UnterminatedTripleString { + pos: start as u32, + detected_line: self.line_of(self.pos), + }); }; if b == b'\\' { // Backslash escapes the next byte for tokenizing in raw @@ -1392,6 +1516,11 @@ impl<'src> Scanner<'src> { (b'>', b'>') => Some(TokenKind::RightShift), (b'=', b'=') => Some(TokenKind::EqEqual), (b'!', b'=') => Some(TokenKind::NotEqual), + // PEP 401: `<>` lexes as NOTEQUAL, like CPython's + // tokenizer; the parser rejects it unless + // `barry_as_FLUFL` is active (and rejects `!=` when it + // is). + (b'<', b'>') => Some(TokenKind::NotEqual), (b'<', b'=') => Some(TokenKind::LessEqual), (b'>', b'=') => Some(TokenKind::GreaterEqual), (b'+', b'=') => Some(TokenKind::PlusEqual), @@ -1421,8 +1550,7 @@ impl<'src> Scanner<'src> { TokenKind::LPar } b')' => { - self.paren_depth = self.paren_depth.saturating_sub(1); - self.open_brackets.pop(); + self.close_bracket(b')', start)?; TokenKind::RPar } b'[' => { @@ -1431,8 +1559,7 @@ impl<'src> Scanner<'src> { TokenKind::LSqb } b']' => { - self.paren_depth = self.paren_depth.saturating_sub(1); - self.open_brackets.pop(); + self.close_bracket(b']', start)?; TokenKind::RSqb } b'{' => { @@ -1441,8 +1568,7 @@ impl<'src> Scanner<'src> { TokenKind::LBrace } b'}' => { - self.paren_depth = self.paren_depth.saturating_sub(1); - self.open_brackets.pop(); + self.close_bracket(b'}', start)?; TokenKind::RBrace } b',' => TokenKind::Comma, @@ -1469,10 +1595,15 @@ impl<'src> Scanner<'src> { let ch = decode_utf8(&self.src[self.pos..]) .map(|(c, _)| c) .unwrap_or('\u{FFFD}'); - // CPython wording: ASCII junk (`$`, `?`, `` ` ``) is a - // plain "invalid syntax"; only non-ASCII gets the - // `invalid character '€' (U+20AC)` message. + // CPython wording: printable ASCII junk (`$`, `?`, + // `` ` ``) is a plain "invalid syntax", but ASCII + // control characters get the non-printable diagnostic + // (`invalid non-printable character U+0017`). Only + // non-ASCII gets `invalid character '€' (U+20AC)`. if ch.is_ascii() { + if ch.is_control() { + return Err(LexError::InvalidNonPrintable { ch, pos }); + } return Err(LexError::InvalidToken { pos }); } // CPython distinguishes printable junk (`invalid character @@ -1499,6 +1630,42 @@ impl<'src> Scanner<'src> { // ---------- helpers ---------- + /// Pop the bracket stack for a closer, producing CPython's + /// "unmatched ')'" (nothing open) or "closing parenthesis ')' does + /// not match opening parenthesis '['" (wrong opener) diagnostics. + fn close_bracket(&mut self, close: u8, start: usize) -> Result<(), LexError> { + let Some((open, open_pos)) = self.open_brackets.pop() else { + return Err(LexError::UnmatchedClose { + close: close as char, + pos: start as u32, + }); + }; + let expected = match open { + b'(' => b')', + b'[' => b']', + _ => b'}', + }; + if close != expected { + let open_line = self.line_of(open_pos); + let close_line = self.line_of(start); + return Err(LexError::MismatchedClose { + close: close as char, + open: open as char, + open_line: (open_line != close_line).then_some(open_line), + pos: start as u32, + }); + } + self.paren_depth = self.paren_depth.saturating_sub(1); + Ok(()) + } + + /// 1-based line number containing byte `pos` — the "detected at + /// line N" in CPython's unterminated-string diagnostics. + fn line_of(&self, pos: usize) -> u32 { + let end = pos.min(self.src.len()); + self.src[..end].iter().filter(|b| **b == b'\n').count() as u32 + 1 + } + fn token(&self, kind: TokenKind, start: usize, end: usize) -> Token { Token { kind, diff --git a/crates/weavepy-parser/src/ast.rs b/crates/weavepy-parser/src/ast.rs index 3151e87..c5c35dd 100644 --- a/crates/weavepy-parser/src/ast.rs +++ b/crates/weavepy-parser/src/ast.rs @@ -69,6 +69,19 @@ pub enum StmtKind { /// PEP 695 type parameters (`class C[T](…)`). type_params: Vec, }, + /// PEP 695 `type Name[T, …] = value`. Kept first-class in the + /// parse AST so `ast.parse` and `symtable` see the real node; + /// the compiler lowers it to the lazy `__weavepy_type_alias__` + /// assignment (see `parser::lower_type_alias_stmt`) before any + /// other pass runs. + TypeAlias { + name: String, + /// Span of the alias-name token (for `ast.TypeAlias.name`). + name_span: Span, + /// PEP 695 type parameters (`type X[T, U] = …`). + type_params: Vec, + value: Box, + }, /// `return value` Return(Option), /// `target = value` (and multi-target: `a = b = c = ...`) @@ -781,6 +794,33 @@ fn dump_stmt(out: &mut String, s: &Stmt, depth: usize) { } out.push_str(", type_comment=None)"); } + S::TypeAlias { + name, + type_params, + value, + .. + } => { + out.push_str("TypeAlias(name=Name(id='"); + out.push_str(name); + out.push_str("', ctx=Store()), type_params=["); + for (i, tp) in type_params.iter().enumerate() { + if i > 0 { + out.push_str(", "); + } + let ctor = match tp.kind { + TypeParamKind::TypeVar { .. } => "TypeVar", + TypeParamKind::TypeVarTuple => "TypeVarTuple", + TypeParamKind::ParamSpec => "ParamSpec", + }; + out.push_str(ctor); + out.push_str("(name='"); + out.push_str(&tp.source_name); + out.push_str("')"); + } + out.push_str("], value="); + dump_expr(out, value, depth); + out.push(')'); + } S::AsyncFunctionDef { name, args, diff --git a/crates/weavepy-parser/src/error.rs b/crates/weavepy-parser/src/error.rs index 0c00edf..10545e2 100644 --- a/crates/weavepy-parser/src/error.rs +++ b/crates/weavepy-parser/src/error.rs @@ -69,7 +69,8 @@ impl ParseError { /// otherwise. pub fn exception_class(&self) -> &'static str { match self { - ParseError::Indentation { .. } | ParseError::Lex(LexError::UnknownDedent { .. }) => { + ParseError::Indentation { .. } + | ParseError::Lex(LexError::UnknownDedent { .. } | LexError::TooDeepIndent { .. }) => { "IndentationError" } ParseError::Lex(LexError::InconsistentIndent { .. }) => "TabError", diff --git a/crates/weavepy-parser/src/lib.rs b/crates/weavepy-parser/src/lib.rs index d9c322c..70eff12 100644 --- a/crates/weavepy-parser/src/lib.rs +++ b/crates/weavepy-parser/src/lib.rs @@ -20,7 +20,9 @@ pub mod unparse; pub use ast::{dump_module, Module}; pub use error::ParseError; -pub use parser::{set_unicode_name_resolver, UnicodeNameResolution}; +pub use parser::{ + build_lazy_type_alias, lower_type_alias_stmt, set_unicode_name_resolver, UnicodeNameResolution, +}; pub use weavepy_lexer::EscapeWarning; /// Parse a Python source buffer into a [`Module`]. @@ -39,7 +41,23 @@ pub fn parse_module(source: &str) -> Result { pub fn parse_module_with_warnings( source: &str, ) -> (Result, Vec) { - parse_with_warnings(source, parser::parse) + parse_module_with_warnings_flags(source, false) +} + +/// [`parse_module_with_warnings`] with PEP 401 `barry_as_FLUFL` +/// pre-activated when the caller passed `CO_FUTURE_BARRY_AS_BDFL` to +/// `compile()` (the parser also self-activates on the future import). +pub fn parse_module_with_warnings_flags( + source: &str, + flufl: bool, +) -> (Result, Vec) { + if flufl { + parse_with_warnings(source, |src, tokens| { + parser::parse_with_flufl(src, tokens, true) + }) + } else { + parse_with_warnings(source, parser::parse) + } } /// Like [`parse_module_with_warnings`], but with CPython's `eval` start @@ -48,7 +66,54 @@ pub fn parse_module_with_warnings( /// accept, never a statement-level diagnostic. Backs `eval(...)` and /// `compile(..., mode="eval")`. pub fn parse_eval_with_warnings(source: &str) -> (Result, Vec) { - parse_with_warnings(source, parser::parse_eval) + parse_eval_with_warnings_flags(source, false) +} + +/// [`parse_eval_with_warnings`] with PEP 401 `barry_as_FLUFL` +/// pre-activated. +pub fn parse_eval_with_warnings_flags( + source: &str, + flufl: bool, +) -> (Result, Vec) { + let (mut result, warnings) = if flufl { + parse_with_warnings(source, |src, tokens| { + parser::parse_eval_with_flufl(src, tokens, true) + }) + } else { + parse_with_warnings(source, parser::parse_eval) + }; + // Eval-mode line-continuation at hard EOF: `compile()` appends a + // newline in exec mode (so `"\\"` reads as continuation-then-EOF, + // "unexpected EOF while parsing"), but the eval grammar tokenizes + // the source as-is, and CPython reports the stray backslash itself + // ("unexpected character after line continuation character") — even + // when a bracket is still open (`eval("(\\")`). + if let Some(stripped) = source.strip_suffix('\\') { + let eof_shaped = matches!( + &result, + Err(ParseError::Lex( + weavepy_lexer::LexError::UnexpectedEofParsing { .. } + | weavepy_lexer::LexError::BracketNeverClosed { .. } + )) + ); + // The backslash only counts if it is *live* syntax (not swallowed + // by a comment or string). Probe by appending a junk byte: a live + // continuation-backslash then trips StrayBackslash right there. + let live_backslash = || { + let probe = format!("{source}x"); + matches!( + weavepy_lexer::tokenize(&probe), + Err(weavepy_lexer::LexError::StrayBackslash { pos }) + if pos as usize == source.len() + ) + }; + if eof_shaped && live_backslash() { + result = Err(ParseError::Lex(weavepy_lexer::LexError::StrayBackslash { + pos: stripped.len() as u32, + })); + } + } + (result, warnings) } fn parse_with_warnings( diff --git a/crates/weavepy-parser/src/parser.rs b/crates/weavepy-parser/src/parser.rs index bcc1ca5..5d32b65 100644 --- a/crates/weavepy-parser/src/parser.rs +++ b/crates/weavepy-parser/src/parser.rs @@ -22,7 +22,19 @@ use crate::ast::{ use crate::error::ParseError; pub(crate) fn parse(source: &str, tokens: Vec) -> Result { + parse_with_flufl(source, tokens, false) +} + +/// [`parse`] with PEP 401 `barry_as_FLUFL` pre-activated (the +/// `CO_FUTURE_BARRY_AS_BDFL` compile flag). The parser also flips the +/// flag itself when it sees `from __future__ import barry_as_FLUFL`. +pub(crate) fn parse_with_flufl( + source: &str, + tokens: Vec, + flufl: bool, +) -> Result { let mut p = Parser::new(source, tokens); + p.flufl = flufl; let module = p.parse_module()?; Ok(module) } @@ -37,7 +49,17 @@ pub(crate) fn parse(source: &str, tokens: Vec) -> Result) -> Result { + parse_eval_with_flufl(source, tokens, false) +} + +/// [`parse_eval`] with PEP 401 `barry_as_FLUFL` pre-activated. +pub(crate) fn parse_eval_with_flufl( + source: &str, + tokens: Vec, + flufl: bool, +) -> Result { let mut p = Parser::new(source, tokens); + p.flufl = flufl; let module = p.parse_eval_module()?; Ok(module) } @@ -46,6 +68,11 @@ struct Parser<'src> { source: &'src str, tokens: Vec, pos: usize, + /// PEP 401: `from __future__ import barry_as_FLUFL` is active + /// (either seen while parsing, or passed as a compile flag). Under + /// FLUFL, `<>` is the inequality operator and `!=` is a + /// SyntaxError. + flufl: bool, } impl<'src> Parser<'src> { @@ -64,6 +91,7 @@ impl<'src> Parser<'src> { source, tokens, pos: 0, + flufl: false, } } @@ -133,9 +161,18 @@ impl<'src> Parser<'src> { if self.check(k) { Ok(self.bump()) } else { + // CPython pegen only names the missing token when the line + // simply stopped short (NEWLINE / EOF): `try` ⏎ says + // "expected ':'". Anywhere else the generic parse failure is + // a bare "invalid syntax" pointing at the offending token. + let message = if matches!(self.peek(), TokenKind::Newline | TokenKind::Endmarker) { + format!("expected {}", what.replace('`', "'")) + } else { + "invalid syntax".to_owned() + }; Err(ParseError::Unexpected { span: self.peek_token().span, - message: format!("expected {what}, got {:?}", self.peek()), + message, }) } } @@ -235,18 +272,37 @@ impl<'src> Parser<'src> { /// of '='?", anchored at the key. fn expect_dict_colon(&mut self, key: &Expr) -> Result<(), ParseError> { if self.check(&TokenKind::Equal) { + return Err(Self::display_equal_error(key)); + } + if !self.check(&TokenKind::Colon) { + // pegen `invalid_dict_key_value` ("{1:2, 3:4, 5}"). return Err(ParseError::Unexpected { - span: key.span, - message: format!( - "cannot assign to {} here. Maybe you meant '==' instead of '='?", - crate::ast::expr_name(key) - ), + span: self.peek_token().span, + message: "':' expected after dictionary key".to_owned(), }); } - self.expect(&TokenKind::Colon, "`:`")?; + self.bump(); Ok(()) } + /// Parse a dict value (the expression after `key:`) with pegen's + /// dedicated diagnostics for a missing or starred value. + fn parse_dict_value(&mut self) -> Result { + if let TokenKind::Star = self.peek() { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "cannot use a starred expression in a dictionary value".to_owned(), + }); + } + if matches!(self.peek(), TokenKind::RBrace | TokenKind::Comma) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "expression expected after dictionary key and ':'".to_owned(), + }); + } + self.parse_ternary() + } + /// Like [`Parser::expect`], but when the failure is two adjacent /// expressions (e.g. `[a b]`), report CPython's /// "invalid syntax. Perhaps you forgot a comma?" spanning from the @@ -444,7 +500,18 @@ impl<'src> Parser<'src> { if self.looks_like_match_statement() { self.parse_match() } else { - self.parse_simple_statement() + // pegen tries the regular statement grammar first, + // then falls back to `invalid_match_stmt`, so + // `match x` ⏎ reports "expected ':'" (not the + // generic failure of the expression path). + let saved = self.pos; + match self.parse_simple_statement() { + Ok(stmt) => Ok(stmt), + Err(expr_err) => { + self.pos = saved; + Err(self.match_stmt_fallback_error(expr_err)) + } + } } } // PEP 695 — `type Alias = T` soft keyword. Disambiguate @@ -494,17 +561,11 @@ impl<'src> Parser<'src> { matches!(self.tokens.get(i).map(|t| &t.kind), Some(TokenKind::Equal)) } - /// Compile a PEP 695 type-alias statement. - /// - /// `type Foo[T, U] = body` desugars to: - /// - /// ```python - /// Foo = (lambda T, U: body)(TypeVar('T'), TypeVar('U')) - /// ``` - /// - /// so the type parameters resolve as `TypeVar` instances in the - /// alias body without leaking into the enclosing scope. The - /// bare form `type Foo = body` lowers to plain `Foo = body`. + /// Parse a PEP 695 type-alias statement into the first-class + /// [`StmtKind::TypeAlias`] node. The compiler lowers it to the + /// lazy `__weavepy_type_alias__` assignment via + /// [`lower_type_alias_stmt`] just before compilation, so + /// `ast.parse` and `symtable` observe the real node. fn parse_type_alias_stmt(&mut self) -> Result { let type_tok = self.bump(); // `type` let name_tok = self.expect(&TokenKind::Name, "type alias name")?; @@ -515,15 +576,12 @@ impl<'src> Parser<'src> { check_type_param_expr(&value, "a type alias")?; self.consume_stmt_end()?; let span = type_tok.span.merge(value.span); - let target = Expr { - kind: ExprKind::Name(name.clone()), - span: name_tok.span, - }; - let rhs = build_lazy_type_alias(&name, value, &type_params, name_tok.span); Ok(Stmt { - kind: StmtKind::Assign { - targets: vec![target], - value: rhs, + kind: StmtKind::TypeAlias { + name, + name_span: name_tok.span, + type_params, + value: Box::new(value), }, span, }) @@ -535,7 +593,14 @@ impl<'src> Parser<'src> { if !matches!(self.peek(), TokenKind::LSqb) { return Ok(Vec::new()); } - self.bump(); // `[` + let lsqb = self.bump(); // `[` + // pegen `invalid_type_params`. + if matches!(self.peek(), TokenKind::RSqb) { + return Err(ParseError::Unexpected { + span: lsqb.span.merge(self.peek_token().span), + message: "Type parameter list cannot be empty".to_owned(), + }); + } let mut params: Vec = Vec::new(); let mut seen_default = false; loop { @@ -718,11 +783,16 @@ impl<'src> Parser<'src> { ) { j += 1; } - // A missing INDENT is still a match statement when - // `case` follows — parse_match then reports CPython's - // "expected an indented block after 'match' statement". + // A missing INDENT is still a match statement — + // parse_match then reports CPython's "expected an + // indented block after 'match' statement" + // (`invalid_match_stmt`). With an INDENT present, + // require `case` so `match[x]:`-style annotations + // aren't misread. if matches!(self.tokens.get(j).map(|t| &t.kind), Some(TokenKind::Indent)) { j += 1; + } else { + return true; } while matches!( self.tokens.get(j).map(|t| &t.kind), @@ -786,6 +856,27 @@ impl<'src> Parser<'src> { fn parse_simple_statement(&mut self) -> Result { let start_span = self.peek_token().span; + // pegen `yield_stmt`: a statement-level `yield`/`yield from` is + // an expression statement and cannot continue into assignment + // or tuple forms (`yield from (), 1` is invalid syntax). + if self.at_keyword(Keyword::Yield) { + let e = self.parse_yield()?; + // `yield = 1` — pegen `invalid_assignment`'s dedicated + // `yield_expr '='` alternative. + if self.check(&TokenKind::Equal) { + return Err(ParseError::Unexpected { + span: e.span, + message: "assignment to yield expression not possible".to_owned(), + }); + } + let end = self.prev_token_span(); + self.consume_stmt_end()?; + return Ok(Stmt { + kind: StmtKind::Expr(e), + span: start_span.merge(end), + }); + } + // A statement opening with `(` makes any annotation target // non-"simple" (CPython pegen: `('(' single_target ')' | ...) ':'` // sets `simple=0`), even when the inner expression is a bare Name. @@ -812,7 +903,7 @@ impl<'src> Parser<'src> { ), }); } - let value = self.parse_expression_list(true)?; + let value = self.parse_assign_rhs()?; let end = self.prev_token_span(); self.consume_stmt_end()?; return Ok(Stmt { @@ -854,7 +945,7 @@ impl<'src> Parser<'src> { self.bump(); let annotation = self.parse_expression(false)?; let value = if self.eat(&TokenKind::Equal) { - Some(self.parse_expression_list(true)?) + Some(self.parse_assign_rhs()?) } else { None }; @@ -884,7 +975,7 @@ impl<'src> Parser<'src> { } // Peek-parse the right-hand side as expression list; // re-classify if another `=` follows. - let next = self.parse_expression_list(true)?; + let next = self.parse_assign_rhs()?; if self.check(&TokenKind::Equal) { targets.push(next); } else { @@ -960,7 +1051,16 @@ impl<'src> Parser<'src> { // (see `desugar_pep695_def`) so annotations referencing them // resolve and `f.__type_params__` is populated. let type_params = self.collect_pep695_type_params()?; - self.expect(&TokenKind::LPar, "`(`")?; + // pegen `invalid_def_raw` uses a *forced* token here, so the + // missing-paren diagnostic fires no matter what follows + // (`def f:` and `def f -> int:` both say "expected '('"). + if !self.check(&TokenKind::LPar) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "expected '('".to_owned(), + }); + } + self.bump(); let args = self.parse_function_arguments()?; self.expect(&TokenKind::RPar, "`)`")?; let returns = if self.eat(&TokenKind::RArrow) { @@ -1094,6 +1194,21 @@ impl<'src> Parser<'src> { let type_params = self.collect_pep695_type_params()?; let (bases, keywords) = if self.eat(&TokenKind::LPar) { let (a, kw) = self.parse_call_args()?; + // `class C(x for x in L):` — pegen's class_def_raw only + // accepts `arguments`, not a bare genexp; plain "invalid + // syntax". A *parenthesized* genexp base is grammatically + // fine (it fails later at runtime instead). + if let [only] = a.as_slice() { + if matches!(only.kind, ExprKind::GeneratorExp { .. }) + && kw.is_empty() + && self.source.as_bytes().get(only.span.start.0 as usize) != Some(&b'(') + { + return Err(ParseError::Unexpected { + span: only.span, + message: "invalid syntax".to_owned(), + }); + } + } self.expect(&TokenKind::RPar, "`)`")?; (a, kw) } else { @@ -1140,16 +1255,17 @@ impl<'src> Parser<'src> { while self.at_keyword(Keyword::Except) { let exc_tok = self.bump(); // `except` let is_star = matches!(self.peek(), TokenKind::Star); + let mut kw_span = exc_tok.span; if is_star { - self.bump(); + kw_span = kw_span.merge(self.bump().span); saw_star = true; } else { saw_plain = true; } if saw_star && saw_plain { return Err(ParseError::Unexpected { - span: exc_tok.span, - message: "cannot have both 'except' and 'except*' on the same try".to_owned(), + span: kw_span, + message: "cannot have both 'except' and 'except*' on the same 'try'".to_owned(), }); } let (type_, name) = if self.check(&TokenKind::Colon) { @@ -1163,7 +1279,21 @@ impl<'src> Parser<'src> { } (None, None) } else { + // `except` ⏎ — pegen `invalid_except_stmt`. + if matches!(self.peek(), TokenKind::Newline) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "expected ':'".to_owned(), + }); + } let t = self.parse_expression(false)?; + // `except A, B:` — pegen `invalid_except_stmt`. + if self.check(&TokenKind::Comma) { + return Err(ParseError::Unexpected { + span: t.span, + message: "multiple exception types must be parenthesized".to_owned(), + }); + } let n = if self.at_keyword(Keyword::As) { self.bump(); let nt = self.expect(&TokenKind::Name, "name after `as`")?; @@ -1174,7 +1304,14 @@ impl<'src> Parser<'src> { (Some(t), n) }; self.expect(&TokenKind::Colon, "`:`")?; - let handler_body = self.parse_block("'except' statement", exc_tok.span)?; + let handler_body = self.parse_block( + if is_star { + "'except*' statement" + } else { + "'except' statement" + }, + exc_tok.span, + )?; let span_end = handler_body.last().map_or(exc_tok.span, |s| s.span); handlers.push(ExceptHandler { type_, @@ -1199,7 +1336,7 @@ impl<'src> Parser<'src> { if handlers.is_empty() && finalbody.is_empty() { return Err(ParseError::Unexpected { span: try_tok.span, - message: "expected `except` or `finally` after `try`".to_owned(), + message: "expected 'except' or 'finally' block".to_owned(), }); } let span_end = finalbody @@ -1349,13 +1486,31 @@ impl<'src> Parser<'src> { // keyword-only argument to follow it; CPython rejects `def f(p, *)` // and `def f(p, *, **kw)` with "named arguments must follow bare *". let mut bare_star_span: Option = None; + // pegen `invalid_parameters` bookkeeping. + let mut saw_slash = false; + let mut saw_star = false; + let mut saw_kwarg = false; loop { if self.check(&TokenKind::RPar) || self.check(&TokenKind::Colon) { break; } + // Nothing may follow `**kwargs` (pegen `invalid_parameters`). + if saw_kwarg { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "arguments cannot follow var-keyword argument".to_owned(), + }); + } // `*args` or bare `*` separator. if self.check(&TokenKind::Star) { let star_tok = self.bump(); + if saw_star { + return Err(ParseError::Unexpected { + span: star_tok.span, + message: "* argument may appear only once".to_owned(), + }); + } + saw_star = true; if matches!(self.peek(), TokenKind::Name) { let n = self.bump(); args.vararg = Some(Arg { @@ -1363,8 +1518,22 @@ impl<'src> Parser<'src> { annotation: self.try_arg_annotation(allow_annotation, true)?, span: n.span, }); - } else { + if self.check(&TokenKind::Equal) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "var-positional argument cannot have default value".to_owned(), + }); + } + } else if matches!( + self.peek(), + TokenKind::Comma | TokenKind::RPar | TokenKind::Colon + ) { bare_star_span = Some(star_tok.span); + } else { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "invalid syntax".to_owned(), + }); } phase = 2; if !self.eat(&TokenKind::Comma) { @@ -1380,13 +1549,48 @@ impl<'src> Parser<'src> { annotation: self.try_arg_annotation(allow_annotation, false)?, span: n.span, }); + if self.check(&TokenKind::Equal) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "var-keyword argument cannot have default value".to_owned(), + }); + } + saw_kwarg = true; if !self.eat(&TokenKind::Comma) { break; } continue; } // `/` separator: everything we've collected becomes posonly. - if self.eat(&TokenKind::Slash) { + if self.check(&TokenKind::Slash) { + let slash_tok = self.bump(); + // pegen `invalid_parameters` slash rules, in CPython's + // precedence order. + if saw_slash { + return Err(ParseError::Unexpected { + span: slash_tok.span, + message: "/ may appear only once".to_owned(), + }); + } + if saw_star { + return Err(ParseError::Unexpected { + span: slash_tok.span, + message: "/ must be ahead of *".to_owned(), + }); + } + if args.args.is_empty() { + return Err(ParseError::Unexpected { + span: slash_tok.span, + message: "at least one argument must precede /".to_owned(), + }); + } + saw_slash = true; + if self.check(&TokenKind::Star) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "expected comma between / and *".to_owned(), + }); + } args.posonlyargs = std::mem::take(&mut args.args); phase = 1; if !self.eat(&TokenKind::Comma) { @@ -1395,10 +1599,31 @@ impl<'src> Parser<'src> { continue; } + // `def f(x, (y, z)):` — pegen `invalid_parameters`. + if self.check(&TokenKind::LPar) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: if allow_annotation { + "Function parameters cannot be parenthesized".to_owned() + } else { + "Lambda expression parameters cannot be parenthesized".to_owned() + }, + }); + } let n = self.expect(&TokenKind::Name, "parameter name")?; let name = self.ident(n.span); let annotation = self.try_arg_annotation(allow_annotation, false)?; let default = if self.eat(&TokenKind::Equal) { + // `def f(a, d=, c):` — pegen `invalid_default`. + if matches!( + self.peek(), + TokenKind::Comma | TokenKind::RPar | TokenKind::Colon | TokenKind::Newline + ) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "expected default value expression".to_owned(), + }); + } Some(self.parse_expression(false)?) } else { None @@ -1419,7 +1644,8 @@ impl<'src> Parser<'src> { } else if had_default { return Err(ParseError::Unexpected { span: n.span, - message: "non-default argument follows default argument".to_owned(), + message: "parameter without a default follows parameter with a default" + .to_owned(), }); } } @@ -1497,6 +1723,10 @@ impl<'src> Parser<'src> { "'if' statement" }; let test = self.parse_expression(false)?; + // `if x = 3:` — pegen `invalid_expression`/`invalid_named_expression`. + if self.check(&TokenKind::Equal) { + return Err(Self::display_equal_error(&test)); + } self.expect(&TokenKind::Colon, "`:`")?; let body = self.parse_block(if_what, if_tok.span)?; let orelse = if self.at_keyword(Keyword::Elif) { @@ -1523,6 +1753,10 @@ impl<'src> Parser<'src> { fn parse_while(&mut self) -> Result { let kw = self.bump(); let test = self.parse_expression(false)?; + // `while x = 3:` — same rule as `if`. + if self.check(&TokenKind::Equal) { + return Err(Self::display_equal_error(&test)); + } self.expect(&TokenKind::Colon, "`:`")?; let body = self.parse_block("'while' statement", kw.span)?; let orelse = if self.at_keyword(Keyword::Else) { @@ -1555,9 +1789,11 @@ impl<'src> Parser<'src> { return Err(self.assign_target_error(offender, true)); } if !self.at_keyword(Keyword::In) { + // pegen has no dedicated diagnostic here: `for i < ():` and + // `for a, b` ⏎ are both the generic failure. return Err(ParseError::Unexpected { span: self.peek_token().span, - message: "expected `in` in for-loop".to_owned(), + message: "invalid syntax".to_owned(), }); } self.bump(); @@ -1606,6 +1842,13 @@ impl<'src> Parser<'src> { fn parse_import(&mut self) -> Result { let kw = self.bump(); + // pegen `invalid_import`. + if matches!(self.peek(), TokenKind::Newline | TokenKind::Endmarker) { + return Err(ParseError::Unexpected { + span: kw.span, + message: "Expected one or more names after 'import'".to_owned(), + }); + } let mut names = Vec::new(); loop { let dotted = self.parse_dotted_name()?; @@ -1624,6 +1867,13 @@ impl<'src> Parser<'src> { break; } } + // pegen `invalid_import`: `import a from b`. + if self.at_keyword(Keyword::From) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "Did you mean to use 'from ... import ...' instead?".to_owned(), + }); + } let end = self.prev_token_span(); self.consume_stmt_end()?; Ok(Stmt { @@ -1646,8 +1896,17 @@ impl<'src> Parser<'src> { fn parse_import_from(&mut self) -> Result { let kw = self.bump(); // `from` let mut level = 0u32; - while self.eat(&TokenKind::Dot) { - level += 1; + // `from ...pkg import x`: the lexer greedily tokenises `...` as + // a single Ellipsis token, so relative-import dots arrive as a + // mix of `.` and `...`. + loop { + if self.eat(&TokenKind::Dot) { + level += 1; + } else if self.eat(&TokenKind::Ellipsis) { + level += 3; + } else { + break; + } } let module = if matches!(self.peek(), TokenKind::Name) { Some(self.parse_dotted_name()?) @@ -1660,13 +1919,20 @@ impl<'src> Parser<'src> { message: "expected `import`".to_owned(), }); } - self.bump(); + let import_tok = self.bump(); let names = if self.eat(&TokenKind::Star) { vec![Alias { name: "*".to_owned(), asname: None, }] } else { + // pegen `invalid_import_from_targets`. + if matches!(self.peek(), TokenKind::Newline | TokenKind::Endmarker) { + return Err(ParseError::Unexpected { + span: import_tok.span, + message: "Expected one or more names after 'import'".to_owned(), + }); + } let paren = self.eat(&TokenKind::LPar); let mut names = Vec::new(); loop { @@ -1675,6 +1941,15 @@ impl<'src> Parser<'src> { if paren && matches!(self.peek(), TokenKind::RPar) { break; } + // pegen `invalid_import_from_targets`: an unparenthesised + // list may not end with a comma. + if !paren && matches!(self.peek(), TokenKind::Newline | TokenKind::Endmarker) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "trailing comma not allowed without surrounding parentheses" + .to_owned(), + }); + } let n = self.expect(&TokenKind::Name, "imported name")?; let name = self.ident(n.span); let asname = if self.at_keyword(Keyword::As) { @@ -1694,6 +1969,16 @@ impl<'src> Parser<'src> { } names }; + // PEP 401: the parser itself activates FLUFL mode when it sees + // the future import, so later tokens on the same parse are + // affected even without compile flags (CPython pegen does the + // same). + if level == 0 + && module.as_deref() == Some("__future__") + && names.iter().any(|a| a.name == "barry_as_FLUFL") + { + self.flufl = true; + } let end = self.prev_token_span(); self.consume_stmt_end()?; Ok(Stmt { @@ -1733,7 +2018,10 @@ impl<'src> Parser<'src> { /// supported with no extra plumbing. fn parse_del(&mut self) -> Result { let kw = self.bump(); - let mut targets = vec![self.parse_ternary()?]; + // Accept starred targets syntactically (`del *x`) so the + // dedicated "cannot delete starred" diagnostic below fires + // instead of a generic parse failure at the `*`. + let mut targets = vec![self.parse_ternary_or_starred()?]; while self.eat(&TokenKind::Comma) { if matches!( self.peek(), @@ -1741,7 +2029,7 @@ impl<'src> Parser<'src> { ) { break; } - targets.push(self.parse_ternary()?); + targets.push(self.parse_ternary_or_starred()?); } // CPython (`invalid_del_stmt`): del targets are validated at parse // time — "cannot delete X" for anything but a name / attribute / @@ -1854,6 +2142,29 @@ impl<'src> Parser<'src> { }) } + /// pegen `invalid_match_stmt`: after the ordinary statement grammar + /// rejected a line starting with the soft keyword `match`, retry it + /// as a match statement; if a subject expression parses cleanly and + /// a `:` doesn't follow, "expected ':'" wins over the expression + /// path's error. Any other shape keeps the original error. + fn match_stmt_fallback_error(&mut self, expr_err: ParseError) -> ParseError { + let saved = self.pos; + self.bump(); // `match` + let subject_ok = self.parse_match_subject().is_ok(); + // Only a line that simply stopped short gets the named-token + // hint (`match x` ⏎); `match x x:` stays "invalid syntax". + if subject_ok && matches!(self.peek(), TokenKind::Newline | TokenKind::Endmarker) { + let span = self.peek_token().span; + self.pos = saved; + return ParseError::Unexpected { + span, + message: "expected ':'".to_owned(), + }; + } + self.pos = saved; + expr_err + } + /// CPython allows `match a, b:` (subject is an implicit tuple). We /// follow. fn parse_match_subject(&mut self) -> Result { @@ -1932,12 +2243,19 @@ impl<'src> Parser<'src> { let pat = self.parse_or_pattern()?; if self.at_keyword(Keyword::As) { self.bump(); - let n = self.expect(&TokenKind::Name, "name after `as`")?; + if !self.check(&TokenKind::Name) { + // pegen `invalid_as_pattern`. + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "invalid pattern target".to_owned(), + }); + } + let n = self.bump(); let name = self.ident(n.span); if name == "_" { return Err(ParseError::Unexpected { span: n.span, - message: "cannot use `_` as a capture target".to_owned(), + message: "cannot use '_' as a target".to_owned(), }); } return Ok(Pattern::As { @@ -2182,7 +2500,7 @@ impl<'src> Parser<'src> { if saw_kw { return Err(ParseError::Unexpected { span: self.peek_token().span, - message: "positional pattern after keyword pattern".to_owned(), + message: "positional patterns follow keyword patterns".to_owned(), }); } positionals.push(self.parse_pattern()?); @@ -2249,7 +2567,15 @@ impl<'src> Parser<'src> { if self.eat(&TokenKind::DoubleStar) { let n = self.expect(&TokenKind::Name, "name after `**` in mapping pattern")?; let name = self.ident(n.span); - rest = Some(if name == "_" { None } else { Some(name) }); + if name == "_" { + // PEP 634 forbids `**_` (pegen `invalid_double_star_pattern` + // is a bare "invalid syntax" in 3.13). + return Err(ParseError::Unexpected { + span: n.span, + message: "invalid syntax".to_owned(), + }); + } + rest = Some(Some(name)); if !self.eat(&TokenKind::Comma) { break; } @@ -2492,12 +2818,12 @@ impl<'src> Parser<'src> { if self.at_keyword(Keyword::Lambda) { return self.parse_lambda(); } - // `yield` and `yield from` are expressions in CPython's grammar. - // They're only legal inside function bodies; the compiler — not - // the parser — enforces that. - if self.at_keyword(Keyword::Yield) { - return self.parse_yield(); - } + // `yield` is *not* a general expression in CPython's grammar: a + // `yield_expr` is only admitted as a whole statement, as the + // sole RHS of an (aug/ann) assignment, or parenthesized. Those + // call sites invoke `parse_yield` themselves; anywhere else — + // `f(yield 1)`, `1, yield`, `not yield` — is "invalid syntax" + // (the atom fallback below the keyword check reports it). // PEP 572 walrus `NAME := expr`. The named-expression form // must syntactically be exactly a name followed by `:=`; the // compiler enforces the rest of the PEP's restrictions @@ -2526,13 +2852,32 @@ impl<'src> Parser<'src> { } let start = self.peek_token().span; let body = self.parse_or()?; + // `(True := 1)` — pegen `invalid_named_expression`: only plain + // names may be walrus targets; constants get named. + if self.check(&TokenKind::ColonEqual) { + return Err(ParseError::Unexpected { + span: body.span, + message: format!( + "cannot use assignment expressions with {}", + crate::ast::expr_name(&body) + ), + }); + } if self.at_keyword(Keyword::If) { self.bump(); let test = self.parse_or()?; if !self.at_keyword(Keyword::Else) { + // pegen `invalid_expression` requires `!(':')` here: + // with a colon next the generic failure wins. + if self.check(&TokenKind::Colon) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "invalid syntax".to_owned(), + }); + } return Err(ParseError::Unexpected { span: self.peek_token().span, - message: "expected `else` in conditional expression".to_owned(), + message: "expected 'else' after 'if' expression".to_owned(), }); } self.bump(); @@ -2550,6 +2895,17 @@ impl<'src> Parser<'src> { Ok(body) } + /// pegen's `(yield_expr | star_expressions)` — the RHS of an + /// assignment / augmented assignment / annotated assignment, where + /// a whole `yield` expression is legal (but never as a tuple + /// element: `x = yield, 1` is invalid syntax). + fn parse_assign_rhs(&mut self) -> Result { + if self.at_keyword(Keyword::Yield) { + return self.parse_yield(); + } + self.parse_expression_list(true) + } + fn parse_yield(&mut self) -> Result { let kw = self.bump(); // `yield` if self.at_keyword(Keyword::From) { @@ -2671,7 +3027,7 @@ impl<'src> Parser<'src> { let left = self.parse_bit_or()?; let mut ops = Vec::new(); let mut comparators = Vec::new(); - while let Some(op) = self.try_cmp_op() { + while let Some(op) = self.try_cmp_op()? { ops.push(op); comparators.push(self.parse_bit_or()?); } @@ -2689,37 +3045,57 @@ impl<'src> Parser<'src> { }) } - fn try_cmp_op(&mut self) -> Option { + fn try_cmp_op(&mut self) -> Result, ParseError> { let op = match self.peek() { TokenKind::Less => CmpOp::Lt, TokenKind::Greater => CmpOp::Gt, TokenKind::LessEqual => CmpOp::LtE, TokenKind::GreaterEqual => CmpOp::GtE, TokenKind::EqEqual => CmpOp::Eq, - TokenKind::NotEqual => CmpOp::NotEq, + TokenKind::NotEqual => { + // PEP 401 (`_PyPegen_check_barry_as_flufl`): the lexer + // produces NOTEQUAL for both spellings; exactly one is + // legal depending on whether `barry_as_FLUFL` is + // active. + let span = self.peek_token().span; + let text = self.lexeme(span); + if self.flufl && text != "<>" { + return Err(ParseError::Unexpected { + span, + message: "with Barry as BDFL, use '<>' instead of '!='".to_owned(), + }); + } + if !self.flufl && text == "<>" { + return Err(ParseError::Unexpected { + span, + message: "invalid syntax".to_owned(), + }); + } + CmpOp::NotEq + } TokenKind::Keyword(Keyword::In) => CmpOp::In, TokenKind::Keyword(Keyword::Is) => { // Two-token `is not` handled below. self.bump(); if self.at_keyword(Keyword::Not) { self.bump(); - return Some(CmpOp::IsNot); + return Ok(Some(CmpOp::IsNot)); } - return Some(CmpOp::Is); + return Ok(Some(CmpOp::Is)); } TokenKind::Keyword(Keyword::Not) => { // `not in` if matches!(self.peek_at(1), Some(TokenKind::Keyword(Keyword::In))) { self.bump(); self.bump(); - return Some(CmpOp::NotIn); + return Ok(Some(CmpOp::NotIn)); } - return None; + return Ok(None); } - _ => return None, + _ => return Ok(None), }; self.bump(); - Some(op) + Ok(Some(op)) } fn parse_bit_or(&mut self) -> Result { @@ -2818,6 +3194,7 @@ impl<'src> Parser<'src> { _ => break, }; self.bump(); + self.reject_not_operand()?; let right = self.parse_muldiv()?; let span = start.merge(self.prev_token_span()); left = Expr { @@ -2845,6 +3222,7 @@ impl<'src> Parser<'src> { _ => break, }; self.bump(); + self.reject_not_operand()?; let right = self.parse_unary()?; let span = start.merge(self.prev_token_span()); left = Expr { @@ -2888,6 +3266,7 @@ impl<'src> Parser<'src> { _ => return self.parse_power(), }; let kw = self.bump(); + self.reject_not_operand()?; let operand = self.parse_unary()?; let span = kw.span.merge(self.prev_token_span()); Ok(Expr { @@ -2899,6 +3278,19 @@ impl<'src> Parser<'src> { }) } + /// pegen `invalid_factor` / `invalid_arithmetic`: `not` binds looser + /// than arithmetic, so it can't appear as an operand (`3 + not 3`, + /// `- not 3`). + fn reject_not_operand(&self) -> Result<(), ParseError> { + if self.at_keyword(Keyword::Not) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "'not' after an operator must be parenthesized".to_owned(), + }); + } + Ok(()) + } + fn parse_power(&mut self) -> Result { let start = self.peek_token().span; let base = self.parse_trailer_chain()?; @@ -2985,19 +3377,51 @@ impl<'src> Parser<'src> { // keyword argument") and a repeated keyword name (CPython: // "keyword argument repeated: "). let mut seen_keyword = false; + let mut seen_kw_unpack = false; let mut kw_names: Vec = Vec::new(); loop { let arg_start = self.peek_token().span; if self.eat(&TokenKind::DoubleStar) { let val = self.parse_ternary()?; + // `f(**kwargs={...})` — pegen `invalid_kwarg`. + if self.check(&TokenKind::Equal) { + return Err(ParseError::Unexpected { + span: arg_start.merge(val.span), + message: "cannot assign to keyword argument unpacking".to_owned(), + }); + } seen_keyword = true; + seen_kw_unpack = true; keywords.push(KwArg { arg: None, value: val, }); - } else if self.eat(&TokenKind::Star) { - let val = self.parse_ternary()?; - let span = val.span; + } else if self.check(&TokenKind::Star) { + let star_tok = self.bump(); + // A `*` whose operand doesn't parse is pegen's + // `starred_expression` fallback: "Invalid star expression". + // This outranks the "follows keyword unpacking" diagnostic: + // `f(**x, *)` reports the bad star, not the ordering. + let val = self.parse_ternary().map_err(|_| ParseError::Unexpected { + span: star_tok.span, + message: "Invalid star expression".to_owned(), + })?; + // `f(**x, *y)` — pegen `invalid_arguments`. + if seen_kw_unpack { + return Err(ParseError::Unexpected { + span: star_tok.span, + message: "iterable argument unpacking follows keyword argument unpacking" + .to_owned(), + }); + } + // `f(*args=[0])` — pegen `invalid_starred_expression_unpacking`. + if self.check(&TokenKind::Equal) { + return Err(ParseError::Unexpected { + span: star_tok.span.merge(val.span), + message: "cannot assign to iterable argument unpacking".to_owned(), + }); + } + let span = star_tok.span.merge(val.span); args.push(Expr { kind: ExprKind::Starred(Box::new(val)), span, @@ -3009,6 +3433,13 @@ impl<'src> Parser<'src> { { let nt = self.bump(); let name = self.ident(nt.span); + // `f(__debug__=1)` — CPython `forbidden_name`. + if name == "__debug__" { + return Err(ParseError::Unexpected { + span: nt.span, + message: "cannot assign to __debug__".to_owned(), + }); + } if kw_names.contains(&name) { return Err(ParseError::Unexpected { span: nt.span, @@ -3016,6 +3447,16 @@ impl<'src> Parser<'src> { }); } self.bump(); // `=` + // `f(a=)` — pegen `invalid_kwarg`. + if matches!( + self.peek(), + TokenKind::Comma | TokenKind::RPar | TokenKind::Newline + ) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "expected argument value expression".to_owned(), + }); + } let val = self.parse_ternary()?; seen_keyword = true; kw_names.push(name.clone()); @@ -3023,11 +3464,27 @@ impl<'src> Parser<'src> { arg: Some(name), value: val, }); + } else if matches!( + self.peek(), + TokenKind::Keyword(Keyword::True | Keyword::False | Keyword::None) + ) && matches!(self.peek_at(1), Some(TokenKind::Equal)) + { + // `f(True=1)` — pegen `invalid_kwarg` names the + // constant. + let nt = self.peek_token().clone(); + return Err(ParseError::Unexpected { + span: nt.span, + message: format!("cannot assign to {}", self.lexeme(nt.span)), + }); } else { if seen_keyword { return Err(ParseError::Unexpected { span: self.peek_token().span, - message: "positional argument follows keyword argument".to_owned(), + message: if seen_kw_unpack { + "positional argument follows keyword argument unpacking".to_owned() + } else { + "positional argument follows keyword argument".to_owned() + }, }); } let e = self.parse_ternary()?; @@ -3172,7 +3629,12 @@ impl<'src> Parser<'src> { if self.check(&TokenKind::Star) { let span = self.peek_token().span; self.bump(); - let value = self.parse_ternary()?; + // `A[*]` / `A[*:]` / `A[*(1:2)]` — pegen's `starred_expression` + // fallback: "Invalid star expression". + let value = self.parse_ternary().map_err(|_| ParseError::Unexpected { + span, + message: "Invalid star expression".to_owned(), + })?; return Ok(Expr { kind: ExprKind::Starred(Box::new(value)), span, @@ -3295,9 +3757,11 @@ impl<'src> Parser<'src> { TokenKind::LPar => self.parse_paren_or_tuple(), TokenKind::LSqb => self.parse_list_or_listcomp(), TokenKind::LBrace => self.parse_dict_or_set(), - other => Err(ParseError::Unexpected { + // pegen's generic parse failure: a bare "invalid syntax" + // pointing at the token the expression grammar rejected. + _ => Err(ParseError::Unexpected { span: tok.span, - message: format!("unexpected token in expression: {other:?}"), + message: "invalid syntax".to_owned(), }), } } @@ -3312,6 +3776,14 @@ impl<'src> Parser<'src> { span: lp.span.merge(rp.span), }); } + // pegen `group`: `(yield ...)` admits a yield expression as the + // *sole* parenthesized content (`(yield, 1)` stays invalid — the + // `expect` below reports the comma). + if self.at_keyword(Keyword::Yield) { + let inner = self.parse_yield()?; + self.expect(&TokenKind::RPar, "`)`")?; + return Ok(inner); + } let first = self.parse_ternary_or_starred()?; let first_starred = matches!(first.kind, ExprKind::Starred(_)); // Generator expression? @@ -3334,6 +3806,12 @@ impl<'src> Parser<'src> { break; } items.push(self.parse_ternary_or_starred()?); + // `(x, y, z=3)` — pegen `invalid_named_expression`. + if self.check(&TokenKind::Equal) { + return Err(Self::display_equal_error( + items.last().expect("just pushed"), + )); + } } let rp = self.expect_or_forgot_comma(&TokenKind::RPar, "`)`", &items)?; return Ok(Expr { @@ -3344,10 +3822,12 @@ impl<'src> Parser<'src> { // A bare `(*a)` with no trailing comma is a syntax error in // CPython — starred expressions are only legal inside a tuple/ // call/assignment context, never as a lone parenthesized value. - if first_starred { + // Only the well-formed `(*a)` gets the dedicated message; + // `(*a:...)` falls through to the generic failure at the `:`. + if first_starred && self.check(&TokenKind::RPar) { return Err(ParseError::Unexpected { span: first.span, - message: "can't use starred expression here".to_owned(), + message: "cannot use starred expression here".to_owned(), }); } // Plain parenthesized expression: no wrapper node, and — exactly @@ -3387,11 +3867,27 @@ impl<'src> Parser<'src> { }); } let mut items = vec![first]; + if self.check(&TokenKind::Equal) { + return Err(Self::display_equal_error(&items[0])); + } while self.eat(&TokenKind::Comma) { if self.check(&TokenKind::RSqb) { break; } items.push(self.parse_ternary_or_starred()?); + if self.check(&TokenKind::Equal) { + return Err(Self::display_equal_error( + items.last().expect("just pushed"), + )); + } + } + // `[x,y for x,y in ...]` — pegen `invalid_comprehension`. + if self.at_keyword(Keyword::For) { + let span = items[0].span.merge(items.last().expect("nonempty").span); + return Err(ParseError::Unexpected { + span, + message: "did you forget parentheses around the comprehension target?".to_owned(), + }); } let rb = self.expect_or_forgot_comma(&TokenKind::RSqb, "`]`", &items)?; Ok(Expr { @@ -3428,7 +3924,7 @@ impl<'src> Parser<'src> { } else { let k = self.parse_ternary()?; self.expect_dict_colon(&k)?; - let v = self.parse_ternary()?; + let v = self.parse_dict_value()?; keys.push(Some(k)); values.push(v); } @@ -3443,7 +3939,7 @@ impl<'src> Parser<'src> { let first_starred = matches!(first.kind, ExprKind::Starred(_)); if !first_starred && self.eat(&TokenKind::Colon) { // Dict literal (or dict comprehension). - let v = self.parse_ternary()?; + let v = self.parse_dict_value()?; if self.at_keyword(Keyword::For) || self.at_keyword(Keyword::Async) { let generators = self.parse_comp_for()?; let rb = self.expect(&TokenKind::RBrace, "`}`")?; @@ -3468,7 +3964,7 @@ impl<'src> Parser<'src> { } else { let k = self.parse_ternary()?; self.expect_dict_colon(&k)?; - let vv = self.parse_ternary()?; + let vv = self.parse_dict_value()?; keys.push(Some(k)); values.push(vv); } @@ -3501,13 +3997,7 @@ impl<'src> Parser<'src> { // "cannot assign to here. Maybe you meant '==' instead // of '='?", anchored at the key. if self.check(&TokenKind::Equal) { - return Err(ParseError::Unexpected { - span: first.span, - message: format!( - "cannot assign to {} here. Maybe you meant '==' instead of '='?", - crate::ast::expr_name(&first) - ), - }); + return Err(Self::display_equal_error(&first)); } let mut items = vec![first]; while self.eat(&TokenKind::Comma) { @@ -3516,16 +4006,19 @@ impl<'src> Parser<'src> { } items.push(self.parse_ternary_or_starred()?); if self.check(&TokenKind::Equal) { - let last = items.last().expect("just pushed"); - return Err(ParseError::Unexpected { - span: last.span, - message: format!( - "cannot assign to {} here. Maybe you meant '==' instead of '='?", - crate::ast::expr_name(last) - ), - }); + return Err(Self::display_equal_error( + items.last().expect("just pushed"), + )); } } + // `{x,y for x,y in ...}` — pegen `invalid_comprehension`. + if self.at_keyword(Keyword::For) { + let span = items[0].span.merge(items.last().expect("nonempty").span); + return Err(ParseError::Unexpected { + span, + message: "did you forget parentheses around the comprehension target?".to_owned(), + }); + } let rb = self.expect_or_forgot_comma(&TokenKind::RBrace, "`}`", &items)?; Ok(Expr { kind: ExprKind::Set(items), @@ -3533,6 +4026,25 @@ impl<'src> Parser<'src> { }) } + /// pegen `invalid_named_expression`: a stray `=` after an element of + /// a display/condition where assignment is impossible. Plain names + /// get the "==' or ':='" hint, anything else the "cannot assign" + /// wording. + fn display_equal_error(expr: &Expr) -> ParseError { + let message = if matches!(expr.kind, ExprKind::Name(_)) { + "invalid syntax. Maybe you meant '==' or ':=' instead of '='?".to_owned() + } else { + format!( + "cannot assign to {} here. Maybe you meant '==' instead of '='?", + crate::ast::expr_name(expr) + ) + }; + ParseError::Unexpected { + span: expr.span, + message, + } + } + fn parse_comp_for(&mut self) -> Result, ParseError> { let mut generators = Vec::new(); loop { @@ -3556,18 +4068,21 @@ impl<'src> Parser<'src> { }; self.bump(); let target = self.parse_target_list_no_tuple()?; + // The missing-`in` diagnostic outranks target validation: + // `[x for a, b, (c+1) if y]` complains about `in`, not the + // target (pegen `invalid_comprehension` ordering). + if !self.at_keyword(Keyword::In) { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "'in' expected after for-loop variables".to_owned(), + }); + } // CPython (`invalid_comprehension` / `invalid_for_target`): // comprehension targets get the same parse-time validation as // statement `for` targets (bare message). if let Some((offender, _)) = Self::find_invalid_target(&target, false) { return Err(self.assign_target_error(offender, true)); } - if !self.at_keyword(Keyword::In) { - return Err(ParseError::Unexpected { - span: self.peek_token().span, - message: "expected `in` in comprehension".to_owned(), - }); - } self.bump(); let iter = self.parse_or()?; let mut ifs = Vec::new(); @@ -3618,14 +4133,19 @@ impl<'src> Parser<'src> { if let TokenKind::Star = self.peek() { let star_tok = self.peek_token().clone(); self.bump(); - let inner = self.parse_unary()?; + let inner = self.parse_bit_or()?; let span = star_tok.span.merge(inner.span); return Ok(Expr { kind: ExprKind::Starred(Box::new(inner)), span, }); } - self.parse_unary() + // Parse at the `bitwise_or` level, exactly like pegen's + // `invalid_for_target` recovery: `for x+1 in y` must consume the + // whole `x+1` so the "cannot assign to expression" diagnostic + // fires instead of a bogus missing-`in` complaint. (Comparisons + // stay excluded so `for i in xs` isn't mis-read as `i in xs`.) + self.parse_bit_or() } /// Handle adjacent-string concatenation, mixing plain strings, @@ -4337,6 +4857,8 @@ impl<'src> Parser<'src> { // tuple form `f'{*a,}'` is fine and parses as a tuple instead. if !partial { if let ExprKind::Starred(_) = value.kind { + // pegen's *unparenthesized* wording ("can't", not + // "cannot") — the same message a bare `x = *a` gets. return Err(ParseError::Unexpected { span: Span::new(expr_abs, expr_abs + expr_text.len() as u32), message: "can't use starred expression here".to_owned(), @@ -4657,9 +5179,23 @@ fn map_fstring_subparse_error( } } ParseError::Unexpected { span, message } | ParseError::Indentation { span, message } => { + // The synthetic `(...)` wrapper makes a bare `f'{*x}'` look + // parenthesized, picking up pegen's "cannot" wording; the + // unparenthesized original gets "can't". + if message == "cannot use starred expression here" { + return ParseError::Unexpected { + span: Span::new(map_back(span.start.0), map_back(span.end.0)), + message: "can't use starred expression here".to_owned(), + }; + } let internal = message.starts_with("expected ") || message.starts_with("unexpected token") - || message == "trailing"; + || message == "trailing" + // pegen's catch-all: inside a replacement field it + // becomes the field-shaped diagnostic ("expecting a + // valid expression after '{'" / the delimiters list), + // never the bare form (`f'{.}'`, `f'{lambda x:x}'`). + || message == "invalid syntax"; if !internal { return ParseError::Unexpected { span: Span::new(map_back(span.start.0), map_back(span.end.0)), @@ -5443,7 +5979,41 @@ fn big_to_i64(b: &num_bigint::BigInt) -> Option { /// )() /// )() /// ``` -fn build_lazy_type_alias(name: &str, body: Expr, params: &[TypeParam], span: Span) -> Expr { +/// Lower a first-class [`StmtKind::TypeAlias`] statement to the +/// runtime assignment form the compiler executes: +/// `Name = __weavepy_type_alias__('Name', (…), )` (see +/// [`build_lazy_type_alias`]). Called by the compiler front-end so +/// every later pass (mangling, scope analysis, codegen) sees the +/// same shape the parser used to emit directly. +/// +/// # Panics +/// +/// Panics if `stmt` is not a [`StmtKind::TypeAlias`]. +pub fn lower_type_alias_stmt(stmt: &Stmt) -> Stmt { + let StmtKind::TypeAlias { + name, + name_span, + type_params, + value, + } = &stmt.kind + else { + panic!("lower_type_alias_stmt on non-TypeAlias statement"); + }; + let target = Expr { + kind: ExprKind::Name(name.clone()), + span: *name_span, + }; + let rhs = build_lazy_type_alias(name, (**value).clone(), type_params, *name_span); + Stmt { + kind: StmtKind::Assign { + targets: vec![target], + value: rhs, + }, + span: stmt.span, + } +} + +pub fn build_lazy_type_alias(name: &str, body: Expr, params: &[TypeParam], span: Span) -> Expr { let thunk = Expr { kind: ExprKind::TypeParamFn { args: Arguments::default(), diff --git a/crates/weavepy-parser/src/unparse.rs b/crates/weavepy-parser/src/unparse.rs index a1ec0ec..657bda6 100644 --- a/crates/weavepy-parser/src/unparse.rs +++ b/crates/weavepy-parser/src/unparse.rs @@ -189,6 +189,16 @@ fn write_expr(out: &mut String, e: &Expr, level: Level) -> Option<()> { } => { write_expr(out, func, Level::Atom)?; out.push('('); + // A sole generator-expression argument shares the call's + // parens: `f(x for x in a)` (CPython `append_ast_call`). + if keywords.is_empty() && args.len() == 1 { + if let ExprKind::GeneratorExp { elt, generators } = &args[0].kind { + write_expr(out, elt, Level::Test)?; + write_comprehensions(out, generators)?; + out.push(')'); + return Some(()); + } + } let mut first = true; for a in args { if !first { @@ -321,11 +331,71 @@ fn write_expr(out: &mut String, e: &Expr, level: Level) -> Option<()> { out.push_str("yield from "); write_expr(out, inner, Level::Test) }), - // f-strings with interpolations and compiler-internal nodes: - // no faithful unparse — the caller falls back to raw source. - ExprKind::JoinedStr(_) | ExprKind::FormattedValue { .. } | ExprKind::TypeParamFn { .. } => { - None + // CPython `append_fstring`: build the body text, then emit + // `f` + the body's repr. (Lossy for literal `{{`/`}}` braces, + // exactly as CPython's unparser is.) + ExprKind::JoinedStr(_) | ExprKind::FormattedValue { .. } => { + let mut body = String::new(); + write_fstring_body(&mut body, e, false)?; + out.push('f'); + write_str_repr(out, &body); + Some(()) + } + // Compiler-internal node: no faithful unparse — the caller + // falls back to raw source. + ExprKind::TypeParamFn { .. } => None, + } +} + +/// CPython `build_fstring_body`: the text between the f-string quotes. +/// Inside a format spec (`is_format_spec`), nested constants append raw. +fn write_fstring_body(out: &mut String, e: &Expr, is_format_spec: bool) -> Option<()> { + match &e.kind { + ExprKind::JoinedStr(parts) => { + for p in parts { + write_fstring_body(out, p, is_format_spec)?; + } + Some(()) + } + ExprKind::FormattedValue { + value, + conversion, + format_spec, + } => { + out.push('{'); + let mut inner = String::new(); + // `PR_TEST + 1` in CPython's `append_formattedvalue`: + // lambdas / conditionals / walruses get wrapped in parens. + write_expr(&mut inner, value, Level::Or)?; + // `{{` would read as an escaped literal brace: CPython + // inserts a space (`{ {…}`). + if inner.starts_with('{') { + out.push(' '); + } + out.push_str(&inner); + if *conversion >= 0 { + out.push('!'); + out.push(char::from_u32(*conversion as u32)?); + } + if let Some(spec) = format_spec { + out.push(':'); + write_fstring_body(out, spec, true)?; + } + out.push('}'); + Some(()) } + ExprKind::Constant(Constant::Str(s)) => { + // Literal braces re-escape as `{{` / `}}` (CPython + // `append_fstring_unicode`). + for c in s.chars() { + out.push(c); + if c == '{' || c == '}' { + out.push(c); + } + } + Some(()) + } + _ => None, } } @@ -485,16 +555,18 @@ fn write_constant(out: &mut String, c: &Constant) -> Option<()> { } } Constant::Complex(real, imag) => { - if *real == 0.0 { - out.push_str(&format!("{imag}j")); + let mut repr = if *real == 0.0 && real.is_sign_positive() { + format!("{imag}j") } else { - out.push('('); - out.push_str(&format!("{real}")); - if imag.is_sign_positive() { - out.push('+'); - } - out.push_str(&format!("{imag}j)")); + let sign = if imag.is_sign_positive() { "+" } else { "" }; + format!("({real}{sign}{imag}j)") + }; + // CPython `append_repr` swaps `inf` for an eval-able + // overflow literal in complex reprs. + if repr.contains("inf") { + repr = repr.replace("inf", "1e309"); } + out.push_str(&repr); } Constant::Str(s) => write_str_repr(out, s), Constant::WStr(_) => return None, diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 5cff490..98ea5ed 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -3663,7 +3663,10 @@ pub(crate) fn code_flags(c: &weavepy_compiler::CodeObject) -> u32 { if c.freevars.is_empty() && c.cellvars.is_empty() { f |= CO_NOFREE; } - f + // `CO_FUTURE_*` bits recorded at compile time (RFC 0052) — what + // lets `compile(..., dont_inherit=False)` inherit the caller's + // future statements, like CPython. + f | c.future_flags } fn attr_set(obj: &Object, name: &str, value: Object) -> Result<(), RuntimeError> { @@ -7805,6 +7808,7 @@ fn b_mark_iterable_coroutine(args: &[Object]) -> Result { name: f.name.clone(), code: RefCell::new(Rc::new(code)), globals: f.globals.clone(), + builtins: f.builtins.clone(), defaults: f.defaults.clone(), kw_defaults: f.kw_defaults.clone(), closure: f.closure.clone(), diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 63785e0..d50fa40 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -97,6 +97,14 @@ struct Frame { stack: Vec, /// Globals shared across frames within the same module. globals: Rc>, + /// The builtins mapping this frame resolves names against — + /// CPython's `f_builtins`, resolved once at frame creation from + /// `globals['__builtins__']` (a dict, or a module whose dict is + /// used) with the interpreter-wide builtins as the fallback. This + /// is what makes `exec(code, {'__builtins__': {…}})` sandboxing + /// effective on both the `LOAD_GLOBAL` fast path (the inline-cache + /// guard fingerprints this dict) and the slow path. + builtins: Rc>, /// For class-body frames, names are stored here instead of globals. /// `None` for ordinary function and module frames. class_namespace: Option>>, @@ -473,6 +481,11 @@ pub struct Interpreter { stdout: Stdout, builtins: Rc>, cache: ModuleCache, + /// The interpreter-wide optimization level (`-O`/`-OO`, mirrored on + /// `sys.flags.optimize`). `compile(..., optimize=-1)` and every + /// internal compile (imports, `exec`/`eval` of source) resolve to + /// this level (RFC 0052). + pub(crate) optimize_level: u8, /// Live call stack of Python-visible frame snapshots, in /// outer-to-inner order. The topmost entry corresponds to the /// currently-executing `Frame`. RFC 0018: used by @@ -538,9 +551,42 @@ impl Default for Interpreter { call_kw: None, })), ); + // Module-identity keys: `sys.modules['builtins'].__dict__` *is* + // this dict (one namespace, RFC 0052 WS5), so it carries the + // module's own metadata exactly like CPython's builtins dict. + builtins_dict.insert( + DictKey(Object::from_static("__name__")), + Object::from_static("builtins"), + ); + builtins_dict.insert( + DictKey(Object::from_static("__doc__")), + Object::from_static( + "Built-in functions, types, exceptions, and other objects.\n\n\ + This module provides direct access to all 'built-in'\n\ + identifiers of Python; for example, builtins.len is\n\ + the full name for the built-in function len().", + ), + ); + builtins_dict.insert( + DictKey(Object::from_static("__package__")), + Object::from_static(""), + ); let builtins = Rc::new(RefCell::new(builtins_dict)); let cache = ModuleCache::default(); stdlib::register_all(&cache); + // RFC 0052 WS5 — patchable builtins: the `builtins` module and + // the interpreter's ambient lookup namespace are *one dict*. + // Any write — `builtins.open = …`, `builtins.__dict__['open'] + // = …`, `mock.patch('builtins.open')` — is immediately visible + // to every frame's name resolution, and vice versa. + cache.insert( + "builtins", + Object::Module(Rc::new(crate::object::PyModule { + name: "builtins".to_owned(), + filename: None, + dict: builtins.clone(), + })), + ); // RFC 0024: teach the cycle GC to walk suspended generator // frames (their `Frame` type is private to this module). static GEN_TRAVERSE: std::sync::Once = std::sync::Once::new(); @@ -596,6 +642,7 @@ impl Default for Interpreter { stdout, builtins, cache, + optimize_level: 0, frame_stack, exc_info_stack, excepthook, @@ -665,6 +712,7 @@ impl Interpreter { stdout: self.stdout.clone(), builtins: self.builtins.clone(), cache: self.cache.clone(), + optimize_level: self.optimize_level, frame_stack: Rc::new(RefCell::new(Vec::new())), exc_info_stack: Rc::new(RefCell::new(Vec::new())), excepthook: self.excepthook.clone(), @@ -679,6 +727,16 @@ impl Interpreter { &self.cache } + /// The [`weavepy_compiler::CompileOptions`] every internal compile + /// (imports, `exec`/`eval` of source) uses: no extra flags, the + /// interpreter-wide optimization level (RFC 0052). + fn default_compile_options(&self) -> weavepy_compiler::CompileOptions { + weavepy_compiler::CompileOptions { + flags: 0, + optimize: self.optimize_level, + } + } + /// Replace `sys.argv` with the given values. The first entry is /// the script name; subsequent entries are passed-through args. /// @@ -728,6 +786,10 @@ impl Interpreter { /// user code runs. pub fn apply_run_options(&mut self, opts: &InterpreterFlags) { let flags = &opts; + // `-O`/`-OO`: the interpreter-wide optimization level every + // internal compile (imports, exec/eval, `compile(..., + // optimize=-1)`) resolves against (RFC 0052). + self.optimize_level = flags.optimize; // `-X no_debug_ranges`: drop PEP 657 column info, like CPython // building code objects without end-position tables. crate::vm_singletons::set_debug_ranges( @@ -1388,7 +1450,7 @@ impl Interpreter { crate::vm_singletons::publish_interpreter_ptr(std::ptr::from_mut::(self)); let _handles = self.activate_thread_handles(); let code_rc = Rc::new(code.clone()); - let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals, true); + let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals, None); self.run_frame(&mut frame) } @@ -1443,7 +1505,7 @@ impl Interpreter { }); self.cache.insert(name, Object::Module(module)); let code_rc = Rc::new(code.clone()); - let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals, true); + let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals, None); let result = self.run_frame(&mut frame); // After the module finishes, run any deferred `__del__` // finalizers queued by the cycle GC. Errors propagate via @@ -1503,7 +1565,7 @@ impl Interpreter { // which in `test_concurrent_futures.test_init` pinned the log // `Queue` and its SemLocks until the next cyclic collection // (RFC 0040: resource-tracker exited nonzero on leaked semaphores). - if let Object::BoundMethod(bm) = &dropped { + if matches!(&dropped, Object::BoundMethod(_)) { // A bound method that escaped into an instance attribute // (`self.cb = self.handler`) was GC-tracked at store time (see // `generic_setattr_instance`); merely dropping our clone would @@ -1516,15 +1578,41 @@ impl Interpreter { // freshly-unpickled object on it until the next cyclic // collection (test_concurrent_futures // `test_ressources_gced_in_workers`). - if gc_trace::is_tracked(crate::weakref_registry::id_of(&dropped)) { - self.reap_dead_subgraph(dropped); - return; - } - let recv = bm.receiver.clone(); - drop(dropped); - if Self::local_needs_prompt_reap(&recv) && Self::looks_reapable_temporary(&recv) { - self.prompt_reap_dropped(recv); + // + // Receivers can themselves be bound methods, arbitrarily deep + // (`f = f.__call__` wrapper towers — test_descr + // test_wrapper_segfault builds a million of them), so the walk + // is a loop, not recursion, and stops as soon as a link is + // still referenced elsewhere (its receiver isn't dead). + let mut cur = dropped; + loop { + let (tracked, alive, recv) = { + let Object::BoundMethod(bm) = &cur else { break }; + let id = crate::weakref_registry::id_of(&cur); + // `cur` is our only handle when nothing else holds the + // Rc (modulo registry strong clones); a method still + // aliased — e.g. as an outer wrapper's `__self__` — + // keeps its receiver alive, so the cascade must not + // descend into it. + let alive = + Rc::strong_count(bm) > 1 + crate::weakref_registry::strong_clone_count(id); + (gc_trace::is_tracked(id), alive, bm.receiver.clone()) + }; + if tracked { + self.reap_dead_subgraph(cur); + return; + } + if alive { + return; + } + drop(cur); + if !(Self::local_needs_prompt_reap(&recv) && Self::looks_reapable_temporary(&recv)) + { + return; + } + cur = recv; } + self.prompt_reap_dropped(cur); return; } let id0 = crate::weakref_registry::id_of(&dropped); @@ -3286,13 +3374,32 @@ impl Interpreter { globals } + /// CPython `_PyEval_BuiltinsFromGlobals`: the builtins mapping a + /// frame executing under `globals` resolves names against — + /// `globals['__builtins__']` when present (a dict, or a module + /// whose dict is used), else the interpreter-wide builtins dict. + fn builtins_for_globals(&self, globals: &Rc>) -> Rc> { + // Borrowed-key lookup: this runs on every frame creation, so it + // must not allocate. The common case (module globals seeded by + // `module_globals`) finds the interpreter dict here anyway. + match globals.borrow().get(&crate::object::StrKey("__builtins__")) { + Some(Object::Dict(d)) => d.clone(), + Some(Object::Module(m)) => m.dict.clone(), + _ => self.builtins.clone(), + } + } + + /// Build an execution frame. `builtins` is the pre-resolved + /// builtins mapping when the caller already has one — a Python + /// function's cached `func_builtins` — and `None` for module / + /// exec / class-body frames, which resolve from `globals` here. fn make_frame( &self, code: Rc, positional: Vec, closure: Vec, globals: Rc>, - _is_module: bool, + builtins: Option>>, ) -> Frame { let mut locals = vec![Object::Unbound; code.varnames.len()]; for (i, v) in positional.into_iter().enumerate() { @@ -3332,12 +3439,14 @@ impl Interpreter { other => cells.push(Rc::new(RefCell::new(other))), } } + let builtins = builtins.unwrap_or_else(|| self.builtins_for_globals(&globals)); Frame { code, locals: Rc::new(RefCell::new(locals)), cells, stack: Vec::with_capacity(16), globals, + builtins, class_namespace: None, class_namespace_obj: None, exc_handlers: Vec::new(), @@ -4112,7 +4221,7 @@ impl Interpreter { Rc::new(PyFrame { code: frame.code.clone(), globals, - builtins: self.builtins.clone(), + builtins: frame.builtins.clone(), lasti: Cell::new(frame.pc), back: RefCell::new(back), locals_cache: RefCell::new(None), @@ -4893,9 +5002,12 @@ impl Interpreter { name = m; } } - let from_ns = match &frame.class_namespace_obj { + let from_ns = match frame.class_namespace_obj.clone() { // PEP 3115 custom namespace: read it before globals. - Some(ns_obj) => self.class_ns_load(ns_obj, &name), + Some(ns_obj) => { + let g = frame.globals.clone(); + self.class_ns_load(&ns_obj, &name, &g)? + } None => frame .class_namespace .as_ref() @@ -4903,7 +5015,7 @@ impl Interpreter { }; let v = match from_ns { Some(v) => v, - None => self.lookup_global_or_builtin(&frame.globals, &name)?, + None => self.lookup_global_or_builtin(frame, &name)?, }; frame.push(v); } @@ -6451,6 +6563,11 @@ impl Interpreter { name, code: RefCell::new(code), globals: frame.globals.clone(), + // The defining frame's builtins, not a fresh + // resolution: same mapping CPython's + // `PyFunction_New` snapshots (framestate's + // f_builtins came from the same globals). + builtins: frame.builtins.clone(), defaults, kw_defaults, closure, @@ -6491,9 +6608,12 @@ impl Interpreter { .get(free_index) .cloned() .unwrap_or_default(); - let from_ns = match &frame.class_namespace_obj { + let from_ns = match frame.class_namespace_obj.clone() { // PEP 3115 custom namespace observes the read. - Some(ns_obj) => self.class_ns_load(ns_obj, &name), + Some(ns_obj) => { + let g = frame.globals.clone(); + self.class_ns_load(&ns_obj, &name, &g)? + } None => frame .class_namespace .as_ref() @@ -6555,7 +6675,7 @@ impl Interpreter { let mapping = frame.pop()?; let v = match self.classdict_get(&mapping, &name)? { Some(v) => v, - None => self.lookup_global_or_builtin(&frame.globals, &name)?, + None => self.lookup_global_or_builtin(frame, &name)?, }; frame.push(v); } @@ -7354,7 +7474,7 @@ impl Interpreter { Rc::new(PyFrame { code: frame.code.clone(), globals: frame.globals.clone(), - builtins: self.builtins.clone(), + builtins: frame.builtins.clone(), lasti: Cell::new(lasti), back: RefCell::new(None), locals_cache: RefCell::new(None), @@ -7737,11 +7857,15 @@ impl Interpreter { ) } + /// `LOAD_GLOBAL` slow path: globals, then the *frame's* builtins + /// mapping (RFC 0052 WS5) — never the interpreter-wide dict + /// directly, so sandboxed `exec` namespaces stay sealed. fn lookup_global_or_builtin( &mut self, - globals: &Rc>, + frame: &Frame, name: &str, ) -> Result { + let globals = &frame.globals; let key = crate::object::StrKey(name); if let Some(v) = globals.borrow().get(&key) { return Ok(v.clone()); @@ -7765,7 +7889,7 @@ impl Interpreter { } } } - if let Some(v) = self.builtins.borrow().get(&key) { + if let Some(v) = frame.builtins.borrow().get(&key) { return Ok(v.clone()); } let err = name_error(format!("name '{name}' is not defined")); @@ -8575,6 +8699,17 @@ impl Interpreter { Object::BoundMethod(bm) => match name { "__func__" => Ok(bm.function.clone()), "__self__" => Ok(bm.receiver.clone()), + // `__call__` is defined on the method type itself, so it + // must NOT forward to `__func__` (which would re-bind to + // the wrapped function and drop the receiver): + // `null.__call__.__call__(f)` calls `null(f)` (PEP 614 + // decorator chains in test_grammar exercise this). + "__call__" => match crate::builtins::builtin_type_dunder("method", "__call__") { + Some(w) => self.descriptor_get(&w, obj, &Object::None), + None => Err(attribute_error( + "'method' object has no attribute '__call__'".to_owned(), + )), + }, // gh-113157: a bound method is its own (no-op) descriptor. // `m.__get__(obj, cls)` returns `m` unchanged instead of // forwarding to `__func__.__get__` and re-binding to `obj`. @@ -9515,6 +9650,32 @@ impl Interpreter { } } + // CPython `type_get_annotations` is a getset *data descriptor* on + // `type`, so it intercepts before the MRO walk below: reading + // `cls.__annotations__` consults only the class's *own* dict (a + // subclass must not see its base's annotations — + // test_annotations_inheritance) and lazily creates-and-caches an + // empty dict when absent (test_lazy_create_annotations). A custom + // metaclass overriding `__annotations__` was handled in (1). + if name == "__annotations__" && meta_attr.is_none() { + let key = DictKey(Object::from_static("__annotations__")); + if let Some(v) = ty.dict.borrow().get(&key) { + return Ok(v.clone()); + } + // Static (built-in) types have no writable dict to cache + // into — CPython raises AttributeError for + // `float.__annotations__` (test_annotations_getset_raises). + if ty.flags.is_builtin { + return Err(attribute_error(format!( + "type object '{}' has no attribute '__annotations__'", + ty.name + ))); + } + let fresh = Object::new_dict(); + ty.dict.borrow_mut().insert(key, fresh.clone()); + return Ok(fresh); + } + // (2) Look up the name in `ty` itself (and its MRO). if let Some(attr) = ty.lookup(name) { // Apply the descriptor protocol with no instance: classmethods @@ -9582,29 +9743,6 @@ impl Interpreter { // reads stay live. "__dict__" => return Ok(Object::MappingProxy(ty.dict.clone())), "__flags__" => return Ok(Object::Int(ty.flags_bits())), - // CPython `type_get_annotations`: reading `cls.__annotations__` - // consults only the class's *own* dict (no MRO inheritance — - // a subclass must not see its base's annotations) and lazily - // creates-and-caches an empty dict when absent - // (test_type_annotations `test_lazy_create_annotations`). - "__annotations__" => { - let key = DictKey(Object::from_static("__annotations__")); - if let Some(v) = ty.dict.borrow().get(&key) { - return Ok(v.clone()); - } - // Static (built-in) types have no writable dict to cache - // into — CPython raises AttributeError for - // `float.__annotations__` (test_annotations_getset_raises). - if ty.flags.is_builtin { - return Err(attribute_error(format!( - "type object '{}' has no attribute '__annotations__'", - ty.name - ))); - } - let fresh = Object::new_dict(); - ty.dict.borrow_mut().insert(key, fresh.clone()); - return Ok(fresh); - } // CPython `tp_basicsize`/`tp_itemsize`. Only `int` reports a // nonzero itemsize among the types tests interrogate // (`test_long.test___sizeof__` cross-checks it against @@ -17614,7 +17752,11 @@ impl Interpreter { builtins_id, key_idx, } => { - if specialize::rc_id(&self.builtins) != builtins_id { + // Guard against the *frame's* builtins (RFC 0052 WS5): + // a sandboxed `exec` frame carries its own mapping, and + // the fingerprint check deopts rather than leak the + // ambient builtins through a stale cache. + if specialize::rc_id(&frame.builtins) != builtins_id { return self.deopt_load_global_slow(frame, cache_pc, name_idx); } // Guard that the name *isn't* shadowed in globals @@ -17635,7 +17777,7 @@ impl Interpreter { { return self.deopt_load_global_slow(frame, cache_pc, name_idx); } - let b = self.builtins.borrow(); + let b = frame.builtins.borrow(); if let Some((k, v)) = b.get_index(key_idx as usize) { // Same staleness guard as LoadGlobalModule: a removal from // the builtins dict renumbers slots without changing its @@ -17653,7 +17795,7 @@ impl Interpreter { specialize::record_specialize_attempt(op_idx); let decision = specialize::attempt_specialize_load_global( &frame.globals, - &self.builtins, + &frame.builtins, &name, ); frame.code.caches.set(cache_pc, decision); @@ -17662,7 +17804,7 @@ impl Interpreter { } else { specialize::record_specialize_success(op_idx); } - self.lookup_global_or_builtin(&frame.globals, &name) + self.lookup_global_or_builtin(frame, &name) } IC::Cooldown(n) => { let next = if n > 0 { @@ -17672,11 +17814,11 @@ impl Interpreter { }; frame.code.caches.set(cache_pc, next); let name = self.name_at(&frame.code, name_idx)?; - self.lookup_global_or_builtin(&frame.globals, &name) + self.lookup_global_or_builtin(frame, &name) } _ => { let name = self.name_at(&frame.code, name_idx)?; - self.lookup_global_or_builtin(&frame.globals, &name) + self.lookup_global_or_builtin(frame, &name) } } } @@ -17695,7 +17837,7 @@ impl Interpreter { .caches .set(cache_pc, weavepy_compiler::InlineCache::Cooldown(COOLDOWN)); let name = self.name_at(&frame.code, name_idx)?; - self.lookup_global_or_builtin(&frame.globals, &name) + self.lookup_global_or_builtin(frame, &name) } /// Specialized `LOAD_ATTR`. The receiver lives at TOS; on a @@ -18804,17 +18946,10 @@ impl Interpreter { self.set_type_attr_direct(ty, name, value) } Object::Module(m) => { - // The `builtins` module's attributes are the ambient - // builtins namespace in CPython (same dict object); - // WeavePy's module is a re-exposure, so writes mirror - // into the live lookup dict (`test_dynamic`'s - // `swap_attr(builtins, "len", …)` must affect every - // frame's name resolution immediately). - if m.name == "builtins" { - self.builtins - .borrow_mut() - .insert(DictKey(Object::from_str(name)), value.clone()); - } + // The `builtins` module's dict *is* the interpreter's + // ambient lookup namespace (RFC 0052 WS5), so no + // mirroring is needed: this plain insert is immediately + // visible to every frame's name resolution. m.dict .borrow_mut() .insert(DictKey(Object::from_str(name)), value); @@ -19446,13 +19581,6 @@ impl Interpreter { // `del module.attr` removes the name from the module dict // (CPython `module_setattro` with a NULL value). Object::Module(m) => { - // Mirror of the `store_attr` special case: deleting a - // `builtins` attribute unhooks it from live name lookup. - if m.name == "builtins" { - self.builtins - .borrow_mut() - .shift_remove(&DictKey(Object::from_str(name))); - } let removed = m .dict .borrow_mut() @@ -20928,6 +21056,18 @@ impl Interpreter { names.push(s.to_string()); } } + } else { + // A custom mapping namespace (`exec(src, g, m)`): + // CPython's `_dir_locals` is `PyMapping_Keys` + + // sort (test_compile + // test_exec_with_general_mapping_for_locals). + let keys_m = self.load_attr(&locals, "keys")?; + let keys = self.call(&keys_m, &[], &[], outer_globals)?; + for k in self.collect_iterable(&keys, outer_globals)? { + if let Object::Str(s) = &k { + names.push(s.to_string()); + } + } } names.sort(); return Ok(Object::new_list( @@ -21015,7 +21155,7 @@ impl Interpreter { return self.do_import(&name, &fromlist, level, outer_globals); } if b.name == "__vm:compile" { - return self.do_compile_call(args, outer_globals); + return self.do_compile_call(args, kwargs, outer_globals); } if b.name == "__vm:exec" { let merged = Self::merge_exec_kwargs("exec", args, kwargs)?; @@ -21699,6 +21839,17 @@ impl Interpreter { let target = self.descriptor_get(&bm.function, &bm.receiver, &owner)?; return self.call(&target, args, kwargs, outer_globals); } + // A *non-descriptor* instance found on the type — + // e.g. `unittest.mock` installs the child Mock + // itself as `type(m).__exit__`. CPython's + // `_PyObject_LookupSpecial` returns such an object + // as-is (no `__get__`, no binding), so the slot + // call passes only the protocol arguments; `self` + // must NOT be prepended (`with MagicMock():` calls + // `__exit__(t, v, tb)`, three args). + if matches!(&bm.function, Object::Instance(_)) { + return self.call(&bm.function, args, kwargs, outer_globals); + } } // `dict.__getitem__` invoked as a *bound method* on a `dict` // subclass — e.g. `quoter = Quoter(safe).__getitem__; @@ -21966,12 +22117,23 @@ impl Interpreter { code.freevars.len(), ))); } - // Seed `__builtins__` so the new function's frames resolve - // builtins even with a bare `{}` globals dict. + // Resolve `func_builtins` from the supplied globals now + // (CPython's `PyFunction_New`), falling back to the running + // interpreter's dict for a bare `{}` globals so calls still see + // builtins. (No `self` here — `function_type_call` is a static + // type-constructor; the seed snapshot shares the same dict.) + let builtins = match globals.borrow().get(&crate::object::StrKey("__builtins__")) { + Some(Object::Dict(d)) => d.clone(), + Some(Object::Module(m)) => m.dict.clone(), + _ => crate::vm_singletons::snapshot_interpreter() + .map(|i| i.builtins.clone()) + .unwrap_or_default(), + }; Ok(Object::Function(Rc::new(crate::object::PyFunction { name, code: RefCell::new(code), globals, + builtins, defaults, kw_defaults: vec![], closure, @@ -22037,15 +22199,42 @@ impl Interpreter { } } - fn class_ns_load(&self, ns_obj: &Object, name: &str) -> Option { + fn class_ns_load( + &mut self, + ns_obj: &Object, + name: &str, + globals: &Rc>, + ) -> Result, RuntimeError> { let key = DictKey(Object::from_str(name)); match ns_obj { - Object::Dict(d) => d.borrow().get(&key).cloned(), - Object::Instance(inst) => match inst.native.get() { - Some(Object::Dict(d)) => d.borrow().get(&key).cloned(), - _ => None, - }, - _ => None, + Object::Dict(d) => Ok(d.borrow().get(&key).cloned()), + Object::Instance(inst) => { + // A dict subclass without a `__getitem__` override reads + // straight from the backing dict; anything else (custom + // `__prepare__` result, `exec(src, g, mapping)` locals, a + // dict subclass *with* an override) dispatches like + // CPython's LOAD_NAME `PyObject_GetItem(locals, name)`, + // treating KeyError as a miss (falling through to + // globals/builtins); other exceptions propagate. + let overridden = inst + .cls() + .lookup("__getitem__") + .is_some_and(|m| !matches!(m, Object::Builtin(_))); + if !overridden { + if let Some(Object::Dict(d)) = inst.native.get() { + return Ok(d.borrow().get(&key).cloned()); + } + } + let Some(m) = instance_method(ns_obj, "__getitem__") else { + return Ok(None); + }; + match self.call(&m, &[Object::from_str(name)], &[], globals) { + Ok(v) => Ok(Some(v)), + Err(RuntimeError::PyException(e)) if e.type_name() == "KeyError" => Ok(None), + Err(e) => Err(e), + } + } + _ => Ok(None), } } @@ -22433,7 +22622,7 @@ impl Interpreter { Vec::new(), body_fn.closure.clone(), body_fn.globals.clone(), - false, + Some(body_fn.builtins.clone()), ); if let Some(obj) = &ns_obj { frame.class_namespace_obj = Some(obj.clone()); @@ -22765,7 +22954,7 @@ impl Interpreter { Vec::new(), body_fn.closure.clone(), body_fn.globals.clone(), - false, + Some(body_fn.builtins.clone()), ); frame.class_namespace = Some(class_ns.clone()); // PEP 695: seed any `__classdict__` cell (see the main build path). @@ -25097,7 +25286,7 @@ impl Interpreter { positional, f.closure.clone(), f.globals.clone(), - false, + Some(f.builtins.clone()), ); if code.is_generator || code.is_coroutine || code.is_async_generator { // `cr_origin`: when origin tracking is on, snapshot the @@ -25192,6 +25381,52 @@ impl Interpreter { } } + /// Bootstrap a generator/coroutine *code object* frame that was not + /// entered through a `PyFunction` call — `eval(co)` / `exec(co)` of + /// PyCF_ALLOW_TOP_LEVEL_AWAIT module code (RFC 0052). Runs the frame + /// to its leading `RETURN_GENERATOR` and wraps it in the right + /// generator flavour, exactly like the function-call bootstrap. + fn start_generator_code_frame( + &mut self, + code: &Rc, + mut frame: Frame, + ) -> Result { + match self.run_until_yield_or_return(&mut frame, None)? { + FrameOutcome::StartGenerator => { + let kind = if code.is_coroutine { + crate::object::CoroutineKind::Coroutine + } else if code.is_async_generator { + crate::object::CoroutineKind::AsyncGenerator + } else { + crate::object::CoroutineKind::Generator + }; + let gen_code = Object::Code(frame.code.clone()); + let gen = Rc::new(PyGenerator::new( + code.name.clone(), + code.qualname.clone(), + kind, + gen_code, + Box::new(frame), + )); + let obj = if code.is_coroutine { + Object::Coroutine(gen) + } else if code.is_async_generator { + Object::AsyncGenerator(gen) + } else { + Object::Generator(gen) + }; + gc_trace::track(obj.clone()); + if gc_trace::maybe_auto_collect() { + self.run_pending_finalizers(); + } + Ok(obj) + } + FrameOutcome::Returned(_) | FrameOutcome::Yielded(_) => Err(RuntimeError::Internal( + "generator bootstrap did not stop at RETURN_GENERATOR".to_owned(), + )), + } + } + /// RFC 0032 — specialized `CALL`. Mirrors the RFC 0021 dispatchers: /// a warm cache takes an argument-binding-free fast path for a /// pinned `PyFunction`; `Empty` runs the generic call and attempts @@ -25457,6 +25692,7 @@ impl Interpreter { cells: Vec::new(), stack: Vec::with_capacity(16), globals: f.globals.clone(), + builtins: f.builtins.clone(), class_namespace: None, class_namespace_obj: None, exc_handlers: Vec::new(), @@ -25478,8 +25714,13 @@ impl Interpreter { f: &Rc, args: Vec, ) -> Result { - let mut frame = - self.make_frame(f.code(), args, f.closure.clone(), f.globals.clone(), false); + let mut frame = self.make_frame( + f.code(), + args, + f.closure.clone(), + f.globals.clone(), + Some(f.builtins.clone()), + ); self.run_frame(&mut frame) } @@ -25670,7 +25911,26 @@ impl Interpreter { source: &str, filename: &str, ) -> Result { - let (parsed, warnings) = weavepy_parser::parse_module_with_warnings(source); + self.parse_source_emitting_warnings_flags(source, filename, 0) + } + + /// [`Self::parse_source_emitting_warnings`] with `CO_FUTURE_*` / + /// `PyCF_*` compile flags that affect *parsing* (currently just + /// PEP 401 `CO_FUTURE_BARRY_AS_BDFL`). + fn parse_source_emitting_warnings_flags( + &mut self, + source: &str, + filename: &str, + flags: u32, + ) -> Result { + let flufl = flags & weavepy_compiler::flags::CO_FUTURE_BARRY_AS_BDFL != 0; + let (parsed, mut warnings) = + weavepy_parser::parse_module_with_warnings_flags(source, flufl); + // CPython's compiler emits these at code-gen; parse success is + // the equivalent gate here (`PyCF_ONLY_AST` bypasses this path). + if let Ok(m) = &parsed { + collect_compiler_syntax_warnings(m, &mut warnings); + } // Emit warnings first: under `simplefilter('always')` they are // recorded and a later parse error still propagates; under // `simplefilter('error')` the first escape escalates to a @@ -25687,7 +25947,22 @@ impl Interpreter { source: &str, filename: &str, ) -> Result { - let (parsed, warnings) = weavepy_parser::parse_eval_with_warnings(source); + self.parse_eval_source_emitting_warnings_flags(source, filename, 0) + } + + /// [`Self::parse_eval_source_emitting_warnings`] with parse-affecting + /// compile flags (PEP 401). + fn parse_eval_source_emitting_warnings_flags( + &mut self, + source: &str, + filename: &str, + flags: u32, + ) -> Result { + let flufl = flags & weavepy_compiler::flags::CO_FUTURE_BARRY_AS_BDFL != 0; + let (parsed, mut warnings) = weavepy_parser::parse_eval_with_warnings_flags(source, flufl); + if let Ok(m) = &parsed { + collect_compiler_syntax_warnings(m, &mut warnings); + } self.emit_escape_warnings(source, filename, &warnings)?; parsed.map_err(|e| parse_error_to_syntax_error(&e, source, filename)) } @@ -26338,85 +26613,350 @@ impl Interpreter { fn do_compile_call( &mut self, args: &[Object], - _outer_globals: &Rc>, + kwargs: &[(String, Object)], + outer_globals: &Rc>, ) -> Result { - let filename = match args.get(1) { - Some(Object::Str(s)) => s.to_string(), - _ => "".to_owned(), - }; - let source = match args.first() { - Some(Object::Str(s)) => s.to_string(), - // Bytes sources go through PEP 263 detection (BOM + coding - // cookie), like CPython's `compile()`. - Some(Object::Bytes(b)) => decode_compile_source_bytes(b, &filename)?, - // An `ast.parse` result: `ast.parse` stashes the original - // text on the tree (`_weavepy_source`), which we recompile — - // CPython lowers the AST directly; we only support the - // unmodified round-trip. - Some(Object::Instance(inst)) => { - let src = inst - .dict - .borrow() - .get(&DictKey(Object::from_static("_weavepy_source"))) - .cloned(); - match src { - Some(Object::Str(s)) => s.to_string(), - _ => { - return Err(type_error( - "compile() argument 1 must be a string or bytes-like", - )) + use weavepy_compiler::flags as cf; + // ---- argument binding (CPython's exact parameter list) ---- + const NAMES: [&str; 6] = [ + "source", + "filename", + "mode", + "flags", + "dont_inherit", + "optimize", + ]; + let mut bound: [Option; 6] = [None, None, None, None, None, None]; + for (i, a) in args.iter().enumerate() { + if i >= NAMES.len() { + return Err(type_error(format!( + "compile() takes at most 6 arguments ({} given)", + args.len() + ))); + } + bound[i] = Some(a.clone()); + } + for (k, v) in kwargs { + // `_feature_version` is accepted and ignored (CPython uses + // it only for `ast.parse` version gating). + if k == "_feature_version" { + continue; + } + match NAMES.iter().position(|n| n == k) { + Some(i) => { + if bound[i].is_some() { + return Err(type_error(format!( + "argument for compile() given by name ('{k}') and position ({})", + i + 1 + ))); } + bound[i] = Some(v.clone()); + } + None => { + return Err(type_error(format!( + "'{k}' is an invalid keyword argument for compile()" + ))) } } - _ => { + } + let source_obj = bound[0] + .clone() + .ok_or_else(|| type_error("compile() missing required argument 'source' (pos 1)"))?; + let filename = match bound[1].clone() { + Some(v) => compile_filename_arg(self, &v)?, + None => { return Err(type_error( - "compile() argument 1 must be a string or bytes-like", + "compile() missing required argument 'filename' (pos 2)", )) } }; - let mode = match args.get(2) { + let mode = match bound[2].clone() { Some(Object::Str(s)) => s.to_string(), - _ => "exec".to_owned(), + Some(other) => { + return Err(type_error(format!( + "compile() argument 'mode' must be str, not {}", + other.type_name() + ))) + } + None => { + return Err(type_error( + "compile() missing required argument 'mode' (pos 3)", + )) + } + }; + let explicit_flags = match bound[3].clone() { + None | Some(Object::None) => 0i64, + Some(Object::Int(i)) => i, + Some(Object::Bool(b)) => i64::from(b), + Some(Object::Long(_)) => { + return Err(crate::error::overflow_error( + "Python int too large to convert to C int", + )) + } + Some(other) => { + return Err(type_error(format!( + "compile() argument 'flags' must be int, not {}", + other.type_name() + ))) + } + }; + let dont_inherit = match bound[4].clone() { + None | Some(Object::None) => false, + // Full `PyObject_IsTrue`: a raising `__bool__` must propagate + // (test_compile_filename_refleak's EvilBool). + Some(v) => self.obj_truthy(&v, outer_globals)?, + }; + let optimize = match bound[5].clone() { + None | Some(Object::None) => -1i64, + Some(Object::Int(i)) => i, + Some(Object::Bool(b)) => i64::from(b), + Some(Object::Long(_)) => { + return Err(crate::error::overflow_error( + "Python int too large to convert to C int", + )) + } + Some(other) => { + return Err(type_error(format!( + "compile() argument 'optimize' must be int, not {}", + other.type_name() + ))) + } }; + + // ---- flag validation (CPython builtin_compile_impl) ---- + let all_known = i64::from(cf::PYCF_MASK | cf::PYCF_MASK_OBSOLETE | cf::PYCF_COMPILE_MASK); + if explicit_flags & !all_known != 0 || explicit_flags < 0 { + return Err(crate::error::value_error("compile(): unrecognised flags")); + } + if !(-1..=2).contains(&optimize) { + return Err(crate::error::value_error( + "compile(): invalid optimize value", + )); + } + let resolved_optimize = if optimize < 0 { + self.optimize_level + } else { + optimize as u8 + }; + let explicit_flags = explicit_flags as u32; + // `dont_inherit=False` merges the *calling code's* future + // statements into the compile, exactly like CPython. + let inherited = if dont_inherit { + 0 + } else { + self.frame_stack + .borrow() + .last() + .map(|f| f.code.future_flags & cf::PYCF_MASK) + .unwrap_or(0) + }; + let opts = weavepy_compiler::CompileOptions { + flags: explicit_flags | inherited, + optimize: resolved_optimize, + }; + let only_ast = explicit_flags & cf::PYCF_ONLY_AST != 0; + + let Some(root_mode) = crate::stdlib::ast_convert::RootMode::from_mode(&mode) else { + return Err(crate::error::value_error( + "compile() mode must be 'exec', 'eval' or 'single'", + )); + }; + // PEP 578 — `compile` audits the call so security-sensitive // hosts can intercept dynamic code paths. crate::stdlib::sys::audit_event( "compile", - &[ - Object::from_str(source.clone()), - Object::from_str(filename.clone()), - ], + &[source_obj.clone(), Object::from_str(filename.clone())], ); - match mode.as_str() { - "exec" => { - let module = self.parse_source_emitting_warnings(&source, &filename)?; - let code = - weavepy_compiler::compile_module_with_source(&module, &source, &filename) - .map_err(|e| compile_error_to_syntax_error(&e, &source, &filename))?; + + let optimized_ast = explicit_flags & (cf::PYCF_OPTIMIZED_AST & !cf::PYCF_ONLY_AST) != 0; + + // ---- AST-object source (RFC 0052: real tree lowering) ---- + if crate::stdlib::ast_convert::is_ast_object(&source_obj) { + if only_ast { + // CPython validates and returns the tree (constant-folded + // when PyCF_OPTIMIZED_AST asks for it). + if optimized_ast { + return self.fold_ast_constants(&source_obj, outer_globals); + } + return Ok(source_obj); + } + let converted = crate::stdlib::ast_convert::convert_ast_root(&source_obj, root_mode)?; + let src = converted.synthetic_source; + let module = converted.module; + let code = match root_mode { + crate::stdlib::ast_convert::RootMode::Exec => { + weavepy_compiler::compile_module_with_options(&module, &src, &filename, opts) + } + crate::stdlib::ast_convert::RootMode::Eval => { + weavepy_compiler::compile_eval_with_options(&module, &src, &filename, opts) + } + crate::stdlib::ast_convert::RootMode::Single => { + weavepy_compiler::compile_interactive_with_options( + &module, &src, &filename, opts, + ) + } + } + .map_err(|e| compile_error_to_syntax_error(&e, &src, &filename))?; + return Ok(Object::Code(Rc::new(code))); + } + + // ---- textual source ---- + let source = match &source_obj { + Object::Str(s) => s.to_string(), + // Bytes sources go through PEP 263 detection (BOM + coding + // cookie), like CPython's `compile()`. `memoryview` (and any + // contiguous buffer) is accepted the same way. + Object::Bytes(b) => decode_compile_source_bytes(b, &filename)?, + Object::ByteArray(b) => decode_compile_source_bytes(&b.borrow(), &filename)?, + Object::MemoryView(mv) => decode_compile_source_bytes(&mv.to_bytes(), &filename)?, + Object::Instance(inst) => match inst.native.get() { + Some(Object::Str(s)) => s.to_string(), + Some(Object::Bytes(b)) => decode_compile_source_bytes(b, &filename)?, + _ => { + return Err(type_error( + "compile() arg 1 must be a string, bytes or AST object", + )) + } + }, + _ => { + return Err(type_error( + "compile() arg 1 must be a string, bytes or AST object", + )) + } + }; + check_compile_source_nulls(&source)?; + + // PyCF_ONLY_AST on text: parse and hand back a Python tree — + // `ast.parse` in builtin form. + if only_ast { + let tree = self.build_ast_object(&source, &filename, &mode, outer_globals)?; + if optimized_ast { + return self.fold_ast_constants(&tree, outer_globals); + } + return Ok(tree); + } + + match root_mode { + crate::stdlib::ast_convert::RootMode::Exec => { + let module = + self.parse_source_emitting_warnings_flags(&source, &filename, opts.flags)?; + let code = weavepy_compiler::compile_module_with_options( + &module, &source, &filename, opts, + ) + .map_err(|e| compile_error_to_syntax_error(&e, &source, &filename))?; Ok(Object::Code(Rc::new(code))) } - "eval" => { - let module = self.parse_eval_source_emitting_warnings(&source, &filename)?; - let code = weavepy_compiler::compile_eval_with_source(&module, &source, &filename) - .map_err(|e| compile_error_to_syntax_error(&e, &source, &filename))?; + crate::stdlib::ast_convert::RootMode::Eval => { + let module = + self.parse_eval_source_emitting_warnings_flags(&source, &filename, opts.flags)?; + let code = + weavepy_compiler::compile_eval_with_options(&module, &source, &filename, opts) + .map_err(|e| compile_error_to_syntax_error(&e, &source, &filename))?; Ok(Object::Code(Rc::new(code))) } // Interactive mode: top-level expression statements echo // through `sys.displayhook` (`PrintExpr`). Powers the REPL, // `code`/`codeop`, and `doctest`'s example execution. - "single" => { - let module = self.parse_source_emitting_warnings(&source, &filename)?; - let code = - weavepy_compiler::compile_interactive_with_source(&module, &source, &filename) - .map_err(|e| compile_error_to_syntax_error(&e, &source, &filename))?; + crate::stdlib::ast_convert::RootMode::Single => { + let module = + self.parse_source_emitting_warnings_flags(&source, &filename, opts.flags)?; + // pegen's `interactive` start rule accepts exactly one + // `statement_newline` — a second statement is a + // SyntaxError located at the end of the *first* one. + // Semicolon-joined simple statements on one physical + // line are a single `simple_stmts`, hence allowed + // (`import ast; ast.parse(...)`). + let line_of = |off: usize| source[..off.min(source.len())].matches('\n').count(); + if module.body.len() > 1 + && module.body.iter().any(|s| { + line_of(s.span.start.0 as usize) + != line_of(module.body[0].span.start.0 as usize) + }) + { + let end = module.body[0].span.end.0 as usize; + let lineno = source[..end.min(source.len())].matches('\n').count() as u32 + 1; + return Err(crate::error::syntax_error_located( + "multiple statements found while compiling a single statement", + Some(&filename), + Some(lineno), + Some(1), + source.lines().nth(lineno as usize - 1), + )); + } + // Interactive grammar: a compound statement needs a + // NEWLINE terminator, so a *one-line* compound with no + // trailing newline — `compile('def f(): pass', …, + // 'single')` — is a SyntaxError ("invalid syntax", + // offset 0). Multi-line blocks are fine: their last + // body line already produced the NEWLINE token. + if let Some(stmt) = module.body.first() { + use weavepy_parser::ast::StmtKind as SK; + let compound = matches!( + stmt.kind, + SK::FunctionDef { .. } + | SK::AsyncFunctionDef { .. } + | SK::ClassDef { .. } + | SK::If { .. } + | SK::While { .. } + | SK::For { .. } + | SK::AsyncFor { .. } + | SK::With { .. } + | SK::AsyncWith { .. } + | SK::Try { .. } + | SK::Match { .. } + ); + if compound && !source.contains('\n') { + return Err(crate::error::syntax_error_located( + "invalid syntax", + Some(&filename), + Some(1), + Some(0), + source.lines().next(), + )); + } + } + let code = weavepy_compiler::compile_interactive_with_options( + &module, &source, &filename, opts, + ) + .map_err(|e| compile_error_to_syntax_error(&e, &source, &filename))?; Ok(Object::Code(Rc::new(code))) } - other => Err(crate::error::value_error(format!( - "compile() mode must be 'exec', 'eval' or 'single', not '{other}'" - ))), } } + /// `compile(src, f, mode, PyCF_ONLY_AST)`: parse `src` and build a + /// real `ast` node tree — parse to the `_ast` spec form, then let + /// the frozen `ast` module rebuild node instances. + fn build_ast_object( + &mut self, + source: &str, + filename: &str, + mode: &str, + outer_globals: &Rc>, + ) -> Result { + let spec = crate::stdlib::ast_mod::parse(&[ + Object::from_str(source.to_owned()), + Object::from_str(filename.to_owned()), + Object::from_str(mode.to_owned()), + ])?; + let ast_module = self.do_import("ast", &Object::None, 0, outer_globals)?; + let builder = self.load_attr(&ast_module, "_from_spec")?; + self.call(&builder, &[spec], &[], outer_globals) + } + + /// PyCF_OPTIMIZED_AST: run the frozen `ast` module's constant folder + /// over a node tree (the pure-Python analogue of `ast_opt.c`). + fn fold_ast_constants( + &mut self, + tree: &Object, + outer_globals: &Rc>, + ) -> Result { + let ast_module = self.do_import("ast", &Object::None, 0, outer_globals)?; + let folder = self.load_attr(&ast_module, "_fold_constants")?; + self.call(&folder, &[tree.clone()], &[], outer_globals) + } + /// Python 3.13 made `globals`/`locals` passable by keyword on /// `exec`/`eval`. Fold keyword arguments into positional slots. fn merge_exec_kwargs( @@ -26487,8 +27027,8 @@ impl Interpreter { // it first — exactly the class-body scoping the VM already models // via `class_namespace`. CPython drives `exec(src, g, l)` codegen // (e.g. `dataclasses` building `__init__`) this way. - let exec_locals: Option>> = match args.get(2) { - Some(Object::Dict(d)) if !Rc::ptr_eq(d, &globals_dict) => Some(d.clone()), + let exec_locals: Option = match args.get(2) { + Some(Object::Dict(d)) if !Rc::ptr_eq(d, &globals_dict) => Some(Object::Dict(d.clone())), Some(Object::Dict(_)) => None, // Defaulted: like `eval`, CPython falls back to the *calling // frame's* live locals (so `exec(c)` inside a function sees @@ -26504,13 +27044,25 @@ impl Interpreter { caller.and_then(|f| { f.invalidate_locals(); match f.locals() { - Object::Dict(d) if !Rc::ptr_eq(&d, &globals_dict) => Some(d), + Object::Dict(d) if !Rc::ptr_eq(&d, &globals_dict) => { + Some(Object::Dict(d)) + } _ => None, } }) } } - _ => return Err(type_error("exec() locals must be a mapping")), + // Any mapping is a legal locals namespace (CPython + // `PyMapping_Check`): name binds route through its + // `__setitem__`, reads through `__getitem__` + // (test_grammar.test_var_annot_custom_maps). + Some(other) => { + if instance_method(other, "__getitem__").is_some() { + Some(other.clone()) + } else { + return Err(type_error("exec() locals must be a mapping")); + } + } }; // Accept str, bytes/bytearray (decoded via the PEP 263 cookie, UTF-8 // default, like CPython), or an already-compiled code object. @@ -26521,6 +27073,10 @@ impl Interpreter { Object::ByteArray(b) => { Some(crate::decode_compile_source_bytes(&b.borrow(), "")?) } + Object::MemoryView(mv) => Some(crate::decode_compile_source_bytes( + &mv.to_bytes(), + "", + )?), other => { return Err(type_error(format!( "exec() expected str or code, got {}", @@ -26528,6 +27084,9 @@ impl Interpreter { ))) } }; + if let Some(src) = &exec_src { + check_compile_source_nulls(src)?; + } let code_rc = match (exec_src, source) { (None, Object::Code(c)) => { // CPython rejects closure code up front (`exec` takes no @@ -26546,9 +27105,13 @@ impl Interpreter { // never raises `ValueError` for bad syntax. Invalid-escape // `SyntaxWarning`s replay here too. let module = self.parse_source_emitting_warnings(&src, "")?; - let compiled = - weavepy_compiler::compile_module_with_source(&module, &src, "") - .map_err(|e| compile_error_to_syntax_error(&e, &src, ""))?; + let compiled = weavepy_compiler::compile_module_with_options( + &module, + &src, + "", + self.default_compile_options(), + ) + .map_err(|e| compile_error_to_syntax_error(&e, &src, ""))?; Rc::new(compiled) } _ => unreachable!(), @@ -26565,10 +27128,20 @@ impl Interpreter { ); } } - let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals_dict, true); + let mut frame = + self.make_frame(code_rc.clone(), Vec::new(), Vec::new(), globals_dict, None); // Run top-level names into the distinct locals mapping when present. - if let Some(locals) = exec_locals { - frame.class_namespace = Some(locals); + match exec_locals { + Some(Object::Dict(d)) => frame.class_namespace = Some(d), + Some(mapping) => frame.class_namespace_obj = Some(mapping), + None => {} + } + // PyCF_ALLOW_TOP_LEVEL_AWAIT module code is a coroutine; CPython's + // `exec` builds the coroutine object and discards it (the caller + // gets None and a "never awaited" situation, not a crash). + if code_rc.is_generator || code_rc.is_coroutine || code_rc.is_async_generator { + self.start_generator_code_frame(&code_rc, frame)?; + return Ok(Object::None); } self.run_frame(&mut frame)?; Ok(Object::None) @@ -26666,7 +27239,14 @@ impl Interpreter { // `eval`-mode code returns its expression value (see // `compile_eval_with_source`); run it in the combined // namespace and hand that value straight back. - let mut frame = self.make_frame(c, Vec::new(), Vec::new(), ns.clone(), true); + let mut frame = + self.make_frame(c.clone(), Vec::new(), Vec::new(), ns.clone(), None); + // PyCF_ALLOW_TOP_LEVEL_AWAIT module code is a coroutine: + // `eval(co)` hands back the coroutine object for the + // caller to await (`asyncio.run(eval(co, g))`). + if c.is_generator || c.is_coroutine || c.is_async_generator { + return self.start_generator_code_frame(&c, frame); + } return self.run_frame(&mut frame); } Object::Str(s) => s.to_string(), @@ -26675,6 +27255,9 @@ impl Interpreter { // `test_subprocess` round-trips child stdout through `eval(...)`. Object::Bytes(b) => crate::decode_compile_source_bytes(&b, "")?, Object::ByteArray(b) => crate::decode_compile_source_bytes(&b.borrow(), "")?, + Object::MemoryView(mv) => { + crate::decode_compile_source_bytes(&mv.to_bytes(), "")? + } other => { return Err(type_error(format!( "eval() expected str or code, got {}", @@ -26682,6 +27265,7 @@ impl Interpreter { ))) } }; + check_compile_source_nulls(&src)?; // `eval` evaluates a single expression. CPython tolerates leading // whitespace/newlines in the source, so trim them, then compile in // *eval mode* — the resulting code object yields the expression's @@ -26691,9 +27275,14 @@ impl Interpreter { // call `eval("f'...'")` and assert `SyntaxError`) rely on. let trimmed = src.trim_start_matches([' ', '\t', '\n', '\r', '\x0c']); let module = self.parse_eval_source_emitting_warnings(trimmed, "")?; - let code = weavepy_compiler::compile_eval_with_source(&module, trimmed, "") - .map_err(|e| compile_error_to_syntax_error(&e, trimmed, ""))?; - let mut frame = self.make_frame(Rc::new(code), Vec::new(), Vec::new(), ns.clone(), true); + let code = weavepy_compiler::compile_eval_with_options( + &module, + trimmed, + "", + self.default_compile_options(), + ) + .map_err(|e| compile_error_to_syntax_error(&e, trimmed, ""))?; + let mut frame = self.make_frame(Rc::new(code), Vec::new(), Vec::new(), ns.clone(), None); self.run_frame(&mut frame) } @@ -27207,7 +27796,7 @@ impl Interpreter { // Register before executing so circular imports observe the partial // module (CPython's `_load` contract). self.cache.insert(full, module_obj.clone()); - let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals, true); + let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals, None); if let Err(e) = self.run_frame(&mut frame) { self.cache.remove(full); return Err(e); @@ -27314,7 +27903,7 @@ impl Interpreter { })); self.cache.insert(full, module_obj.clone()); let code_rc = Rc::new(code); - let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals, true); + let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals, None); if let Err(e) = self.run_frame(&mut frame) { self.cache.remove(full); return Err(e); @@ -27346,7 +27935,10 @@ impl Interpreter { is_package: bool, ) -> Result { let filename = path.to_string_lossy().into_owned(); - let (code, source_for_diag) = if let Some(cached) = crate::pycache::try_load(path) { + let (code, source_for_diag) = if let Some(cached) = (self.optimize_level == 0) + .then(|| crate::pycache::try_load(path)) + .flatten() + { (cached, String::new()) } else { // PEP 263: imported sources go through the same BOM/coding-cookie @@ -27357,9 +27949,17 @@ impl Interpreter { let source = decode_source_bytes(&raw, &filename)?; let module = weavepy_parser::parse_module(&source) .map_err(|e| parse_error_to_syntax_error(&e, &source, &filename))?; - let code = weavepy_compiler::compile_module_with_source(&module, &source, &filename) - .map_err(|e| compile_error_to_syntax_error(&e, &source, &filename))?; - if !self.bytecode_writes_disabled() { + let code = weavepy_compiler::compile_module_with_options( + &module, + &source, + &filename, + self.default_compile_options(), + ) + .map_err(|e| compile_error_to_syntax_error(&e, &source, &filename))?; + // The bytecode cache is keyed by path only (no `.opt-N` + // variants like CPython) — never poison it with `-O`-level + // code a later default-level run would reuse. + if !self.bytecode_writes_disabled() && self.optimize_level == 0 { crate::pycache::try_write(path, &code); } (code, source) @@ -27396,7 +27996,7 @@ impl Interpreter { // Run the body. On failure, drop the partial module so a // subsequent retry can try again from scratch. let code_rc = Rc::new(code); - let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals, true); + let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals, None); if let Err(e) = self.run_frame(&mut frame) { self.cache.remove(full); return Err(e); @@ -27571,6 +28171,548 @@ impl Interpreter { /// `(1-based line, 1-based column, line text without newline)` for a byte /// offset into `source`. Drives `SyntaxError` `lineno`/`offset`/`text`. +/// CPython's compile-time SyntaxWarnings (Python/compile.c): +/// `compiler_check_compare` (`is` with a literal), `check_caller` +/// (calling a display: "perhaps you missed a comma?"), +/// `check_subscripter`/`check_index` (statically-invalid subscripts), +/// and the always-true `assert (x, "msg")` tuple warning. Collected as +/// parse-style warnings so [`Vm::emit_escape_warnings`] routes them +/// through the `warnings` machinery (and escalates to SyntaxError +/// under an `error` filter) exactly like escape warnings. +fn collect_compiler_syntax_warnings( + module: &weavepy_parser::Module, + out: &mut Vec, +) { + use weavepy_parser::ast::{CmpOp, Constant as C, Expr, ExprKind, Stmt, StmtKind}; + + fn literal_type_name(e: &Expr) -> Option<&'static str> { + fn is_const_expr(e: &Expr) -> bool { + match &e.kind { + ExprKind::Constant(_) => true, + ExprKind::Tuple(items) => items.iter().all(is_const_expr), + _ => false, + } + } + match &e.kind { + ExprKind::Constant(c) => match c { + C::Int(_) | C::BigInt(_) => Some("int"), + C::Float(_) => Some("float"), + C::Complex(..) => Some("complex"), + C::Str(_) => Some("str"), + C::Bytes(_) => Some("bytes"), + // None / True / False / Ellipsis are legitimate `is` + // operands (check_is_arg). + _ => None, + }, + // AST constant folding turns all-literal tuple displays into + // Constant tuples before CPython's codegen sees them. + ExprKind::Tuple(items) if items.iter().all(is_const_expr) => Some("tuple"), + _ => None, + } + } + + /// CPython `infer_type` (compile.c): the runtime type a display or + /// literal expression is statically known to produce. + fn infer_type_name(e: &Expr) -> Option<&'static str> { + match &e.kind { + ExprKind::Tuple(_) => Some("tuple"), + ExprKind::List(_) | ExprKind::ListComp { .. } => Some("list"), + ExprKind::Dict { .. } | ExprKind::DictComp { .. } => Some("dict"), + ExprKind::Set(_) | ExprKind::SetComp { .. } => Some("set"), + ExprKind::GeneratorExp { .. } => Some("generator"), + ExprKind::Lambda { .. } => Some("function"), + ExprKind::JoinedStr(_) | ExprKind::FormattedValue { .. } => Some("str"), + ExprKind::Constant(c) => Some(match c { + C::None => "NoneType", + C::Bool(_) => "bool", + C::Int(_) | C::BigInt(_) => "int", + C::Float(_) => "float", + C::Complex(..) => "complex", + C::Bytes(_) => "bytes", + C::Ellipsis => "ellipsis", + _ => "str", + }), + _ => None, + } + } + + /// CPython `check_caller`: calling a display or literal draws + /// "'X' object is not callable; perhaps you missed a comma?". + fn check_caller(func: &Expr, out: &mut Vec) { + if matches!( + &func.kind, + ExprKind::Constant(_) + | ExprKind::Tuple(_) + | ExprKind::List(_) + | ExprKind::ListComp { .. } + | ExprKind::Dict { .. } + | ExprKind::DictComp { .. } + | ExprKind::Set(_) + | ExprKind::SetComp { .. } + | ExprKind::GeneratorExp { .. } + | ExprKind::JoinedStr(_) + | ExprKind::FormattedValue { .. } + ) { + if let Some(name) = infer_type_name(func) { + out.push(weavepy_parser::EscapeWarning { + offset: func.span.start.0, + message: format!( + "'{name}' object is not callable; perhaps you missed a comma?" + ), + }); + } + } + } + + /// CPython `check_subscripter` + `check_index`: subscripting a + /// value statically known not to support it (or with a statically + /// bad index type) draws the corresponding SyntaxWarning. + fn check_subscript(value: &Expr, slice: &Expr, out: &mut Vec) { + let not_subscriptable = match &value.kind { + ExprKind::Constant(c) => matches!( + c, + C::None + | C::Ellipsis + | C::Bool(_) + | C::Int(_) + | C::BigInt(_) + | C::Float(_) + | C::Complex(..) + ), + ExprKind::Set(_) + | ExprKind::SetComp { .. } + | ExprKind::GeneratorExp { .. } + | ExprKind::Lambda { .. } => true, + _ => false, + }; + if not_subscriptable { + if let Some(name) = infer_type_name(value) { + out.push(weavepy_parser::EscapeWarning { + offset: value.span.start.0, + message: format!( + "'{name}' object is not subscriptable; perhaps you missed a comma?" + ), + }); + } + return; + } + // check_index: container is a str/bytes/tuple constant or a + // tuple/list/f-string display; index is statically known and + // neither an int (incl. bool) nor a slice. + let Some(index_name) = infer_type_name(slice) else { + return; + }; + if matches!(index_name, "int" | "bool") { + return; + } + let container_ok = match &value.kind { + ExprKind::Constant(c) => matches!(c, C::Str(_) | C::Bytes(_)), + ExprKind::Tuple(_) + | ExprKind::List(_) + | ExprKind::ListComp { .. } + | ExprKind::JoinedStr(_) + | ExprKind::FormattedValue { .. } => true, + _ => false, + }; + if container_ok { + if let Some(container_name) = infer_type_name(value) { + out.push(weavepy_parser::EscapeWarning { + offset: value.span.start.0, + message: format!( + "{container_name} indices must be integers or slices, not {index_name}; perhaps you missed a comma?" + ), + }); + } + } + } + + fn check_compare(e: &Expr, out: &mut Vec) { + let ExprKind::Compare { + left, + ops, + comparators, + } = &e.kind + else { + return; + }; + let mut prev = literal_type_name(left); + for (op, comp) in ops.iter().zip(comparators) { + let right = literal_type_name(comp); + if matches!(op, CmpOp::Is | CmpOp::IsNot) { + if let Some(name) = prev.or(right) { + let message = if matches!(op, CmpOp::Is) { + format!("\"is\" with '{name}' literal. Did you mean \"==\"?") + } else { + format!("\"is not\" with '{name}' literal. Did you mean \"!=\"?") + }; + out.push(weavepy_parser::EscapeWarning { + offset: e.span.start.0, + message, + }); + // CPython warns at most once per comparison chain. + return; + } + } + prev = right; + } + } + + fn visit_expr(e: &Expr, out: &mut Vec) { + check_compare(e, out); + match &e.kind { + ExprKind::Constant(_) | ExprKind::Name(_) => {} + ExprKind::Attribute { value, .. } + | ExprKind::Starred(value) + | ExprKind::YieldFrom(value) + | ExprKind::Await(value) => visit_expr(value, out), + ExprKind::Yield(v) => { + if let Some(v) = v { + visit_expr(v, out); + } + } + ExprKind::Subscript { value, slice } => { + check_subscript(value, slice, out); + visit_expr(value, out); + visit_expr(slice, out); + } + ExprKind::Slice { lower, upper, step } => { + for part in [lower, upper, step].into_iter().flatten() { + visit_expr(part, out); + } + } + ExprKind::BinOp { left, right, .. } => { + visit_expr(left, out); + visit_expr(right, out); + } + ExprKind::BoolOp { values, .. } => { + for v in values { + visit_expr(v, out); + } + } + ExprKind::UnaryOp { operand, .. } => visit_expr(operand, out), + ExprKind::Compare { + left, comparators, .. + } => { + visit_expr(left, out); + for c in comparators { + visit_expr(c, out); + } + } + ExprKind::IfExp { test, body, orelse } => { + visit_expr(test, out); + visit_expr(body, out); + visit_expr(orelse, out); + } + ExprKind::NamedExpr { target, value } => { + visit_expr(target, out); + visit_expr(value, out); + } + ExprKind::Lambda { args, body } | ExprKind::TypeParamFn { args, body } => { + visit_args(args, out); + visit_expr(body, out); + } + ExprKind::Call { + func, + args, + keywords, + } => { + check_caller(func, out); + visit_expr(func, out); + for a in args { + visit_expr(a, out); + } + for k in keywords { + visit_expr(&k.value, out); + } + } + ExprKind::Tuple(items) | ExprKind::List(items) | ExprKind::Set(items) => { + for it in items { + visit_expr(it, out); + } + } + ExprKind::Dict { keys, values } => { + for k in keys.iter().flatten() { + visit_expr(k, out); + } + for v in values { + visit_expr(v, out); + } + } + ExprKind::ListComp { elt, generators } + | ExprKind::SetComp { elt, generators } + | ExprKind::GeneratorExp { elt, generators } => { + visit_expr(elt, out); + visit_generators(generators, out); + } + ExprKind::DictComp { + key, + value, + generators, + } => { + visit_expr(key, out); + visit_expr(value, out); + visit_generators(generators, out); + } + ExprKind::JoinedStr(parts) => { + for p in parts { + visit_expr(p, out); + } + } + ExprKind::FormattedValue { + value, format_spec, .. + } => { + visit_expr(value, out); + if let Some(fs) = format_spec { + visit_expr(fs, out); + } + } + } + } + + /// Visit an assignment/deletion *target*: the outermost node is in + /// Store/Del context, where CPython's codegen skips the caller / + /// subscripter warnings — but nested value positions (the object + /// and index of a subscript target, say) are ordinary loads. + fn visit_target(e: &Expr, out: &mut Vec) { + match &e.kind { + ExprKind::Name(_) => {} + ExprKind::Attribute { value, .. } => visit_expr(value, out), + ExprKind::Subscript { value, slice } => { + visit_expr(value, out); + visit_expr(slice, out); + } + ExprKind::Tuple(items) | ExprKind::List(items) => { + for it in items { + visit_target(it, out); + } + } + ExprKind::Starred(inner) => visit_target(inner, out), + _ => visit_expr(e, out), + } + } + + fn visit_generators( + gens: &[weavepy_parser::ast::Comprehension], + out: &mut Vec, + ) { + for g in gens { + visit_target(&g.target, out); + visit_expr(&g.iter, out); + for i in &g.ifs { + visit_expr(i, out); + } + } + } + + fn visit_args( + args: &weavepy_parser::ast::Arguments, + out: &mut Vec, + ) { + for a in args + .posonlyargs + .iter() + .chain(&args.args) + .chain(&args.kwonlyargs) + .chain(args.vararg.iter()) + .chain(args.kwarg.iter()) + { + if let Some(ann) = &a.annotation { + visit_expr(ann, out); + } + } + for d in args + .defaults + .iter() + .chain(args.kw_defaults.iter().flatten()) + { + visit_expr(d, out); + } + } + + fn visit_stmt(s: &Stmt, out: &mut Vec) { + match &s.kind { + StmtKind::FunctionDef { + args, + body, + decorator_list, + returns, + .. + } + | StmtKind::AsyncFunctionDef { + args, + body, + decorator_list, + returns, + .. + } => { + visit_args(args, out); + for d in decorator_list { + visit_expr(d, out); + } + if let Some(r) = returns { + visit_expr(r, out); + } + visit_body(body, out); + } + StmtKind::ClassDef { + bases, + keywords, + body, + decorator_list, + .. + } => { + for b in bases { + visit_expr(b, out); + } + for k in keywords { + visit_expr(&k.value, out); + } + for d in decorator_list { + visit_expr(d, out); + } + visit_body(body, out); + } + StmtKind::Return(v) => { + if let Some(v) = v { + visit_expr(v, out); + } + } + StmtKind::Assign { targets, value } => { + for t in targets { + visit_target(t, out); + } + visit_expr(value, out); + } + StmtKind::TypeAlias { + type_params, value, .. + } => { + for tp in type_params { + if let weavepy_parser::ast::TypeParamKind::TypeVar { bound: Some(b) } = &tp.kind + { + visit_expr(b, out); + } + if let Some(d) = &tp.default { + visit_expr(d, out); + } + } + visit_expr(value, out); + } + StmtKind::AugAssign { target, value, .. } => { + visit_target(target, out); + visit_expr(value, out); + } + StmtKind::AnnAssign { + target, + annotation, + value, + .. + } => { + visit_target(target, out); + visit_expr(annotation, out); + if let Some(v) = value { + visit_expr(v, out); + } + } + StmtKind::If { test, body, orelse } | StmtKind::While { test, body, orelse } => { + visit_expr(test, out); + visit_body(body, out); + visit_body(orelse, out); + } + StmtKind::For { + target, + iter, + body, + orelse, + } + | StmtKind::AsyncFor { + target, + iter, + body, + orelse, + } => { + visit_target(target, out); + visit_expr(iter, out); + visit_body(body, out); + visit_body(orelse, out); + } + StmtKind::Try { + body, + handlers, + orelse, + finalbody, + } => { + visit_body(body, out); + for h in handlers { + if let Some(t) = &h.type_ { + visit_expr(t, out); + } + visit_body(&h.body, out); + } + visit_body(orelse, out); + visit_body(finalbody, out); + } + StmtKind::Raise { exc, cause } => { + for e in [exc, cause].into_iter().flatten() { + visit_expr(e, out); + } + } + StmtKind::With { items, body } | StmtKind::AsyncWith { items, body } => { + for it in items { + visit_expr(&it.context_expr, out); + if let Some(v) = &it.optional_vars { + visit_target(v, out); + } + } + visit_body(body, out); + } + StmtKind::Match { subject, cases } => { + visit_expr(subject, out); + for c in cases { + if let Some(g) = &c.guard { + visit_expr(g, out); + } + visit_body(&c.body, out); + } + } + StmtKind::Expr(e) => visit_expr(e, out), + StmtKind::Delete(targets) => { + for t in targets { + visit_target(t, out); + } + } + StmtKind::Assert { test, msg } => { + // CPython codegen: `assert (x, "msg")` — a non-empty + // tuple display is always truthy. + if matches!(&test.kind, ExprKind::Tuple(items) if !items.is_empty()) { + out.push(weavepy_parser::EscapeWarning { + offset: s.span.start.0, + message: "assertion is always true, perhaps remove parentheses?".to_owned(), + }); + } + visit_expr(test, out); + if let Some(m) = msg { + visit_expr(m, out); + } + } + StmtKind::Import(_) + | StmtKind::ImportFrom { .. } + | StmtKind::Global(_) + | StmtKind::Nonlocal(_) + | StmtKind::Pass + | StmtKind::Break + | StmtKind::Continue => {} + } + } + + fn visit_body(body: &[Stmt], out: &mut Vec) { + for s in body { + visit_stmt(s, out); + } + } + + visit_body(&module.body, out); +} + fn line_col_text(source: &str, byte: u32) -> (u32, u32, String) { let byte = (byte as usize).min(source.len()); let mut line_start = 0usize; @@ -27634,6 +28776,54 @@ pub fn decode_compile_source_bytes(bytes: &[u8], filename: &str) -> Result Result<(), RuntimeError> { + if source.contains('\0') { + return Err(crate::error::syntax_error( + "source code string cannot contain null bytes", + )); + } + Ok(()) +} + +/// Coerce `compile()`'s `filename` argument: `str`, `bytes`, or any +/// `os.PathLike` (via `__fspath__`), mirroring CPython's +/// `PyUnicode_FSDecoder`. +fn compile_filename_arg(interp: &mut Interpreter, obj: &Object) -> Result { + let reject = |o: &Object| { + type_error(format!( + "compile() argument 'filename' must be str, bytes or os.PathLike, not {}", + o.type_name() + )) + }; + match obj { + Object::Str(s) => Ok(s.to_string()), + Object::Bytes(b) => Ok(String::from_utf8_lossy(b).into_owned()), + Object::Instance(inst) => { + // str/bytes subclasses carry their value natively. + match inst.native.get() { + Some(Object::Str(s)) => return Ok(s.to_string()), + Some(Object::Bytes(b)) => return Ok(String::from_utf8_lossy(b).into_owned()), + _ => {} + } + let fspath = interp + .load_attr(obj, "__fspath__") + .map_err(|_| reject(obj))?; + match interp.call_object(fspath, &[], &[])? { + Object::Str(s) => Ok(s.to_string()), + Object::Bytes(b) => Ok(String::from_utf8_lossy(&b).into_owned()), + other => Err(type_error(format!( + "expected __fspath__() to return str or bytes, not {}", + other.type_name() + ))), + } + } + other => Err(reject(other)), + } +} + fn decode_source_bytes_inner( bytes: &[u8], filename: &str, @@ -27642,6 +28832,22 @@ fn decode_source_bytes_inner( let had_bom = bytes.starts_with(b"\xEF\xBB\xBF"); let payload = if had_bom { &bytes[3..] } else { bytes }; + // File sources: the NUL check fires before any decode attempt, so a + // file that is both non-UTF-8 *and* NUL-ridden reports the NUL + // (test_compile's "evil undecodable" cases). + if from_file { + if let Some(pos) = payload.iter().position(|&b| b == 0) { + let line = payload[..pos].iter().filter(|&&b| b == b'\n').count() + 1; + return Err(crate::error::syntax_error_located( + "source code cannot contain null bytes", + Some(filename), + Some(line as u32), + Some(1), + None, + )); + } + } + // PEP 263 cookie: `^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)` // on one of the first two lines. fn cookie_of_line(line: &[u8]) -> Option { diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 101eb53..30220db 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -2473,6 +2473,13 @@ pub struct PyFunction { pub code: RefCell>, /// Module-level globals shared with the defining module. pub globals: Rc>, + /// CPython `func_builtins`: the builtins mapping this function's + /// frames resolve names against. Resolved from + /// `globals['__builtins__']` once, at function creation, and never + /// re-read — rebinding `globals()['__builtins__']` between calls + /// must not change resolution (test_dynamic's + /// `test_cannot_replace_builtins_dict_between_calls`). + pub builtins: Rc>, pub defaults: Vec, pub kw_defaults: Vec<(String, Object)>, /// Closure cells matching `code.freevars` in order. diff --git a/crates/weavepy-vm/src/stdlib/ast_convert.rs b/crates/weavepy-vm/src/stdlib/ast_convert.rs new file mode 100644 index 0000000..ff7bcd3 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/ast_convert.rs @@ -0,0 +1,1226 @@ +//! Python-AST → Rust-AST lowering for `compile()` (RFC 0052). +//! +//! CPython's `compile()` accepts an `ast.AST` tree and compiles the +//! tree the caller built — the contract pytest's assertion rewriting, +//! coverage.py, and every AST-mutating tool rely on. This module is +//! WeavePy's analogue of `Python/Python-ast.c`'s `obj2ast_*` family: +//! it walks a Python node tree (any objects exposing the node class +//! names and `_fields`-shaped attributes) and rebuilds the +//! [`weavepy_parser::ast`] tree the compiler consumes. +//! +//! # Position synthesis +//! +//! The Rust AST carries *byte spans* into source text, while Python +//! AST nodes carry `(lineno, col_offset)` pairs — and a synthetic tree +//! has no source text at all. We bridge the two by building a +//! **synthetic source**: pass 1 walks the tree recording the maximum +//! column used on every line, pass 2 lays the lines out as runs of +//! spaces so that byte offset ↔ `(line, col)` is a bijection matching +//! the node positions exactly. The compiler's `LineIndex` over that +//! synthetic source then reproduces every node's line and column in +//! `co_positions()`, tracebacks, and error locations — CPython +//! likewise trusts the tree's positions verbatim. + +use crate::sync::Rc; + +use weavepy_lexer::token::Span; +use weavepy_parser::ast as past; + +use crate::error::{type_error, value_error, RuntimeError}; +use crate::object::{Object, StrKey}; +use crate::types::PyInstance; + +/// Which root node a `compile()` mode requires. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RootMode { + Exec, + Eval, + Single, +} + +impl RootMode { + pub fn from_mode(mode: &str) -> Option { + match mode { + "exec" => Some(Self::Exec), + "eval" => Some(Self::Eval), + "single" => Some(Self::Single), + _ => None, + } + } + + fn expected_node(self) -> &'static str { + match self { + Self::Exec => "Module", + Self::Eval => "Expression", + Self::Single => "Interactive", + } + } +} + +/// The result of lowering a Python AST object: a parser-shaped module +/// (an `Expression` root becomes a single-statement module the eval +/// compile entry accepts) plus the synthetic source whose `LineIndex` +/// reproduces the tree's positions. +#[derive(Debug)] +pub struct ConvertedAst { + pub module: past::Module, + pub synthetic_source: String, +} + +/// Is `obj` an AST node instance? Mirrors `PyAST_Check` by duck-typing +/// on the class hierarchy: every `ast.AST` subclass carries `_fields`. +pub fn is_ast_object(obj: &Object) -> bool { + match obj { + Object::Instance(inst) => inst.cls().lookup("_fields").is_some(), + _ => false, + } +} + +/// Lower a Python AST root object into a compilable module. +pub fn convert_ast_root(obj: &Object, mode: RootMode) -> Result { + let inst = match obj { + Object::Instance(inst) => inst, + _ => { + return Err(type_error(format!( + "expected {} node, got {}", + mode.expected_node(), + obj.type_name() + ))) + } + }; + let name = inst.cls().name.clone(); + if name != mode.expected_node() { + return Err(type_error(format!( + "expected {} node, got {}", + mode.expected_node(), + name + ))); + } + + // Pass 1: collect per-line maximum columns for span synthesis. + let mut collector = PosCollector::default(); + collector.walk(obj, 0)?; + let pos = collector.finish(); + + let mut conv = Conv { pos: &pos }; + let module = match mode { + RootMode::Exec | RootMode::Single => { + let body = field(inst, "body").ok_or_else(|| missing_field("body", &name))?; + past::Module { + body: conv.stmt_list(&body, &name)?, + } + } + RootMode::Eval => { + let body = field(inst, "body").ok_or_else(|| missing_field("body", "Expression"))?; + let expr = conv.expr(&body)?; + let span = expr.span; + past::Module { + body: vec![past::Stmt { + kind: past::StmtKind::Expr(expr), + span, + }], + } + } + }; + Ok(ConvertedAst { + module, + synthetic_source: pos.synthetic_source(), + }) +} + +// --------------------------------------------------------------------------- +// Position synthesis +// --------------------------------------------------------------------------- + +/// Recursion limit for tree walks — synthetic trees can be arbitrarily +/// deep; CPython fails with RecursionError, so do we. +const MAX_DEPTH: usize = 2000; + +#[derive(Default)] +struct PosCollector { + /// 1-based line → maximum (0-based) column observed. + line_max: std::collections::BTreeMap, +} + +impl PosCollector { + fn record(&mut self, line: i64, col: i64) { + if line >= 1 && col >= 0 { + let e = self.line_max.entry(line).or_insert(0); + if col > *e { + *e = col; + } + } + } + + fn walk(&mut self, obj: &Object, depth: usize) -> Result<(), RuntimeError> { + if depth > MAX_DEPTH { + return Err(crate::error::recursion_error( + "maximum recursion depth exceeded during compilation", + )); + } + match obj { + Object::Instance(inst) => { + let d = inst.dict.borrow(); + let int_of = |name: &str| -> Option { + match d.get(&StrKey(name)) { + Some(Object::Int(i)) => Some(*i), + Some(Object::Bool(b)) => Some(i64::from(*b)), + _ => None, + } + }; + if let (Some(l), Some(c)) = (int_of("lineno"), int_of("col_offset")) { + self.record(l, c); + } + if let (Some(l), Some(c)) = (int_of("end_lineno"), int_of("end_col_offset")) { + self.record(l, c); + } + // Recurse into attribute values (fields hold the child + // nodes; unrelated attributes are harmless to scan). + let children: Vec = d + .iter() + .filter(|(_, v)| matches!(v, Object::Instance(_) | Object::List(_))) + .map(|(_, v)| v.clone()) + .collect(); + drop(d); + for child in children { + self.walk(&child, depth + 1)?; + } + } + Object::List(items) => { + let snapshot: Vec = items.borrow().clone(); + for item in snapshot { + self.walk(&item, depth + 1)?; + } + } + _ => {} + } + Ok(()) + } + + fn finish(self) -> PosMap { + let max_line = self.line_max.keys().next_back().copied().unwrap_or(1); + let mut line_lengths = vec![0u32; max_line.max(1) as usize]; + for (line, col) in &self.line_max { + // Line length must admit the max column as a valid offset + // (plus one so `end == start + 1` spans stay in-line). + line_lengths[(*line - 1) as usize] = (*col as u32) + 1; + } + let mut line_starts = Vec::with_capacity(line_lengths.len()); + let mut acc = 0u32; + for len in &line_lengths { + line_starts.push(acc); + acc += len + 1; // '\n' + } + PosMap { + line_lengths, + line_starts, + } + } +} + +/// The byte-offset ↔ `(line, col)` bijection over the synthetic source. +struct PosMap { + line_lengths: Vec, + line_starts: Vec, +} + +impl PosMap { + fn byte(&self, line: i64, col: i64) -> u32 { + if line < 1 || self.line_starts.is_empty() { + return 0; + } + let idx = ((line - 1) as usize).min(self.line_starts.len() - 1); + let col = (col.max(0) as u32).min(self.line_lengths[idx]); + self.line_starts[idx] + col + } + + fn synthetic_source(&self) -> String { + let total: usize = self + .line_lengths + .iter() + .map(|l| *l as usize + 1) + .sum::(); + let mut s = String::with_capacity(total); + for len in &self.line_lengths { + for _ in 0..*len { + s.push(' '); + } + s.push('\n'); + } + s + } +} + +// --------------------------------------------------------------------------- +// Field access helpers +// --------------------------------------------------------------------------- + +fn field(inst: &Rc, name: &str) -> Option { + if let Some(v) = inst.dict.borrow().get(&StrKey(name)) { + return Some(v.clone()); + } + // Class-level defaults (CPython's obj2ast reads through + // `PyObject_GetAttr`, which sees class attributes too). + inst.cls().lookup(name) +} + +fn missing_field(name: &str, node: &str) -> RuntimeError { + type_error(format!("required field \"{name}\" missing from {node}")) +} + +/// The class name of a node instance, or a `TypeError` shaped like +/// CPython's "expected some sort of {category}, but got {value}". +fn node_name(obj: &Object, category: &str) -> Result<(Rc, String), RuntimeError> { + match obj { + Object::Instance(inst) => { + let name = inst.cls().name.clone(); + Ok((inst.clone(), name)) + } + other => Err(type_error(format!( + "expected some sort of {category}, but got {}", + repr_lite(other) + ))), + } +} + +/// A best-effort `repr` for error messages (no interpreter handle here). +fn repr_lite(obj: &Object) -> String { + match obj { + Object::None => "None".to_owned(), + Object::Bool(b) => if *b { "True" } else { "False" }.to_owned(), + Object::Int(i) => i.to_string(), + Object::Float(f) => f.to_string(), + Object::Str(s) => format!("{s:?}").replace('"', "'"), + Object::Instance(inst) => format!("<{} object>", inst.cls().name), + other => format!("<{}>", other.type_name()), + } +} + +fn identifier(obj: &Object) -> Result { + match obj { + Object::Str(s) => Ok(s.to_string()), + _ => Err(type_error("AST identifier must be of type str")), + } +} + +fn opt_identifier(obj: Option) -> Result, RuntimeError> { + match obj { + None | Some(Object::None) => Ok(None), + Some(v) => Ok(Some(identifier(&v)?)), + } +} + +fn int_field(obj: &Object, what: &str) -> Result { + match obj { + Object::Int(i) => Ok(*i), + Object::Bool(b) => Ok(i64::from(*b)), + _ => Err(value_error(format!( + "invalid integer value for field {what}" + ))), + } +} + +fn list_items(obj: &Object, node: &str, fieldname: &str) -> Result, RuntimeError> { + match obj { + Object::List(items) => Ok(items.borrow().clone()), + // CPython accepts only exact lists here. + other => Err(type_error(format!( + "{node} field \"{fieldname}\" must be a list, not a {}", + other.type_name() + ))), + } +} + +// --------------------------------------------------------------------------- +// Node conversion +// --------------------------------------------------------------------------- + +struct Conv<'a> { + pos: &'a PosMap, +} + +impl Conv<'_> { + fn span_of(&self, inst: &Rc, category: &str) -> Result { + let lineno = field(inst, "lineno") + .ok_or_else(|| missing_field("lineno", category)) + .and_then(|v| int_field(&v, "lineno"))?; + let col = field(inst, "col_offset") + .ok_or_else(|| missing_field("col_offset", category)) + .and_then(|v| int_field(&v, "col_offset"))?; + let end_lineno = match field(inst, "end_lineno") { + Some(Object::None) | None => lineno, + Some(v) => int_field(&v, "end_lineno")?, + }; + let end_col = match field(inst, "end_col_offset") { + Some(Object::None) | None => col, + Some(v) => int_field(&v, "end_col_offset")?, + }; + let start = self.pos.byte(lineno, col); + let end = self.pos.byte(end_lineno, end_col).max(start); + Ok(Span::new(start, end)) + } + + /// Optional positions (keyword / alias nodes may omit them). + fn span_opt(&self, inst: &Rc, fallback: Span) -> Span { + let int_of = |name: &str| -> Option { + match field(inst, name) { + Some(Object::Int(i)) => Some(i), + _ => None, + } + }; + match (int_of("lineno"), int_of("col_offset")) { + (Some(l), Some(c)) => { + let start = self.pos.byte(l, c); + let end = match (int_of("end_lineno"), int_of("end_col_offset")) { + (Some(el), Some(ec)) => self.pos.byte(el, ec).max(start), + _ => start, + }; + Span::new(start, end) + } + _ => fallback, + } + } + + fn stmt_list(&mut self, obj: &Object, node: &str) -> Result, RuntimeError> { + list_items(obj, node, "body")? + .iter() + .map(|s| self.stmt(s)) + .collect() + } + + fn expr_list( + &mut self, + obj: &Object, + node: &str, + fieldname: &str, + ) -> Result, RuntimeError> { + list_items(obj, node, fieldname)? + .iter() + .map(|e| self.expr(e)) + .collect() + } + + fn opt_expr(&mut self, obj: Option) -> Result, RuntimeError> { + match obj { + None | Some(Object::None) => Ok(None), + Some(v) => Ok(Some(self.expr(&v)?)), + } + } + + fn opt_boxed(&mut self, obj: Option) -> Result>, RuntimeError> { + Ok(self.opt_expr(obj)?.map(Box::new)) + } + + fn req(&self, inst: &Rc, name: &str, node: &str) -> Result { + field(inst, name).ok_or_else(|| missing_field(name, node)) + } + + // ---------------- statements ---------------- + + fn stmt(&mut self, obj: &Object) -> Result { + let (inst, name) = node_name(obj, "stmt")?; + let span = self.span_of(&inst, "stmt")?; + let kind = match name.as_str() { + "FunctionDef" | "AsyncFunctionDef" => { + let args = self.arguments(&self.req(&inst, "args", &name)?)?; + let body = self.stmt_list(&self.req(&inst, "body", &name)?, &name)?; + let decorator_list = self.expr_list( + &self.req(&inst, "decorator_list", &name)?, + &name, + "decorator_list", + )?; + let returns = self.opt_boxed(field(&inst, "returns"))?; + let type_params = self.type_params(field(&inst, "type_params"))?; + let fname = identifier(&self.req(&inst, "name", &name)?)?; + if name == "FunctionDef" { + past::StmtKind::FunctionDef { + name: fname, + args, + body, + decorator_list, + type_params, + returns, + } + } else { + past::StmtKind::AsyncFunctionDef { + name: fname, + args, + body, + decorator_list, + type_params, + returns, + } + } + } + "ClassDef" => past::StmtKind::ClassDef { + name: identifier(&self.req(&inst, "name", "ClassDef")?)?, + bases: self.expr_list( + &self.req(&inst, "bases", "ClassDef")?, + "ClassDef", + "bases", + )?, + keywords: self.keywords(&self.req(&inst, "keywords", "ClassDef")?)?, + body: self.stmt_list(&self.req(&inst, "body", "ClassDef")?, "ClassDef")?, + decorator_list: self.expr_list( + &self.req(&inst, "decorator_list", "ClassDef")?, + "ClassDef", + "decorator_list", + )?, + type_params: self.type_params(field(&inst, "type_params"))?, + }, + "Return" => past::StmtKind::Return(self.opt_expr(field(&inst, "value"))?), + "Delete" => past::StmtKind::Delete(self.expr_list( + &self.req(&inst, "targets", "Delete")?, + "Delete", + "targets", + )?), + "Assign" => past::StmtKind::Assign { + targets: self.expr_list( + &self.req(&inst, "targets", "Assign")?, + "Assign", + "targets", + )?, + value: self.expr(&self.req(&inst, "value", "Assign")?)?, + }, + "AugAssign" => past::StmtKind::AugAssign { + target: self.expr(&self.req(&inst, "target", "AugAssign")?)?, + op: bin_op(&self.req(&inst, "op", "AugAssign")?)?, + value: self.expr(&self.req(&inst, "value", "AugAssign")?)?, + }, + "AnnAssign" => past::StmtKind::AnnAssign { + target: self.expr(&self.req(&inst, "target", "AnnAssign")?)?, + annotation: self.expr(&self.req(&inst, "annotation", "AnnAssign")?)?, + value: self.opt_expr(field(&inst, "value"))?, + simple: int_field(&self.req(&inst, "simple", "AnnAssign")?, "simple")? != 0, + }, + "TypeAlias" => { + // The parser desugars `type X = v` to `X = + // __weavepy_type_alias__(...)`; mirror it for trees. + let target = self.expr(&self.req(&inst, "name", "TypeAlias")?)?; + let alias_name = match &target.kind { + past::ExprKind::Name(n) => n.clone(), + // CPython: Python/compile.c `codegen_typealias` message. + _ => return Err(type_error("TypeAlias with non-Name name")), + }; + let value = self.expr(&self.req(&inst, "value", "TypeAlias")?)?; + let type_params = self.type_params(field(&inst, "type_params"))?; + let rhs = weavepy_parser::build_lazy_type_alias( + &alias_name, + value, + &type_params, + target.span, + ); + past::StmtKind::Assign { + targets: vec![target], + value: rhs, + } + } + "For" | "AsyncFor" => { + let target = self.expr(&self.req(&inst, "target", &name)?)?; + let iter = self.expr(&self.req(&inst, "iter", &name)?)?; + let body = self.stmt_list(&self.req(&inst, "body", &name)?, &name)?; + let orelse = self.stmt_list(&self.req(&inst, "orelse", &name)?, &name)?; + if name == "For" { + past::StmtKind::For { + target, + iter, + body, + orelse, + } + } else { + past::StmtKind::AsyncFor { + target, + iter, + body, + orelse, + } + } + } + "While" => past::StmtKind::While { + test: self.expr(&self.req(&inst, "test", "While")?)?, + body: self.stmt_list(&self.req(&inst, "body", "While")?, "While")?, + orelse: self.stmt_list(&self.req(&inst, "orelse", "While")?, "While")?, + }, + "If" => past::StmtKind::If { + test: self.expr(&self.req(&inst, "test", "If")?)?, + body: self.stmt_list(&self.req(&inst, "body", "If")?, "If")?, + orelse: self.stmt_list(&self.req(&inst, "orelse", "If")?, "If")?, + }, + "With" | "AsyncWith" => { + let items = self.withitems(&self.req(&inst, "items", &name)?)?; + let body = self.stmt_list(&self.req(&inst, "body", &name)?, &name)?; + if name == "With" { + past::StmtKind::With { items, body } + } else { + past::StmtKind::AsyncWith { items, body } + } + } + "Match" => past::StmtKind::Match { + subject: self.expr(&self.req(&inst, "subject", "Match")?)?, + cases: list_items(&self.req(&inst, "cases", "Match")?, "Match", "cases")? + .iter() + .map(|c| self.match_case(c)) + .collect::>()?, + }, + "Raise" => past::StmtKind::Raise { + exc: self.opt_expr(field(&inst, "exc"))?, + cause: self.opt_expr(field(&inst, "cause"))?, + }, + "Try" | "TryStar" => { + let is_star = name == "TryStar"; + let handlers = list_items(&self.req(&inst, "handlers", &name)?, &name, "handlers")? + .iter() + .map(|h| self.handler(h, is_star)) + .collect::>()?; + past::StmtKind::Try { + body: self.stmt_list(&self.req(&inst, "body", &name)?, &name)?, + handlers, + orelse: self.stmt_list(&self.req(&inst, "orelse", &name)?, &name)?, + finalbody: self.stmt_list(&self.req(&inst, "finalbody", &name)?, &name)?, + } + } + "Assert" => past::StmtKind::Assert { + test: self.expr(&self.req(&inst, "test", "Assert")?)?, + msg: self.opt_expr(field(&inst, "msg"))?, + }, + "Import" => past::StmtKind::Import( + list_items(&self.req(&inst, "names", "Import")?, "Import", "names")? + .iter() + .map(alias) + .collect::>()?, + ), + "ImportFrom" => past::StmtKind::ImportFrom { + module: opt_identifier(field(&inst, "module"))?, + names: list_items( + &self.req(&inst, "names", "ImportFrom")?, + "ImportFrom", + "names", + )? + .iter() + .map(alias) + .collect::>()?, + level: match field(&inst, "level") { + None | Some(Object::None) => 0, + Some(v) => int_field(&v, "level")?.max(0) as u32, + }, + }, + "Global" => past::StmtKind::Global( + list_items(&self.req(&inst, "names", "Global")?, "Global", "names")? + .iter() + .map(identifier) + .collect::>()?, + ), + "Nonlocal" => past::StmtKind::Nonlocal( + list_items(&self.req(&inst, "names", "Nonlocal")?, "Nonlocal", "names")? + .iter() + .map(identifier) + .collect::>()?, + ), + "Expr" => past::StmtKind::Expr(self.expr(&self.req(&inst, "value", "Expr")?)?), + "Pass" => past::StmtKind::Pass, + "Break" => past::StmtKind::Break, + "Continue" => past::StmtKind::Continue, + other => { + return Err(type_error(format!( + "expected some sort of stmt, but got <{other} object>" + ))) + } + }; + Ok(past::Stmt { kind, span }) + } + + // ---------------- expressions ---------------- + + fn expr(&mut self, obj: &Object) -> Result { + let (inst, name) = node_name(obj, "expr")?; + let span = self.span_of(&inst, "expr")?; + let kind = match name.as_str() { + "Constant" => { + let value = self.req(&inst, "value", "Constant")?; + past::ExprKind::Constant(constant_value(&value)?) + } + "Name" => past::ExprKind::Name(identifier(&self.req(&inst, "id", "Name")?)?), + "Attribute" => past::ExprKind::Attribute { + value: Box::new(self.expr(&self.req(&inst, "value", "Attribute")?)?), + attr: identifier(&self.req(&inst, "attr", "Attribute")?)?, + }, + "Subscript" => past::ExprKind::Subscript { + value: Box::new(self.expr(&self.req(&inst, "value", "Subscript")?)?), + slice: Box::new(self.expr(&self.req(&inst, "slice", "Subscript")?)?), + }, + "Slice" => past::ExprKind::Slice { + lower: self.opt_boxed(field(&inst, "lower"))?, + upper: self.opt_boxed(field(&inst, "upper"))?, + step: self.opt_boxed(field(&inst, "step"))?, + }, + "BinOp" => past::ExprKind::BinOp { + left: Box::new(self.expr(&self.req(&inst, "left", "BinOp")?)?), + op: bin_op(&self.req(&inst, "op", "BinOp")?)?, + right: Box::new(self.expr(&self.req(&inst, "right", "BinOp")?)?), + }, + "BoolOp" => past::ExprKind::BoolOp { + op: bool_op(&self.req(&inst, "op", "BoolOp")?)?, + values: self.expr_list( + &self.req(&inst, "values", "BoolOp")?, + "BoolOp", + "values", + )?, + }, + "UnaryOp" => past::ExprKind::UnaryOp { + op: unary_op(&self.req(&inst, "op", "UnaryOp")?)?, + operand: Box::new(self.expr(&self.req(&inst, "operand", "UnaryOp")?)?), + }, + "Compare" => { + let ops: Vec<_> = + list_items(&self.req(&inst, "ops", "Compare")?, "Compare", "ops")? + .iter() + .map(cmp_op) + .collect::>()?; + let comparators = self.expr_list( + &self.req(&inst, "comparators", "Compare")?, + "Compare", + "comparators", + )?; + // CPython's validate_expr rejects these shapes before + // compilation; the compiler assumes ops/comparators pair up. + if comparators.is_empty() { + return Err(value_error("Compare with no comparators".to_owned())); + } + if ops.len() != comparators.len() { + return Err(value_error( + "Compare has a different number of comparators and operands".to_owned(), + )); + } + past::ExprKind::Compare { + left: Box::new(self.expr(&self.req(&inst, "left", "Compare")?)?), + ops, + comparators, + } + } + "IfExp" => past::ExprKind::IfExp { + test: Box::new(self.expr(&self.req(&inst, "test", "IfExp")?)?), + body: Box::new(self.expr(&self.req(&inst, "body", "IfExp")?)?), + orelse: Box::new(self.expr(&self.req(&inst, "orelse", "IfExp")?)?), + }, + "NamedExpr" => { + let target_obj = self.req(&inst, "target", "NamedExpr")?; + // CPython's `validate_expr` rejects this before + // compilation (gh-109351). + if !matches!(node_name(&target_obj, "expression"), Ok((_, n)) if n == "Name") { + return Err(type_error("NamedExpr target must be a Name")); + } + past::ExprKind::NamedExpr { + target: Box::new(self.expr(&target_obj)?), + value: Box::new(self.expr(&self.req(&inst, "value", "NamedExpr")?)?), + } + } + "Lambda" => past::ExprKind::Lambda { + args: self.arguments(&self.req(&inst, "args", "Lambda")?)?, + body: Box::new(self.expr(&self.req(&inst, "body", "Lambda")?)?), + }, + "Call" => past::ExprKind::Call { + func: Box::new(self.expr(&self.req(&inst, "func", "Call")?)?), + args: self.expr_list(&self.req(&inst, "args", "Call")?, "Call", "args")?, + keywords: self.keywords(&self.req(&inst, "keywords", "Call")?)?, + }, + "Tuple" => past::ExprKind::Tuple(self.expr_list( + &self.req(&inst, "elts", "Tuple")?, + "Tuple", + "elts", + )?), + "List" => past::ExprKind::List(self.expr_list( + &self.req(&inst, "elts", "List")?, + "List", + "elts", + )?), + "Set" => past::ExprKind::Set(self.expr_list( + &self.req(&inst, "elts", "Set")?, + "Set", + "elts", + )?), + "Dict" => { + let keys = list_items(&self.req(&inst, "keys", "Dict")?, "Dict", "keys")? + .iter() + .map(|k| match k { + Object::None => Ok(None), + other => Ok(Some(self.expr(other)?)), + }) + .collect::, RuntimeError>>()?; + past::ExprKind::Dict { + keys, + values: self.expr_list( + &self.req(&inst, "values", "Dict")?, + "Dict", + "values", + )?, + } + } + "ListComp" | "SetComp" | "GeneratorExp" => { + let elt = Box::new(self.expr(&self.req(&inst, "elt", &name)?)?); + let generators = + self.comprehensions(&self.req(&inst, "generators", &name)?, &name)?; + match name.as_str() { + "ListComp" => past::ExprKind::ListComp { elt, generators }, + "SetComp" => past::ExprKind::SetComp { elt, generators }, + _ => past::ExprKind::GeneratorExp { elt, generators }, + } + } + "DictComp" => past::ExprKind::DictComp { + key: Box::new(self.expr(&self.req(&inst, "key", "DictComp")?)?), + value: Box::new(self.expr(&self.req(&inst, "value", "DictComp")?)?), + generators: self + .comprehensions(&self.req(&inst, "generators", "DictComp")?, "DictComp")?, + }, + "Starred" => { + past::ExprKind::Starred(Box::new(self.expr(&self.req(&inst, "value", "Starred")?)?)) + } + "Yield" => past::ExprKind::Yield(self.opt_boxed(field(&inst, "value"))?), + "YieldFrom" => past::ExprKind::YieldFrom(Box::new(self.expr(&self.req( + &inst, + "value", + "YieldFrom", + )?)?)), + "Await" => { + past::ExprKind::Await(Box::new(self.expr(&self.req(&inst, "value", "Await")?)?)) + } + "JoinedStr" => past::ExprKind::JoinedStr(self.expr_list( + &self.req(&inst, "values", "JoinedStr")?, + "JoinedStr", + "values", + )?), + "FormattedValue" => past::ExprKind::FormattedValue { + value: Box::new(self.expr(&self.req(&inst, "value", "FormattedValue")?)?), + conversion: match field(&inst, "conversion") { + None | Some(Object::None) => -1, + Some(v) => int_field(&v, "conversion")? as i32, + }, + format_spec: self.opt_boxed(field(&inst, "format_spec"))?, + }, + other => { + return Err(type_error(format!( + "expected some sort of expr, but got <{other} object>" + ))) + } + }; + Ok(past::Expr { kind, span }) + } + + // ---------------- supporting nodes ---------------- + + fn arguments(&mut self, obj: &Object) -> Result { + let (inst, _name) = node_name(obj, "arguments")?; + let args_of = |conv: &mut Self, fieldname: &str| -> Result, RuntimeError> { + match field(&inst, fieldname) { + None | Some(Object::None) => Ok(Vec::new()), + Some(v) => list_items(&v, "arguments", fieldname)? + .iter() + .map(|a| conv.arg(a)) + .collect(), + } + }; + let posonlyargs = args_of(self, "posonlyargs")?; + let args = args_of(self, "args")?; + let kwonlyargs = args_of(self, "kwonlyargs")?; + let vararg = match field(&inst, "vararg") { + None | Some(Object::None) => None, + Some(v) => Some(self.arg(&v)?), + }; + let kwarg = match field(&inst, "kwarg") { + None | Some(Object::None) => None, + Some(v) => Some(self.arg(&v)?), + }; + let kw_defaults = match field(&inst, "kw_defaults") { + None | Some(Object::None) => Vec::new(), + Some(v) => list_items(&v, "arguments", "kw_defaults")? + .iter() + .map(|d| match d { + Object::None => Ok(None), + other => Ok(Some(self.expr(other)?)), + }) + .collect::, RuntimeError>>()?, + }; + let defaults = match field(&inst, "defaults") { + None | Some(Object::None) => Vec::new(), + Some(v) => list_items(&v, "arguments", "defaults")? + .iter() + .map(|e| self.expr(e)) + .collect::, RuntimeError>>()?, + }; + Ok(past::Arguments { + posonlyargs, + args, + vararg, + kwonlyargs, + kw_defaults, + kwarg, + defaults, + }) + } + + fn arg(&mut self, obj: &Object) -> Result { + let (inst, _name) = node_name(obj, "arg")?; + let span = self.span_of(&inst, "arg")?; + Ok(past::Arg { + name: identifier(&self.req(&inst, "arg", "arg")?)?, + annotation: self.opt_boxed(field(&inst, "annotation"))?, + span, + }) + } + + fn keywords(&mut self, obj: &Object) -> Result, RuntimeError> { + list_items(obj, "Call", "keywords")? + .iter() + .map(|k| { + let (inst, _name) = node_name(k, "keyword")?; + Ok(past::Keyword { + arg: opt_identifier(field(&inst, "arg"))?, + value: self.expr(&self.req(&inst, "value", "keyword")?)?, + }) + }) + .collect() + } + + fn comprehensions( + &mut self, + obj: &Object, + node: &str, + ) -> Result, RuntimeError> { + list_items(obj, node, "generators")? + .iter() + .map(|c| { + let (inst, _name) = node_name(c, "comprehension")?; + Ok(past::Comprehension { + target: self.expr(&self.req(&inst, "target", "comprehension")?)?, + iter: self.expr(&self.req(&inst, "iter", "comprehension")?)?, + ifs: self.expr_list( + &self.req(&inst, "ifs", "comprehension")?, + "comprehension", + "ifs", + )?, + is_async: match field(&inst, "is_async") { + None | Some(Object::None) => false, + Some(v) => int_field(&v, "is_async")? != 0, + }, + }) + }) + .collect() + } + + fn withitems(&mut self, obj: &Object) -> Result, RuntimeError> { + list_items(obj, "With", "items")? + .iter() + .map(|w| { + let (inst, _name) = node_name(w, "withitem")?; + Ok(past::WithItem { + context_expr: self.expr(&self.req(&inst, "context_expr", "withitem")?)?, + optional_vars: self.opt_expr(field(&inst, "optional_vars"))?, + }) + }) + .collect() + } + + fn handler( + &mut self, + obj: &Object, + is_star: bool, + ) -> Result { + let (inst, _name) = node_name(obj, "excepthandler")?; + let span = self.span_of(&inst, "excepthandler")?; + Ok(past::ExceptHandler { + type_: self.opt_expr(field(&inst, "type"))?, + name: opt_identifier(field(&inst, "name"))?, + body: self.stmt_list(&self.req(&inst, "body", "ExceptHandler")?, "ExceptHandler")?, + span, + is_star, + }) + } + + fn match_case(&mut self, obj: &Object) -> Result { + let (inst, _name) = node_name(obj, "match_case")?; + let pattern_obj = self.req(&inst, "pattern", "match_case")?; + let pattern = self.pattern(&pattern_obj)?; + // `match_case` carries no positions in CPython; fall back to + // the pattern node's span. + let span = match &pattern_obj { + Object::Instance(pinst) => self.span_opt(pinst, Span::new(0, 0)), + _ => Span::new(0, 0), + }; + Ok(past::MatchCase { + pattern, + guard: self.opt_expr(field(&inst, "guard"))?, + body: self.stmt_list(&self.req(&inst, "body", "match_case")?, "match_case")?, + span, + }) + } + + fn pattern(&mut self, obj: &Object) -> Result { + let (inst, name) = node_name(obj, "pattern")?; + Ok(match name.as_str() { + "MatchValue" => { + past::Pattern::Value(self.expr(&self.req(&inst, "value", "MatchValue")?)?) + } + "MatchSingleton" => past::Pattern::Singleton(constant_value(&self.req( + &inst, + "value", + "MatchSingleton", + )?)?), + "MatchSequence" => past::Pattern::Sequence( + list_items( + &self.req(&inst, "patterns", "MatchSequence")?, + "MatchSequence", + "patterns", + )? + .iter() + .map(|p| self.pattern(p)) + .collect::>()?, + ), + "MatchStar" => past::Pattern::Star(opt_identifier(field(&inst, "name"))?), + "MatchMapping" => past::Pattern::Mapping { + keys: self.expr_list( + &self.req(&inst, "keys", "MatchMapping")?, + "MatchMapping", + "keys", + )?, + patterns: list_items( + &self.req(&inst, "patterns", "MatchMapping")?, + "MatchMapping", + "patterns", + )? + .iter() + .map(|p| self.pattern(p)) + .collect::>()?, + rest: match opt_identifier(field(&inst, "rest"))? { + Some(n) => Some(Some(n)), + None => None, + }, + }, + "MatchClass" => { + let kwd_attrs = list_items( + &self.req(&inst, "kwd_attrs", "MatchClass")?, + "MatchClass", + "kwd_attrs", + )? + .iter() + .map(identifier) + .collect::, _>>()?; + let kwd_patterns = list_items( + &self.req(&inst, "kwd_patterns", "MatchClass")?, + "MatchClass", + "kwd_patterns", + )? + .iter() + .map(|p| self.pattern(p)) + .collect::, _>>()?; + if kwd_attrs.len() != kwd_patterns.len() { + return Err(value_error( + "MatchClass doesn't have the same number of keyword attributes as patterns", + )); + } + past::Pattern::Class { + cls: self.expr(&self.req(&inst, "cls", "MatchClass")?)?, + positionals: list_items( + &self.req(&inst, "patterns", "MatchClass")?, + "MatchClass", + "patterns", + )? + .iter() + .map(|p| self.pattern(p)) + .collect::>()?, + keywords: kwd_attrs.into_iter().zip(kwd_patterns).collect(), + } + } + "MatchOr" => past::Pattern::Or( + list_items( + &self.req(&inst, "patterns", "MatchOr")?, + "MatchOr", + "patterns", + )? + .iter() + .map(|p| self.pattern(p)) + .collect::>()?, + ), + "MatchAs" => { + let sub = match field(&inst, "pattern") { + None | Some(Object::None) => None, + Some(p) => Some(self.pattern(&p)?), + }; + let capture = opt_identifier(field(&inst, "name"))?; + match (sub, capture) { + (None, n) => past::Pattern::Capture(n), + (Some(p), Some(n)) => past::Pattern::As { + pattern: Box::new(p), + name: n, + }, + (Some(_), None) => { + return Err(value_error( + "MatchAs must specify a target name if a pattern is given", + )) + } + } + } + other => { + return Err(type_error(format!( + "expected some sort of pattern, but got <{other} object>" + ))) + } + }) + } + + fn type_params(&mut self, obj: Option) -> Result, RuntimeError> { + let list = match obj { + None | Some(Object::None) => return Ok(Vec::new()), + Some(v) => list_items(&v, "type_params", "type_params")?, + }; + list.iter() + .map(|tp| { + let (inst, name) = node_name(tp, "type_param")?; + let span = self.span_opt(&inst, Span::new(0, 0)); + let pname = identifier(&self.req(&inst, "name", &name)?)?; + let default = self.opt_boxed(field(&inst, "default_value"))?; + let kind = match name.as_str() { + "TypeVar" => past::TypeParamKind::TypeVar { + bound: self.opt_boxed(field(&inst, "bound"))?, + }, + "TypeVarTuple" => past::TypeParamKind::TypeVarTuple, + "ParamSpec" => past::TypeParamKind::ParamSpec, + other => { + return Err(type_error(format!( + "expected some sort of type_param, but got <{other} object>" + ))) + } + }; + Ok(past::TypeParam { + name: pname.clone(), + source_name: pname, + kind, + default, + span, + }) + }) + .collect() + } +} + +// --------------------------------------------------------------------------- +// Operators and constants +// --------------------------------------------------------------------------- + +fn op_name(obj: &Object, category: &str) -> Result { + match obj { + Object::Instance(inst) => Ok(inst.cls().name.clone()), + other => Err(type_error(format!( + "expected some sort of {category}, but got {}", + repr_lite(other) + ))), + } +} + +fn bin_op(obj: &Object) -> Result { + Ok(match op_name(obj, "operator")?.as_str() { + "Add" => past::BinOp::Add, + "Sub" => past::BinOp::Sub, + "Mult" => past::BinOp::Mult, + "MatMult" => past::BinOp::MatMult, + "Div" => past::BinOp::Div, + "Mod" => past::BinOp::Mod, + "Pow" => past::BinOp::Pow, + "LShift" => past::BinOp::LShift, + "RShift" => past::BinOp::RShift, + "BitOr" => past::BinOp::BitOr, + "BitXor" => past::BinOp::BitXor, + "BitAnd" => past::BinOp::BitAnd, + "FloorDiv" => past::BinOp::FloorDiv, + other => { + return Err(type_error(format!( + "expected some sort of operator, but got <{other} object>" + ))) + } + }) +} + +fn bool_op(obj: &Object) -> Result { + Ok(match op_name(obj, "boolop")?.as_str() { + "And" => past::BoolOp::And, + "Or" => past::BoolOp::Or, + other => { + return Err(type_error(format!( + "expected some sort of boolop, but got <{other} object>" + ))) + } + }) +} + +fn unary_op(obj: &Object) -> Result { + Ok(match op_name(obj, "unaryop")?.as_str() { + "Invert" => past::UnaryOp::Invert, + "Not" => past::UnaryOp::Not, + "UAdd" => past::UnaryOp::UAdd, + "USub" => past::UnaryOp::USub, + other => { + return Err(type_error(format!( + "expected some sort of unaryop, but got <{other} object>" + ))) + } + }) +} + +fn cmp_op(obj: &Object) -> Result { + Ok(match op_name(obj, "cmpop")?.as_str() { + "Eq" => past::CmpOp::Eq, + "NotEq" => past::CmpOp::NotEq, + "Lt" => past::CmpOp::Lt, + "LtE" => past::CmpOp::LtE, + "Gt" => past::CmpOp::Gt, + "GtE" => past::CmpOp::GtE, + "Is" => past::CmpOp::Is, + "IsNot" => past::CmpOp::IsNot, + "In" => past::CmpOp::In, + "NotIn" => past::CmpOp::NotIn, + other => { + return Err(type_error(format!( + "expected some sort of cmpop, but got <{other} object>" + ))) + } + }) +} + +fn alias(obj: &Object) -> Result { + let (inst, _name) = node_name(obj, "alias")?; + let name = field(&inst, "name") + .ok_or_else(|| missing_field("name", "alias")) + .and_then(|v| identifier(&v))?; + Ok(past::Alias { + name, + asname: opt_identifier(field(&inst, "asname"))?, + }) +} + +/// Lower a `Constant.value` runtime object back into a parser +/// constant. Mirrors CPython's compiler validation: only genuinely +/// constant types are admitted. +fn constant_value(obj: &Object) -> Result { + Ok(match obj { + Object::None => past::Constant::None, + Object::Bool(b) => past::Constant::Bool(*b), + Object::Int(i) => past::Constant::Int(*i), + Object::Long(b) => past::Constant::BigInt(b.to_string()), + Object::Float(f) => past::Constant::Float(*f), + Object::Complex(c) => past::Constant::Complex(c.real, c.imag), + Object::Str(s) => past::Constant::Str(s.to_string()), + Object::WStr(cps) => past::Constant::WStr(cps.to_vec()), + Object::Bytes(b) => past::Constant::Bytes(b.to_vec()), + Object::Tuple(items) => { + past::Constant::Tuple(items.iter().map(constant_value).collect::>()?) + } + other => { + if crate::vm_singletons::is_ellipsis(other) { + past::Constant::Ellipsis + } else { + return Err(value_error(format!( + "got an invalid type in Constant: {}", + other.type_name() + ))); + } + } + }) +} diff --git a/crates/weavepy-vm/src/stdlib/ast_mod.rs b/crates/weavepy-vm/src/stdlib/ast_mod.rs index 9e5a140..cb9cc56 100644 --- a/crates/weavepy-vm/src/stdlib/ast_mod.rs +++ b/crates/weavepy-vm/src/stdlib/ast_mod.rs @@ -54,6 +54,20 @@ pub fn build(_cache: &ModuleCache) -> Rc { DictKey(Object::from_static("parse")), Object::Builtin(Rc::new(bf)), ); + // `compile()` control flags (CPython `_ast` exposes these; + // `ast.py` re-exports them) — RFC 0052. + use weavepy_compiler::flags as cf; + for (name, value) in [ + ("PyCF_ONLY_AST", cf::PYCF_ONLY_AST), + ("PyCF_TYPE_COMMENTS", cf::PYCF_TYPE_COMMENTS), + ("PyCF_ALLOW_TOP_LEVEL_AWAIT", cf::PYCF_ALLOW_TOP_LEVEL_AWAIT), + ("PyCF_OPTIMIZED_AST", cf::PYCF_OPTIMIZED_AST), + ] { + d.insert( + DictKey(Object::from_static(name)), + Object::Int(i64::from(value)), + ); + } } Rc::new(PyModule { name: "_ast".to_owned(), @@ -211,7 +225,7 @@ impl Builder<'_> { body, decorator_list, returns, - .. + type_params, } => node( "FunctionDef", vec![ @@ -224,7 +238,7 @@ impl Builder<'_> { returns.as_deref().map_or(Object::None, |r| self.expr(r)), ), ("type_comment", Object::None), - ("type_params", Object::new_list(vec![])), + ("type_params", self.type_params(type_params)), ], sp, self.lm, @@ -235,7 +249,7 @@ impl Builder<'_> { body, decorator_list, returns, - .. + type_params, } => node( "AsyncFunctionDef", vec![ @@ -248,7 +262,7 @@ impl Builder<'_> { returns.as_deref().map_or(Object::None, |r| self.expr(r)), ), ("type_comment", Object::None), - ("type_params", Object::new_list(vec![])), + ("type_params", self.type_params(type_params)), ], sp, self.lm, @@ -259,7 +273,7 @@ impl Builder<'_> { keywords, body, decorator_list, - .. + type_params, } => node( "ClassDef", vec![ @@ -268,7 +282,30 @@ impl Builder<'_> { ("keywords", list_of(keywords, |k| self.keyword(k))), ("body", list_of(body, |x| self.stmt(x))), ("decorator_list", list_of(decorator_list, |x| self.expr(x))), - ("type_params", Object::new_list(vec![])), + ("type_params", self.type_params(type_params)), + ], + sp, + self.lm, + ), + S::TypeAlias { + name, + name_span, + type_params, + value, + } => node( + "TypeAlias", + vec![ + ( + "name", + node( + "Name", + vec![("id", ident(name)), ("ctx", singleton("Store"))], + *name_span, + self.lm, + ), + ), + ("type_params", self.type_params(type_params)), + ("value", self.expr(value)), ], sp, self.lm, @@ -746,6 +783,34 @@ impl Builder<'_> { ) } + /// PEP 695 type-parameter list → `[ast.TypeVar | ast.TypeVarTuple | + /// ast.ParamSpec, …]` (with PEP 696 `default_value`). + fn type_params(&self, tps: &[past::TypeParam]) -> Object { + list_of(tps, |tp| { + let default_value = tp.default.as_deref().map_or(Object::None, |d| self.expr(d)); + let fields = match &tp.kind { + past::TypeParamKind::TypeVar { bound } => vec![ + ("name", ident(&tp.source_name)), + ( + "bound", + bound.as_deref().map_or(Object::None, |b| self.expr(b)), + ), + ("default_value", default_value), + ], + past::TypeParamKind::TypeVarTuple | past::TypeParamKind::ParamSpec => vec![ + ("name", ident(&tp.source_name)), + ("default_value", default_value), + ], + }; + let ty = match &tp.kind { + past::TypeParamKind::TypeVar { .. } => "TypeVar", + past::TypeParamKind::TypeVarTuple => "TypeVarTuple", + past::TypeParamKind::ParamSpec => "ParamSpec", + }; + node(ty, fields, tp.span, self.lm) + }) + } + fn comprehension(&self, c: &past::Comprehension) -> Object { node_noloc( "comprehension", diff --git a/crates/weavepy-vm/src/stdlib/marshal_mod.rs b/crates/weavepy-vm/src/stdlib/marshal_mod.rs index 3bb3772..660259c 100644 --- a/crates/weavepy-vm/src/stdlib/marshal_mod.rs +++ b/crates/weavepy-vm/src/stdlib/marshal_mod.rs @@ -425,7 +425,9 @@ fn code_flags(co: &CodeObject) -> u32 { if co.is_async_generator { f |= CO_ASYNC_GENERATOR; } - f + // Persist active `__future__` bits so an unmarshalled code object + // still reports them on `co_flags` (RFC 0052). + f | co.future_flags } /// Pack a `BigInt` into CPython's marshal digit form: a signed count of @@ -761,6 +763,7 @@ impl<'a> MarshalReader<'a> { is_coroutine: flags & CO_COROUTINE != 0, is_async_generator: flags & CO_ASYNC_GENERATOR != 0, is_iterable_coroutine: flags & CO_ITERABLE_COROUTINE != 0, + future_flags: flags & weavepy_compiler::flags::PYCF_MASK, cp_cache: cpython_code::CpCache::default(), }; Ok(Object::Code(Rc::new(co))) diff --git a/crates/weavepy-vm/src/stdlib/mod.rs b/crates/weavepy-vm/src/stdlib/mod.rs index 89c28c9..582e724 100644 --- a/crates/weavepy-vm/src/stdlib/mod.rs +++ b/crates/weavepy-vm/src/stdlib/mod.rs @@ -15,6 +15,7 @@ use crate::import::{FrozenSource, ModuleCache}; +pub mod ast_convert; pub mod ast_mod; pub mod binascii_mod; pub mod bisect_accel; @@ -63,6 +64,7 @@ pub mod tempfile_mod; pub mod testinternalcapi_mod; pub mod thread; pub mod time; +pub mod tokenize_mod; pub mod tracemalloc_real; pub mod ucd; pub mod unicodedata_mod; @@ -161,6 +163,9 @@ pub fn register_all(cache: &ModuleCache) { cache.register_builtin("_ast", ast_mod::build); // RFC 0033 — native symbol-table core behind the frozen `symtable` module. cache.register_builtin("_symtable", symtable_mod::build); + // RFC 0052 — native lexer core behind the frozen `_tokenize` module + // (the CPython 3.13 `Parser/lexer` port `tokenize.py` drives). + cache.register_builtin("_tokenize_core", tokenize_mod::build); cache.register_builtin("_gzip", gzip_mod::build); cache.register_builtin("_bz2", bz2_mod::build); cache.register_builtin("_lzma", lzma_mod::build); @@ -258,11 +263,9 @@ fn frozen_sources() -> &'static [FrozenSource] { // A `static`, not a promoted local: the table is far past clippy's // stack-array budget (`large_stack_arrays`). static SOURCES: &[FrozenSource] = &[ - FrozenSource { - name: "builtins", - source: include_str!("python/builtins.py"), - is_package: false, - }, + // `builtins` is *not* frozen source: the module is created + // eagerly in `Interpreter::default()` sharing the interpreter's + // ambient builtins dict (RFC 0052 WS5 — patchable builtins). // RFC 0046 (wave 5): `ctypes`. The verbatim CPython `ctypes` package // runs over our frozen `_ctypes` reimplementation (CPython's real // `_ctypes` is a core-built C extension linking `_PyRuntime`, so it @@ -2485,6 +2488,13 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/tokenize.py"), is_package: false, }, + // RFC 0052 — `TokenizerIter` shim over the native + // `_tokenize_core` lexer port (CPython's `_tokenize` C module). + FrozenSource { + name: "_tokenize", + source: include_str!("python/_tokenize.py"), + is_package: false, + }, FrozenSource { name: "sysconfig", source: include_str!("python/sysconfig.py"), diff --git a/crates/weavepy-vm/src/stdlib/python/_tokenize.py b/crates/weavepy-vm/src/stdlib/python/_tokenize.py new file mode 100644 index 0000000..d170024 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_tokenize.py @@ -0,0 +1,87 @@ +"""_tokenize — WeavePy's port of CPython's C accelerator (RFC 0052). + +CPython's ``_tokenize.TokenizerIter`` drives the readline flavour of the +pegen tokenizer lazily. WeavePy's native lexer port lives in the +``_tokenize_core`` builtin, which tokenizes a *slurped* list of source +lines in one call; this shim keeps the readline dispatch (str/bytes type +checks, per-line decoding with ``errors="replace"``) in Python, then +yields the precomputed 5-tuples and re-raises any tokenization error +exactly where CPython would — after the tokens that precede it. +""" + +import _tokenize_core + +__all__ = ["TokenizerIter"] + + +class TokenizerIter: + """Iterator of raw token 5-tuples over a readline callable. + + Mirrors ``_tokenize.TokenizerIter(readline, *, extra_tokens, + encoding='utf-8')``: with an *encoding*, ``readline()`` must return + bytes (decoded per line, like ``tok_readline_string``); without one + it must return str. ``StopIteration`` or an empty line signals EOF. + """ + + def __init__(self, readline, /, *, extra_tokens, encoding=None): + self._readline = readline + self._encoding = encoding + self._extra_tokens = bool(extra_tokens) + self._tokens = None + self._index = 0 + self._error = None + + def __iter__(self): + return self + + def _tokenize(self): + lines = [] + readline = self._readline + encoding = self._encoding + while True: + try: + line = readline() + except StopIteration: + break + if encoding is not None: + if not isinstance(line, bytes): + raise TypeError("readline() returned a non-bytes object") + line = line.decode(encoding, "replace") + else: + if not isinstance(line, str): + raise TypeError("readline() returned a non-string object") + if not line: + break + lines.append(line) + self._tokens, self._error = _tokenize_core.tokens( + lines, self._extra_tokens + ) + + def __next__(self): + if self._tokens is None: + self._tokenize() + if self._index < len(self._tokens): + tok = self._tokens[self._index] + self._index += 1 + return tok + if self._error is not None: + kind, msg, lineno, offset, text, end_lineno, end_offset = self._error + self._error = None + if kind == "indent": + exc_type = IndentationError + elif kind == "tab": + exc_type = TabError + else: + exc_type = SyntaxError + if text is None: + # The bare-location E_EOF flavour + # (PyErr_SyntaxLocationObject): no source text attached. + exc = exc_type(msg) + exc.filename = "" + exc.lineno = lineno + exc.offset = offset + raise exc + raise exc_type( + msg, ("", lineno, offset, text, end_lineno, end_offset) + ) + raise StopIteration("EOF") diff --git a/crates/weavepy-vm/src/stdlib/python/ast.py b/crates/weavepy-vm/src/stdlib/python/ast.py index efcc629..e7cdafd 100644 --- a/crates/weavepy-vm/src/stdlib/python/ast.py +++ b/crates/weavepy-vm/src/stdlib/python/ast.py @@ -16,6 +16,13 @@ from enum import IntEnum, auto from contextlib import contextmanager, nullcontext +# `compile()` control flags (CPython exposes these on `_ast`; values +# from Include/cpython/compile.h). +PyCF_ONLY_AST = 0x0400 +PyCF_TYPE_COMMENTS = 0x1000 +PyCF_ALLOW_TOP_LEVEL_AWAIT = 0x2000 +PyCF_OPTIMIZED_AST = 0x8000 | PyCF_ONLY_AST + # --------------------------------------------------------------------------- # Base node @@ -611,21 +618,27 @@ def _fix_contexts(tree): return tree +def _from_spec(spec): + """Build a node tree from an `_ast` spec (used by the native + `compile(..., PyCF_ONLY_AST)` path — RFC 0052).""" + return _fix_contexts(_build(spec)) + + def parse(source, filename="", mode="exec", type_comments=False, feature_version=None, optimize=-1): """Parse source into a CPython-shaped AST (RFC 0033).""" if isinstance(source, (bytes, bytearray)): source = bytes(source).decode("utf-8") + if type_comments: + # Full PEP 484 type-comment harvesting is not implemented; we do + # enforce pegen's `invalid_parameters` rule that a bare `*` + # parameter must not carry a type comment. + for line in source.splitlines(): + code, _sep, comment = line.partition("#") + if _sep and comment.lstrip().startswith("type:") and code.strip() in ("*", "*,"): + raise SyntaxError("bare * has associated type comment") spec = _ast.parse(source, filename, mode) - tree = _fix_contexts(_build(spec)) - # Remember the original text so `compile(tree, ...)` can recompile it - # (WeavePy compiles from source; an unmodified `ast.parse` round-trip - # is by far the common case). - try: - tree._weavepy_source = source - except Exception: - pass - return tree + return _fix_contexts(_build(spec)) # --------------------------------------------------------------------------- @@ -2036,3 +2049,123 @@ def visit_MatchOr(self, node): def unparse(ast_obj): unparser = _Unparser() return unparser.visit(ast_obj) + + +# ---- PyCF_OPTIMIZED_AST (RFC 0052) ---- +# +# CPython folds constants on the AST (Python/ast_opt.c) when +# PyCF_OPTIMIZED_AST is passed to compile(). This is the pure-Python +# analogue covering the same value-level folds: binary/unary operations +# over constants and all-constant Load tuples. + +_FOLD_BINOP = { + "Add": lambda a, b: a + b, + "Sub": lambda a, b: a - b, + "Mult": lambda a, b: a * b, + "Div": lambda a, b: a / b, + "FloorDiv": lambda a, b: a // b, + "Mod": lambda a, b: a % b, + "Pow": lambda a, b: a ** b, + "LShift": lambda a, b: a << b, + "RShift": lambda a, b: a >> b, + "BitOr": lambda a, b: a | b, + "BitXor": lambda a, b: a ^ b, + "BitAnd": lambda a, b: a & b, +} + +_FOLD_UNARYOP = { + "Invert": lambda v: ~v, + "Not": lambda v: not v, + "UAdd": lambda v: +v, + "USub": lambda v: -v, +} + + +def _fold_result_ok(v): + # Mirror ast_opt.c's "don't grow the code object" guards: cap folded + # int/str/bytes sizes; allow the other constant-able types as-is. + if isinstance(v, int): + return v.bit_length() <= 256 + if isinstance(v, (str, bytes)): + return len(v) <= 4096 + if isinstance(v, tuple): + return len(v) <= 256 + return v is None or isinstance(v, (bool, float, complex, frozenset)) + + +def _fold_args_ok(op_name, a, b): + # Pre-guards so folding can't be tricked into huge computation + # (10 ** 10**6, 1 << 10**6, 'x' * 10**6 …). + if op_name in ("Pow", "LShift"): + return isinstance(b, (int, bool)) and abs(b) <= 512 or isinstance(b, float) + if op_name == "Mult": + if isinstance(a, (str, bytes, tuple)) and isinstance(b, int): + return len(a) * max(b, 0) <= 4096 + if isinstance(b, (str, bytes, tuple)) and isinstance(a, int): + return len(b) * max(a, 0) <= 4096 + return True + + +class _ConstantFolder(NodeTransformer): + def visit_BinOp(self, node): + self.generic_visit(node) + left, right = node.left, node.right + if type(left) is Constant and type(right) is Constant: + func = _FOLD_BINOP.get(type(node.op).__name__) + if func is not None and _fold_args_ok( + type(node.op).__name__, left.value, right.value): + try: + value = func(left.value, right.value) + except Exception: + return node + if _fold_result_ok(value): + return copy_location(Constant(value), node) + return node + + def visit_UnaryOp(self, node): + self.generic_visit(node) + operand = node.operand + if type(operand) is Constant: + func = _FOLD_UNARYOP.get(type(node.op).__name__) + if func is not None: + try: + value = func(operand.value) + except Exception: + return node + if _fold_result_ok(value): + return copy_location(Constant(value), node) + return node + + def visit_Tuple(self, node): + self.generic_visit(node) + if isinstance(node.ctx, Load) and all( + type(e) is Constant for e in node.elts): + value = tuple(e.value for e in node.elts) + if _fold_result_ok(value): + return copy_location(Constant(value), node) + return node + + +def _fold_constants(tree): + """Apply PyCF_OPTIMIZED_AST constant folding in place; returns the tree.""" + return _ConstantFolder().visit(tree) + + +def _export_node_classes_to_native(): + # In CPython the node classes are *defined* in the C `_ast` module and + # `ast.py` star-imports them. WeavePy defines them here instead, so we + # push them back onto `_ast` — code that does `import _ast` after `ast` + # (e.g. `type(tree) == _ast.Module` in test_compile) sees the same + # class objects. + for _name, _obj in list(globals().items()): + if isinstance(_obj, type) and issubclass(_obj, AST): + setattr(_ast, _name, _obj) + _ast.AST = AST + _ast.PyCF_ONLY_AST = PyCF_ONLY_AST + _ast.PyCF_TYPE_COMMENTS = PyCF_TYPE_COMMENTS + _ast.PyCF_ALLOW_TOP_LEVEL_AWAIT = PyCF_ALLOW_TOP_LEVEL_AWAIT + _ast.PyCF_OPTIMIZED_AST = PyCF_OPTIMIZED_AST + + +_export_node_classes_to_native() +del _export_node_classes_to_native diff --git a/crates/weavepy-vm/src/stdlib/python/builtins.py b/crates/weavepy-vm/src/stdlib/python/builtins.py deleted file mode 100644 index ff0797d..0000000 --- a/crates/weavepy-vm/src/stdlib/python/builtins.py +++ /dev/null @@ -1,35 +0,0 @@ -"""CPython-compatible `builtins` module. - -In CPython this is the dict that backs every frame's `__builtins__`. -WeavePy registers the same dict ambiently at frame creation; the -import here just re-exposes those names as attributes of a real -module object so that callers like `pickle._find_class("builtins", -"len")` work. - -We can't `from import *`, so we walk the running frame's -builtins dictionary at import time and stamp each entry as a module -attribute. The set of names is intentionally lazy: anything not -already in `__builtins__` simply won't appear here. -""" - -import sys as _sys - - -def _populate(): - # Reach into the frame whose `f_builtins` we want to copy. Using - # `_getframe(0)` returns this module's frame; `f_builtins` is the - # dict the VM populated with `default_builtins()`. - frame = _sys._getframe(0) - src = frame.f_builtins - mod = _sys.modules[__name__] - for k, v in src.items(): - try: - setattr(mod, k, v) - except Exception: - # Some names (e.g. `__builtins__` itself) can't be - # round-tripped; skip them silently. - pass - - -_populate() -del _populate diff --git a/crates/weavepy-vm/src/stdlib/python/codeop.py b/crates/weavepy-vm/src/stdlib/python/codeop.py index 0a85ce5..6d185fb 100644 --- a/crates/weavepy-vm/src/stdlib/python/codeop.py +++ b/crates/weavepy-vm/src/stdlib/python/codeop.py @@ -31,6 +31,11 @@ def _is_incomplete(exc, source): if "was never closed" in msg: return True if "unexpected EOF" in msg or "incomplete input" in msg: + # A backslash-newline at EOF already consumed the continuation: + # CPython treats "a = 9+ \\\n" as a hard error, while a bare + # trailing backslash can still be continued. + if source.endswith("\\\n") or source.endswith("\\\r\n"): + return False return True # A pending suite ("if 1:" …) is incomplete only when nothing but # blank lines follows the suite *header* — a dedented statement diff --git a/crates/weavepy-vm/src/stdlib/python/symtable.py b/crates/weavepy-vm/src/stdlib/python/symtable.py index 07c96ac..cb2ce1b 100644 --- a/crates/weavepy-vm/src/stdlib/python/symtable.py +++ b/crates/weavepy-vm/src/stdlib/python/symtable.py @@ -37,6 +37,11 @@ def __init__(self, d): self.varnames = d["varnames"] self.children = [_RawTable(c) for c in d["children"]] + def __repr__(self): + # CPython's PySTEntry repr: "". + return "".format( + self.name, self.id, self.lineno) + def symtable(code, filename, compile_type): """ Return the toplevel *SymbolTable* for the source code. @@ -275,13 +280,29 @@ def is_local_symbol(ident): for st in self._table.children: # pick the function-like symbols that are local identifiers if is_local_symbol(st.name): - if st.type == _symtable.TYPE_FUNCTION: - # generators are of type TYPE_FUNCTION with a ".0" - # parameter as a first parameter (which makes them - # distinguishable from a function named 'genexpr') - if st.name == 'genexpr' and '.0' in st.varnames: - continue - d[st.name] = 1 + match st.type: + case _symtable.TYPE_FUNCTION: + # generators are of type TYPE_FUNCTION with a ".0" + # parameter as a first parameter (which makes them + # distinguishable from a function named 'genexpr') + if st.name == 'genexpr' and '.0' in st.varnames: + continue + d[st.name] = 1 + case _symtable.TYPE_TYPE_PARAMETERS: + # Get the function-def block in the annotation + # scope 'st' with the same identifier, if any. + scope_name = st.name + for c in st.children: + if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION: + # A generic generator of type TYPE_FUNCTION + # cannot be a direct child of 'st' (but it + # can be a descendant), e.g.: + # + # class A: + # type genexpr[genexpr] = (x for x in []) + assert scope_name != 'genexpr' or '.0' not in c.varnames + d[scope_name] = 1 + break self.__methods = tuple(d) return self.__methods diff --git a/crates/weavepy-vm/src/stdlib/python/tokenize.py b/crates/weavepy-vm/src/stdlib/python/tokenize.py index 69284ca..7ca552c 100644 --- a/crates/weavepy-vm/src/stdlib/python/tokenize.py +++ b/crates/weavepy-vm/src/stdlib/python/tokenize.py @@ -1,35 +1,48 @@ -"""Tokenization help for Python programs — WeavePy port. - -CPython 3.13's ``tokenize`` is a thin shell over the C ``_tokenize`` -accelerator (``TokenizerIter``). WeavePy has no C extensions, so — -following the same approach as the RFC 0035 pure-Python ``re`` engine — -this module is the *classic* pure-Python tokenizer (the reference -implementation that shipped in ``Lib/tokenize.py`` through CPython 3.11) -exposing the 3.13 public surface: ``TokenInfo``, ``tokenize``, -``generate_tokens``, ``detect_encoding``, ``untokenize``, ``open``, -``TokenError`` and the ``token`` constants. - -Known fidelity gap vs. 3.13: f-strings are produced as single ``STRING`` -tokens (the pre-PEP-701 tokenization) rather than ``FSTRING_START`` / -``FSTRING_MIDDLE`` / ``FSTRING_END`` triples. +"""Tokenization help for Python programs. + +tokenize(readline) is a generator that breaks a stream of bytes into +Python tokens. It decodes the bytes according to PEP-0263 for +determining source file encoding. + +It accepts a readline-like method which is called repeatedly to get the +next line of input (or b"" for EOF). It generates 5-tuples with these +members: + + the token type (see token.py) + the token (a string) + the starting (row, column) indices of the token (a 2-tuple of ints) + the ending (row, column) indices of the token (a 2-tuple of ints) + the original line (string) + +It is designed to match the working of the Python tokenizer exactly, except +that it produces COMMENT tokens for comments and gives type OP for all +operators. Additionally, all token lists start with an ENCODING token +which tells you which encoding was used to decode the bytes stream. """ +__author__ = 'Ka-Ping Yee ' +__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' + 'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' + 'Michael Foord') from builtins import open as _builtin_open +from codecs import lookup, BOM_UTF8 import collections +import functools +from io import TextIOWrapper import itertools as _itertools import re import sys - from token import * from token import EXACT_TOKEN_TYPES -import token - -__all__ = token.__all__ + ["tokenize", "generate_tokens", "detect_encoding", - "untokenize", "TokenInfo", "open", "TokenError"] +import _tokenize cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) -blank_re = re.compile(rb'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) +blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) +import token +__all__ = token.__all__ + ["tokenize", "generate_tokens", "detect_encoding", + "untokenize", "TokenInfo", "open", "TokenError"] +del token class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): def __repr__(self): @@ -44,7 +57,6 @@ def exact_type(self): else: return self.type - def group(*choices): return '(' + '|'.join(choices) + ')' def any(*choices): return group(*choices) + '*' def maybe(*choices): return group(*choices) + '?' @@ -69,10 +81,9 @@ def maybe(*choices): return group(*choices) + '?' Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) - # Return the empty string, plus all of the valid string prefixes. def _all_string_prefixes(): - # The valid string prefixes. Only contain the lowercase versions, + # The valid string prefixes. Only contain the lower case versions, # and don't contain any permutations (include 'fr', but not # 'rf'). The various permutations will be generated. _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] @@ -86,6 +97,12 @@ def _all_string_prefixes(): result.add(''.join(u)) return result +@functools.lru_cache +def _compile(expr): + return re.compile(expr, re.UNICODE) + +# Note that since _all_string_prefixes includes the empty string, +# StringPrefix can be the empty string (making it optional). StringPrefix = group(*_all_string_prefixes()) # Tail end of ' string. @@ -102,8 +119,8 @@ def _all_string_prefixes(): StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') # Sorting in reverse order puts the long operators before their prefixes. -# Otherwise if = came before ==, == would get interpreted as -# two instances of =. +# Otherwise if = came before ==, == would get recognized as two instances +# of =. Special = group(*map(re.escape, sorted(EXACT_TOKEN_TYPES, reverse=True))) Funny = group(r'\r?\n', Special) @@ -119,7 +136,7 @@ def _all_string_prefixes(): PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) # For a given string prefix plus quotes, endpats maps it to a regex -# to match the remainder of that string. _prefixes can be empty, for +# to match the remainder of that string. _prefix can be empty, for # a normal single or triple quoted string (with no prefix). endpats = {} for _prefix in _all_string_prefixes(): @@ -142,24 +159,17 @@ def _all_string_prefixes(): tabsize = 8 -# Compile the workhorse patterns once at import; the tokenizer loop -# matches one of these per token. -_pseudo_prog = re.compile(PseudoToken) -_endprogs = {pat: re.compile(pat) for pat in set(endpats.values())} - - class TokenError(Exception): pass -class StopTokenizing(Exception): pass - - class Untokenizer: def __init__(self): self.tokens = [] self.prev_row = 1 self.prev_col = 0 + self.prev_type = None + self.prev_line = "" self.encoding = None def add_whitespace(self, start): @@ -167,14 +177,51 @@ def add_whitespace(self, start): if row < self.prev_row or row == self.prev_row and col < self.prev_col: raise ValueError("start ({},{}) precedes previous end ({},{})" .format(row, col, self.prev_row, self.prev_col)) - row_offset = row - self.prev_row - if row_offset: - self.tokens.append("\\\n" * row_offset) - self.prev_col = 0 + self.add_backslash_continuation(start) col_offset = col - self.prev_col if col_offset: self.tokens.append(" " * col_offset) + def add_backslash_continuation(self, start): + """Add backslash continuation characters if the row has increased + without encountering a newline token. + + This also inserts the correct amount of whitespace before the backslash. + """ + row = start[0] + row_offset = row - self.prev_row + if row_offset == 0: + return + + newline = '\r\n' if self.prev_line.endswith('\r\n') else '\n' + line = self.prev_line.rstrip('\\\r\n') + ws = ''.join(_itertools.takewhile(str.isspace, reversed(line))) + self.tokens.append(ws + f"\\{newline}" * row_offset) + self.prev_col = 0 + + def escape_brackets(self, token): + characters = [] + consume_until_next_bracket = False + for character in token: + if character == "}": + if consume_until_next_bracket: + consume_until_next_bracket = False + else: + characters.append(character) + if character == "{": + n_backslashes = sum( + 1 for char in _itertools.takewhile( + "\\".__eq__, + characters[-2::-1] + ) + ) + if n_backslashes % 2 == 0 or characters[-1] != "N": + characters.append(character) + else: + consume_until_next_bracket = True + characters.append(character) + return "".join(characters) + def untokenize(self, iterable): it = iter(iterable) indents = [] @@ -200,17 +247,26 @@ def untokenize(self, iterable): startline = True elif startline and indents: indent = indents[-1] - start_row, start_col = start - if start_col >= len(indent): + if start[1] >= len(indent): self.tokens.append(indent) self.prev_col = len(indent) startline = False + elif tok_type == FSTRING_MIDDLE: + if '{' in token or '}' in token: + token = self.escape_brackets(token) + last_line = token.splitlines()[-1] + end_line, end_col = end + extra_chars = last_line.count("{{") + last_line.count("}}") + end = (end_line, end_col + extra_chars) + self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end if tok_type in (NEWLINE, NL): self.prev_row += 1 self.prev_col = 0 + self.prev_type = tok_type + self.prev_line = line return "".join(self.tokens) def compat(self, token, iterable): @@ -218,6 +274,7 @@ def compat(self, token, iterable): toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False + in_fstring = 0 for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] @@ -236,6 +293,10 @@ def compat(self, token, iterable): else: prevstring = False + if toknum == FSTRING_START: + in_fstring += 1 + elif toknum == FSTRING_END: + in_fstring -= 1 if toknum == INDENT: indents.append(tokval) continue @@ -247,7 +308,19 @@ def compat(self, token, iterable): elif startline and indents: toks_append(indents[-1]) startline = False + elif toknum == FSTRING_MIDDLE: + tokval = self.escape_brackets(tokval) + + # Insert a space between two consecutive brackets if we are in an f-string + if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring: + tokval = ' ' + tokval + + # Insert a space between two consecutive f-strings + if toknum in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): + self.tokens.append(" ") + toks_append(tokval) + self.prev_type = toknum def untokenize(iterable): @@ -282,7 +355,6 @@ def _get_normal_name(orig_enc): return "iso-8859-1" return orig_enc - def detect_encoding(readline): """ The detect_encoding() function is used to detect the encoding that should @@ -292,10 +364,10 @@ def detect_encoding(readline): It will call readline a maximum of twice, and return the encoding used (as a string) and a list of any lines (left as bytes) it has read in. - It detects the encoding from the presence of a UTF-8 BOM or an encoding - cookie as specified in PEP-0263. If both a BOM and a cookie are present, + It detects the encoding from the presence of a utf-8 bom or an encoding + cookie as specified in pep-0263. If both a bom and a cookie are present, but disagree, a SyntaxError will be raised. If the encoding cookie is an - invalid charset, raise a SyntaxError. Note that if a UTF-8 BOM is found, + invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, 'utf-8-sig' is returned. If no encoding is specified, then the default of 'utf-8' will be returned. @@ -307,7 +379,6 @@ def detect_encoding(readline): bom_found = False encoding = None default = 'utf-8' - def read_or_stop(): try: return readline() @@ -331,15 +402,14 @@ def find_cookie(line): return None encoding = _get_normal_name(match.group(1)) try: - import codecs - codecs.lookup(encoding) + codec = lookup(encoding) except LookupError: # This behaviour mimics the Python interpreter if filename is None: msg = "unknown encoding: " + encoding else: msg = "unknown encoding for {!r}: {}".format(filename, - encoding) + encoding) raise SyntaxError(msg) if bom_found: @@ -354,7 +424,7 @@ def find_cookie(line): return encoding first = read_or_stop() - if first.startswith(b'\xef\xbb\xbf'): + if first.startswith(BOM_UTF8): bom_found = True first = first[3:] default = 'utf-8-sig' @@ -382,20 +452,16 @@ def open(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ - # CPython wraps the original binary buffer in a TextIOWrapper; - # WeavePy's `io` has no public TextIOWrapper-over-buffer, so we - # detect on a first binary pass and reopen in text mode with the - # detected encoding — same observable contract. buffer = _builtin_open(filename, 'rb') try: encoding, lines = detect_encoding(buffer.readline) - finally: + buffer.seek(0) + text = TextIOWrapper(buffer, encoding, line_buffering=True) + text.mode = 'r' + return text + except: buffer.close() - if encoding == 'utf-8-sig': - encoding = 'utf-8' - text = _builtin_open(filename, 'r', encoding=encoding) - return text - + raise def tokenize(readline): """ @@ -417,192 +483,13 @@ def tokenize(readline): which tells you which encoding was used to decode the bytes stream. """ encoding, consumed = detect_encoding(readline) - empty = _itertools.repeat(b"") - rl_gen = _itertools.chain(consumed, iter(readline, b""), empty) - return _tokenize(rl_gen.__next__, encoding) - - -def _tokenize(readline, encoding): - lnum = parenlev = continued = 0 - numchars = '0123456789' - contstr, needcont = '', 0 - contline = None - indents = [0] - + rl_gen = _itertools.chain(consumed, iter(readline, b"")) if encoding is not None: if encoding == "utf-8-sig": # BOM will already have been stripped. encoding = "utf-8" yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') - last_line = b'' - line = b'' - while True: # loop over lines in stream - try: - # We capture the value of the line variable here because - # readline uses the empty string `''` to signal end of input, - # unlike the `None` default of `line` we use here. - last_line = line - line = readline() - except StopIteration: - line = b'' - - if encoding is not None and isinstance(line, bytes): - line = line.decode(encoding) - lnum += 1 - pos, max = 0, len(line) - - if contstr: # continued string - if not line: - raise TokenError("EOF in multi-line string", strstart) - endmatch = endprog.match(line) - if endmatch: - pos = end = endmatch.end(0) - yield TokenInfo(STRING, contstr + line[:end], - strstart, (lnum, end), contline + line) - contstr, needcont = '', 0 - contline = None - elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': - yield TokenInfo(ERRORTOKEN, contstr + line, - strstart, (lnum, len(line)), contline) - contstr = '' - contline = None - continue - else: - contstr = contstr + line - contline = contline + line - continue - - elif parenlev == 0 and not continued: # new statement - if not line: break - column = 0 - while pos < max: # measure leading whitespace - if line[pos] == ' ': - column += 1 - elif line[pos] == '\t': - column = (column//tabsize + 1)*tabsize - elif line[pos] == '\f': - column = 0 - else: - break - pos += 1 - if pos == max: - break - - if line[pos] in '#\r\n': # skip comments or blank lines - if line[pos] == '#': - comment_token = line[pos:].rstrip('\r\n') - yield TokenInfo(COMMENT, comment_token, - (lnum, pos), (lnum, pos + len(comment_token)), line) - pos += len(comment_token) - - yield TokenInfo(NL, line[pos:], - (lnum, pos), (lnum, len(line)), line) - continue - - if column > indents[-1]: # count indents or dedents - indents.append(column) - yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) - while column < indents[-1]: - if column not in indents: - raise IndentationError( - "unindent does not match any outer indentation level", - ("", lnum, pos, line)) - indents = indents[:-1] - - yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) - - else: # continued statement - if not line: - raise TokenError("EOF in multi-line statement", (lnum, 0)) - continued = 0 - - while pos < max: - pseudomatch = _pseudo_prog.match(line, pos) - if pseudomatch: # scan for tokens - start, end = pseudomatch.span(1) - spos, epos, pos = (lnum, start), (lnum, end), end - if start == end: - continue - token, initial = line[start:end], line[start] - - if (initial in numchars or - (initial == '.' and token != '.' and token != '...')): # ordinary number - yield TokenInfo(NUMBER, token, spos, epos, line) - elif initial in '\r\n': - if parenlev > 0: - yield TokenInfo(NL, token, spos, epos, line) - else: - yield TokenInfo(NEWLINE, token, spos, epos, line) - - elif initial == '#': - assert not token.endswith("\n") - yield TokenInfo(COMMENT, token, spos, epos, line) - - elif token in triple_quoted: - endprog = _endprogs[endpats[token]] - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - pos = endmatch.end(0) - token = line[start:pos] - yield TokenInfo(STRING, token, spos, (lnum, pos), line) - else: - strstart = (lnum, start) # multiple lines - contstr = line[start:] - contline = line - break - - # Check up to the first 3 chars of the token to see if - # they're in the single_quoted set. If so, they start - # a string. - # We're using the first 3, because we're looking for - # "rb'" (for example) at the start of the token. If - # we switch to longer prefixes, this needs to be - # adjusted. - # Note that initial == token[:1]. - # Also note that single quote checking must come after - # triple quote checking (above). - elif (initial in single_quoted or - token[:2] in single_quoted or - token[:3] in single_quoted): - if token[-1] == '\n': # continued string - strstart = (lnum, start) - # Again, we're computing the matching regex here - # by using the first few chars of the token to - # find the corresponding endpat. - endpat = (endpats.get(initial) or - endpats.get(token[1]) or - endpats.get(token[2])) - endprog = _endprogs[endpat] - contstr, needcont = line[start:], 1 - contline = line - break - else: # ordinary string - yield TokenInfo(STRING, token, spos, epos, line) - - elif initial.isidentifier(): # ordinary name - yield TokenInfo(NAME, token, spos, epos, line) - elif initial == '\\': # continued stmt - continued = 1 - else: - if initial in '([{': - parenlev += 1 - elif initial in ')]}': - parenlev -= 1 - yield TokenInfo(OP, token, spos, epos, line) - else: - yield TokenInfo(ERRORTOKEN, line[pos], - (lnum, pos), (lnum, pos+1), line) - pos += 1 - - # Add an implicit NEWLINE if the input doesn't end in one - if last_line and last_line[-1] not in '\r\n' and \ - not last_line.strip().startswith('#'): - yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), - (lnum - 1, len(last_line) + 1), '') - for indent in indents[1:]: # pop remaining indent levels - yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') - yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') - + yield from _generate_tokens_from_c_tokenizer(rl_gen.__next__, encoding, extra_tokens=True) def generate_tokens(readline): """Tokenize a source reading Python code as unicode strings. @@ -610,4 +497,96 @@ def generate_tokens(readline): This has the same API as tokenize(), except that it expects the *readline* callable to return str objects instead of bytes. """ - return _tokenize(readline, None) + return _generate_tokens_from_c_tokenizer(readline, extra_tokens=True) + +def main(): + import argparse + + # Helper error handling routines + def perror(message): + sys.stderr.write(message) + sys.stderr.write('\n') + + def error(message, filename=None, location=None): + if location: + args = (filename,) + location + (message,) + perror("%s:%d:%d: error: %s" % args) + elif filename: + perror("%s: error: %s" % (filename, message)) + else: + perror("error: %s" % message) + sys.exit(1) + + # Parse the arguments and options + parser = argparse.ArgumentParser(prog='python -m tokenize') + parser.add_argument(dest='filename', nargs='?', + metavar='filename.py', + help='the file to tokenize; defaults to stdin') + parser.add_argument('-e', '--exact', dest='exact', action='store_true', + help='display token names using the exact type') + args = parser.parse_args() + + try: + # Tokenize the input + if args.filename: + filename = args.filename + with _builtin_open(filename, 'rb') as f: + tokens = list(tokenize(f.readline)) + else: + filename = "" + tokens = _generate_tokens_from_c_tokenizer( + sys.stdin.readline, extra_tokens=True) + + + # Output the tokenization + for token in tokens: + token_type = token.type + if args.exact: + token_type = token.exact_type + token_range = "%d,%d-%d,%d:" % (token.start + token.end) + print("%-20s%-15s%-15r" % + (token_range, tok_name[token_type], token.string)) + except IndentationError as err: + line, column = err.args[1][1:3] + error(err.args[0], filename, (line, column)) + except TokenError as err: + line, column = err.args[1] + error(err.args[0], filename, (line, column)) + except SyntaxError as err: + error(err, filename) + except OSError as err: + error(err) + except KeyboardInterrupt: + print("interrupted\n") + except Exception as err: + perror("unexpected error: %s" % err) + raise + +def _transform_msg(msg): + """Transform error messages from the C tokenizer into the Python tokenize + + The C tokenizer is more picky than the Python one, so we need to massage + the error messages a bit for backwards compatibility. + """ + if "unterminated triple-quoted string literal" in msg: + return "EOF in multi-line string" + return msg + +def _generate_tokens_from_c_tokenizer(source, encoding=None, extra_tokens=False): + """Tokenize a source reading Python code as unicode strings using the internal C tokenizer""" + if encoding is None: + it = _tokenize.TokenizerIter(source, extra_tokens=extra_tokens) + else: + it = _tokenize.TokenizerIter(source, encoding=encoding, extra_tokens=extra_tokens) + try: + for info in it: + yield TokenInfo._make(info) + except SyntaxError as e: + if type(e) != SyntaxError: + raise e from None + msg = _transform_msg(e.msg) + raise TokenError(msg, (e.lineno, e.offset)) from None + + +if __name__ == "__main__": + main() diff --git a/crates/weavepy-vm/src/stdlib/symtable_mod.rs b/crates/weavepy-vm/src/stdlib/symtable_mod.rs index 70791fc..66982cc 100644 --- a/crates/weavepy-vm/src/stdlib/symtable_mod.rs +++ b/crates/weavepy-vm/src/stdlib/symtable_mod.rs @@ -29,7 +29,7 @@ use std::collections::{HashMap, HashSet}; use weavepy_lexer::token::Span; use weavepy_parser::ast as past; -use crate::error::{value_error, RuntimeError}; +use crate::error::{type_error, value_error, RuntimeError}; use crate::import::ModuleCache; use crate::object::{BuiltinFn, DictData, DictKey, Object, PyModule}; @@ -42,6 +42,9 @@ const USE: i64 = 16; const DEF_FREE_CLASS: i64 = 64; const DEF_IMPORT: i64 = 128; const DEF_ANNOT: i64 = 256; +/// CPython `DEF_TYPE_PARAM` (`2<<9`): the name is a PEP 695 type +/// parameter. +const DEF_TYPE_PARAM: i64 = 2 << 9; const DEF_BOUND: i64 = DEF_LOCAL | DEF_PARAM | DEF_IMPORT; // 134 const SCOPE_OFF: i64 = 12; @@ -59,23 +62,46 @@ const CELL: i64 = 5; const TYPE_FUNCTION: i64 = 0; const TYPE_CLASS: i64 = 1; const TYPE_MODULE: i64 = 2; +const TYPE_ANNOTATION: i64 = 3; +const TYPE_TYPE_ALIAS: i64 = 4; +const TYPE_TYPE_PARAMETERS: i64 = 5; +const TYPE_TYPE_VARIABLE: i64 = 6; #[derive(Clone, Copy, PartialEq, Eq)] enum BlockType { Function, Class, Module, + /// PEP 695 `type X = …` value scope (CPython `TypeAliasBlock`). + TypeAlias, + /// PEP 695 hidden `[T, …]` scope wrapping a generic + /// function/class/alias (CPython `TypeParametersBlock`). + TypeParameters, + /// PEP 695 scope for one type parameter's bound/constraints/ + /// default expression (CPython `TypeVariableBlock`). + TypeVariable, } impl BlockType { + /// CPython `_PyST_IsFunctionLike`: the PEP 695 annotation scopes + /// resolve names like function scopes do. fn is_function_like(self) -> bool { - matches!(self, BlockType::Function) + matches!( + self, + BlockType::Function + | BlockType::TypeAlias + | BlockType::TypeParameters + | BlockType::TypeVariable + ) } fn cpython(self) -> i64 { match self { BlockType::Function => TYPE_FUNCTION, BlockType::Class => TYPE_CLASS, BlockType::Module => TYPE_MODULE, + BlockType::TypeAlias => TYPE_TYPE_ALIAS, + BlockType::TypeParameters => TYPE_TYPE_PARAMETERS, + BlockType::TypeVariable => TYPE_TYPE_VARIABLE, } } } @@ -85,6 +111,9 @@ struct Block { name: String, lineno: i64, nested: bool, + /// CPython `ste_can_see_class_scope`: a PEP 695 annotation scope + /// immediately inside a class body closes over `__classdict__`. + can_see_class_scope: bool, /// name → accumulated flag word (def bits during phase 1; the scope /// is OR'd into the high bits during phase 2). symbols: IndexMap, @@ -126,12 +155,10 @@ pub fn build(_cache: &ModuleCache) -> Rc { ("TYPE_FUNCTION", TYPE_FUNCTION), ("TYPE_CLASS", TYPE_CLASS), ("TYPE_MODULE", TYPE_MODULE), - // Type-parameter / type-alias blocks (PEP 695) aren't produced - // by WeavePy yet, but the wrapper imports the type tags. - ("TYPE_ANNOTATION", 3), - ("TYPE_TYPE_ALIAS", 4), - ("TYPE_TYPE_PARAMETERS", 5), - ("TYPE_TYPE_VARIABLE", 6), + ("TYPE_ANNOTATION", TYPE_ANNOTATION), + ("TYPE_TYPE_ALIAS", TYPE_TYPE_ALIAS), + ("TYPE_TYPE_PARAMETERS", TYPE_TYPE_PARAMETERS), + ("TYPE_TYPE_VARIABLE", TYPE_TYPE_VARIABLE), ]; for (k, v) in consts { d.insert(DictKey(Object::from_str(*k)), Object::Int(*v)); @@ -155,14 +182,59 @@ pub fn build(_cache: &ModuleCache) -> Rc { } /// `_symtable.symtable(source, filename, compile_type)` → raw block tree. +/// +/// Argument conversion mirrors CPython's `_symtablemodule.c` clinic +/// order: `filename` goes through `PyUnicode_FSDecoder` (str/bytes), +/// then `compile_type` must be a `str` naming a compile mode. Parse +/// errors and symtable-build-time errors surface as `SyntaxError`s +/// carrying `filename`/`lineno`/`offset`/`text` like CPython's. pub fn symtable(args: &[Object]) -> Result { - let source = match args.first() { + // `filename` — CPython's FSDecoder: str, bytes, or os.PathLike. + // (PathLike needs a VM re-entry for `__fspath__`; str/bytes covers + // every real caller, and everything else is the same TypeError.) + let filename = match args.get(1) { Some(Object::Str(s)) => s.to_string(), Some(Object::Bytes(b)) => String::from_utf8_lossy(b).into_owned(), - _ => return Err(value_error("symtable() requires a str or bytes source")), + Some(other) => { + return Err(type_error(format!( + "symtable() argument 'filename' must be str, bytes or os.PathLike, not {}", + other.type_name() + ))) + } + None => return Err(type_error("symtable expected 3 arguments")), + }; + let compile_type = match args.get(2) { + Some(Object::Str(s)) => s.to_string(), + Some(other) => { + return Err(type_error(format!( + "symtable() argument 'compile_type' must be str, not {}", + other.type_name() + ))) + } + None => return Err(type_error("symtable expected 3 arguments")), + }; + if !matches!(compile_type.as_str(), "exec" | "eval" | "single") { + return Err(value_error( + "symtable() arg 3 must be 'exec' or 'eval' or 'single'", + )); + } + let source = match args.first() { + Some(Object::Str(s)) => s.to_string(), + // Bytes decode per PEP 263 (BOM + `# -*- coding: … -*-`). + Some(Object::Bytes(b)) => crate::decode_compile_source_bytes(b, &filename)?, + _ => { + return Err(type_error( + "symtable() argument 'source' must be str or bytes", + )) + } }; let module = weavepy_parser::parse_module(&source) - .map_err(|e| value_error(format!("invalid syntax: {e}")))?; + .map_err(|e| crate::parse_error_to_syntax_error(&e, &source, &filename))?; + // CPython's symtable build raises `SyntaxError` for directive + // conflicts (`global` vs. parameter, …) — run the compiler's + // validation pass (no codegen) to surface the same diagnostics. + weavepy_compiler::validate_module_only(&module, &source) + .map_err(|e| crate::compile_error_to_syntax_error(&e, &source, &filename))?; let mut b = Builder::new(&source); let root = b.run(&module); @@ -230,6 +302,7 @@ impl Builder { name: name.to_owned(), lineno, nested, + can_see_class_scope: false, symbols: IndexMap::new(), varnames: Vec::new(), children: Vec::new(), @@ -289,31 +362,104 @@ impl Builder { } } - /// Visit parameter/return annotations and defaults in the *enclosing* - /// scope (CPython evaluates them where the `def`/`lambda` appears). - fn visit_defaults_and_annotations(&mut self, args: &past::Arguments, annotations: bool) { + /// Visit parameter defaults in the *enclosing* scope (CPython + /// evaluates them where the `def`/`lambda` appears). + fn visit_defaults(&mut self, args: &past::Arguments) { for d in &args.defaults { self.visit_expr(d); } for d in args.kw_defaults.iter().flatten() { self.visit_expr(d); } - if annotations { - let all = args - .posonlyargs - .iter() - .chain(&args.args) - .chain(args.vararg.iter()) - .chain(&args.kwonlyargs) - .chain(args.kwarg.iter()); - for a in all { - if let Some(ann) = &a.annotation { - self.visit_expr(ann); - } + } + + /// Visit parameter and return annotations. For a generic `def` + /// the caller enters the hidden type-parameters block first, so + /// these resolve in that annotation scope (CPython + /// `symtable_visit_annotations`). + fn visit_annotations(&mut self, args: &past::Arguments, returns: Option<&past::Expr>) { + let all = args + .posonlyargs + .iter() + .chain(&args.args) + .chain(args.vararg.iter()) + .chain(&args.kwonlyargs) + .chain(args.kwarg.iter()); + for a in all { + if let Some(ann) = &a.annotation { + self.visit_expr(ann); + } + } + if let Some(r) = returns { + self.visit_expr(r); + } + } + + /// CPython `symtable_enter_type_param_block`: open the hidden + /// `TypeParametersBlock` wrapping a generic `def`/`class`/`type` + /// statement. + fn enter_type_param_block( + &mut self, + name: &str, + lineno: i64, + is_class_def: bool, + has_defaults: bool, + has_kwdefaults: bool, + ) { + let parent_is_class = self.arena[self.cur()].ty == BlockType::Class; + self.enter(BlockType::TypeParameters, name, lineno); + if parent_is_class { + let cur = self.cur(); + self.arena[cur].can_see_class_scope = true; + self.add_def("__classdict__", USE); + } + if is_class_def { + // "Set" when the type-params tuple is created, "used" when + // the bases are built; `.generic_base` powers the implicit + // `Generic[…]` base. + self.add_def(".type_params", DEF_LOCAL); + self.add_def(".type_params", USE); + self.add_def(".generic_base", DEF_LOCAL); + self.add_def(".generic_base", USE); + } + if has_defaults { + self.add_def(".defaults", DEF_PARAM); + } + if has_kwdefaults { + self.add_def(".kwdefaults", DEF_PARAM); + } + } + + /// CPython `symtable_visit_type_param`: bind each parameter in + /// the current (type-parameters) block; bounds/constraints and + /// PEP 696 defaults each evaluate in their own + /// `TypeVariableBlock`. + fn visit_type_params(&mut self, type_params: &[past::TypeParam]) { + for tp in type_params { + self.add_def(&tp.name, DEF_TYPE_PARAM | DEF_LOCAL); + if let past::TypeParamKind::TypeVar { bound: Some(b) } = &tp.kind { + self.visit_type_var_block(&tp.name, b); + } + if let Some(d) = &tp.default { + self.visit_type_var_block(&tp.name, d); } } } + /// One `TypeVariableBlock` holding a type parameter's bound, + /// constraints, or default expression. + fn visit_type_var_block(&mut self, name: &str, e: &past::Expr) { + let can_see = self.arena[self.cur()].can_see_class_scope; + self.enter(BlockType::TypeVariable, name, self.lineno(e.span)); + if can_see { + let cur = self.cur(); + self.arena[cur].can_see_class_scope = true; + self.add_def("__classdict__", USE); + } + self.visit_expr(e); + self.exit(); + } + fn visit_stmt(&mut self, s: &past::Stmt) { use past::StmtKind as S; let lineno = self.lineno(s.span); @@ -324,7 +470,7 @@ impl Builder { body, decorator_list, returns, - .. + type_params, } | S::AsyncFunctionDef { name, @@ -332,22 +478,32 @@ impl Builder { body, decorator_list, returns, - .. + type_params, } => { self.add_def(name, DEF_LOCAL); - self.visit_defaults_and_annotations(args, true); - if let Some(r) = returns { - self.visit_expr(r); - } + self.visit_defaults(args); for d in decorator_list { self.visit_expr(d); } + let generic = !type_params.is_empty(); + if generic { + let has_defaults = !args.defaults.is_empty(); + let has_kwdefaults = args.kw_defaults.iter().any(Option::is_some); + self.enter_type_param_block(name, lineno, false, has_defaults, has_kwdefaults); + self.visit_type_params(type_params); + } + // Annotations resolve inside the hidden type-parameters + // block when the `def` is generic. + self.visit_annotations(args, returns.as_deref()); self.enter(BlockType::Function, name, lineno); self.add_params(args); for st in body { self.visit_stmt(st); } self.exit(); + if generic { + self.exit(); + } } S::ClassDef { name, @@ -355,23 +511,63 @@ impl Builder { keywords, body, decorator_list, - .. + type_params, } => { self.add_def(name, DEF_LOCAL); + for d in decorator_list { + self.visit_expr(d); + } + let generic = !type_params.is_empty(); + if generic { + self.enter_type_param_block(name, lineno, true, false, false); + self.visit_type_params(type_params); + } + // A generic class's bases/keywords evaluate inside the + // hidden type-parameters block. for b in bases { self.visit_expr(b); } for k in keywords { self.visit_expr(&k.value); } - for d in decorator_list { - self.visit_expr(d); - } self.enter(BlockType::Class, name, lineno); + if generic { + self.add_def("__type_params__", DEF_LOCAL); + self.add_def(".type_params", USE); + } for st in body { self.visit_stmt(st); } self.exit(); + if generic { + self.exit(); + } + } + S::TypeAlias { + name, + type_params, + value, + .. + } => { + // The alias name is a Store in the enclosing scope. + self.add_def(name, DEF_LOCAL); + let is_in_class = self.arena[self.cur()].ty == BlockType::Class; + let generic = !type_params.is_empty(); + if generic { + self.enter_type_param_block(name, lineno, false, false, false); + self.visit_type_params(type_params); + } + self.enter(BlockType::TypeAlias, name, lineno); + if is_in_class { + let cur = self.cur(); + self.arena[cur].can_see_class_scope = true; + self.add_def("__classdict__", USE); + } + self.visit_expr(value); + self.exit(); + if generic { + self.exit(); + } } S::Return(v) => { if let Some(e) = v { @@ -607,7 +803,7 @@ impl Builder { } } E::Lambda { args, body } | E::TypeParamFn { args, body } => { - self.visit_defaults_and_annotations(args, false); + self.visit_defaults(args); self.enter(BlockType::Function, "lambda", self.lineno(span)); self.add_params(args); self.visit_expr(body); @@ -834,7 +1030,10 @@ impl Analyzer<'_> { newbound.extend(bound.iter().cloned()); newglobal.extend(global.iter().cloned()); } else { + // Classes provide implicit cells for `__class__` and + // `__classdict__` to nested scopes. newbound.insert("__class__".to_owned()); + newbound.insert("__classdict__".to_owned()); } let children = self.arena[idx].children.clone(); @@ -855,12 +1054,13 @@ impl Analyzer<'_> { newfree.remove("__classdict__"); } + let classflag = is_class || self.arena[idx].can_see_class_scope; update_symbols( &mut self.arena[idx].symbols, &scopes, bound, &newfree, - is_class, + classflag, ); free.extend(newfree); diff --git a/crates/weavepy-vm/src/stdlib/tokenize_mod.rs b/crates/weavepy-vm/src/stdlib/tokenize_mod.rs new file mode 100644 index 0000000..82d7324 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/tokenize_mod.rs @@ -0,0 +1,1908 @@ +//! `_tokenize_core` — the native tokenizer behind the frozen +//! `_tokenize` module (RFC 0052). +//! +//! CPython 3.13's `tokenize.py` is a thin wrapper over the C +//! `_tokenize.TokenizerIter`, which drives the *readline* flavour of the +//! pegen tokenizer (`Parser/lexer/lexer.c` + +//! `Parser/tokenizer/readline_tokenizer.c`) and post-processes each raw +//! token into the classic 5-tuple (`Python/Python-tokenize.c`). This +//! module is a line-by-line port of that C code: the same buffer +//! discipline (the buffer resets between tokens and *accumulates* across +//! lines while a multi-line token — triple-quoted string, open bracket, +//! f-string — is in flight), the same indentation/dedent stack with the +//! alternate-tabsize consistency check, the same PEP 701 f-string mode +//! stack producing `FSTRING_START`/`FSTRING_MIDDLE`/`FSTRING_END` +//! triples, and the same `E_*` done-code → exception mapping +//! (`SyntaxError` / `IndentationError` / `TabError` with CPython's exact +//! messages and locations). +//! +//! The single entry point is `tokens(lines, extra_tokens)` where `lines` +//! is the list of source lines a `readline` callable produced (the +//! frozen `_tokenize.TokenizerIter` slurps them — WeavePy builtins keep +//! the readline dispatch in Python). It returns `(token_tuples, error)` +//! where `error` is `None` or a structured descriptor the shim re-raises +//! faithfully. + +use crate::sync::Rc; +use crate::sync::RefCell; + +use crate::error::{type_error, RuntimeError}; +use crate::import::ModuleCache; +use crate::object::{BuiltinFn, DictData, DictKey, Object, PyModule}; + +// ---- token types (Grammar/Tokens, pycore_token.h) ---- +const ENDMARKER: i32 = 0; +const NAME: i32 = 1; +const NUMBER: i32 = 2; +const STRING: i32 = 3; +const NEWLINE: i32 = 4; +const INDENT: i32 = 5; +const DEDENT: i32 = 6; +const LPAR: i32 = 7; +const RPAR: i32 = 8; +const LSQB: i32 = 9; +const RSQB: i32 = 10; +const COLON: i32 = 11; +const COMMA: i32 = 12; +const SEMI: i32 = 13; +const PLUS: i32 = 14; +const MINUS: i32 = 15; +const STAR: i32 = 16; +const SLASH: i32 = 17; +const VBAR: i32 = 18; +const AMPER: i32 = 19; +const LESS: i32 = 20; +const GREATER: i32 = 21; +const EQUAL: i32 = 22; +const DOT: i32 = 23; +const PERCENT: i32 = 24; +const LBRACE: i32 = 25; +const RBRACE: i32 = 26; +const EQEQUAL: i32 = 27; +const NOTEQUAL: i32 = 28; +const LESSEQUAL: i32 = 29; +const GREATEREQUAL: i32 = 30; +const TILDE: i32 = 31; +const CIRCUMFLEX: i32 = 32; +const LEFTSHIFT: i32 = 33; +const RIGHTSHIFT: i32 = 34; +const DOUBLESTAR: i32 = 35; +const PLUSEQUAL: i32 = 36; +const MINEQUAL: i32 = 37; +const STAREQUAL: i32 = 38; +const SLASHEQUAL: i32 = 39; +const PERCENTEQUAL: i32 = 40; +const AMPEREQUAL: i32 = 41; +const VBAREQUAL: i32 = 42; +const CIRCUMFLEXEQUAL: i32 = 43; +const LEFTSHIFTEQUAL: i32 = 44; +const RIGHTSHIFTEQUAL: i32 = 45; +const DOUBLESTAREQUAL: i32 = 46; +const DOUBLESLASH: i32 = 47; +const DOUBLESLASHEQUAL: i32 = 48; +const AT: i32 = 49; +const ATEQUAL: i32 = 50; +const RARROW: i32 = 51; +const ELLIPSIS: i32 = 52; +const COLONEQUAL: i32 = 53; +const EXCLAMATION: i32 = 54; +const OP: i32 = 55; +const FSTRING_START: i32 = 59; +const FSTRING_MIDDLE: i32 = 60; +const FSTRING_END: i32 = 61; +const COMMENT: i32 = 62; +const NL: i32 = 63; +const ERRORTOKEN: i32 = 64; + +const EOF: i32 = -1; + +const MAXINDENT: usize = 100; // Max indentation level +const MAXLEVEL: usize = 200; // Max parentheses level +const MAXFSTRINGLEVEL: usize = 150; // Max f-string nesting level +const MAX_EXPR_NESTING: i32 = 3; +const TABSIZE: i64 = 8; +const ALTTABSIZE: i64 = 1; + +fn is_potential_identifier_start(c: i32) -> bool { + (c >= 'a' as i32 && c <= 'z' as i32) + || (c >= 'A' as i32 && c <= 'Z' as i32) + || c == '_' as i32 + || c >= 128 +} + +fn is_potential_identifier_char(c: i32) -> bool { + (c >= 'a' as i32 && c <= 'z' as i32) + || (c >= 'A' as i32 && c <= 'Z' as i32) + || (c >= '0' as i32 && c <= '9' as i32) + || c == '_' as i32 + || c >= 128 +} + +fn is_digit(c: i32) -> bool { + c >= '0' as i32 && c <= '9' as i32 +} + +fn is_xdigit(c: i32) -> bool { + is_digit(c) || (c >= 'a' as i32 && c <= 'f' as i32) || (c >= 'A' as i32 && c <= 'F' as i32) +} + +/// `_PyToken_OneChar` (Parser/token.c). +fn one_char_token(c: i32) -> i32 { + match c as u8 as char { + '!' => EXCLAMATION, + '%' => PERCENT, + '&' => AMPER, + '(' => LPAR, + ')' => RPAR, + '*' => STAR, + '+' => PLUS, + ',' => COMMA, + '-' => MINUS, + '.' => DOT, + '/' => SLASH, + ':' => COLON, + ';' => SEMI, + '<' => LESS, + '=' => EQUAL, + '>' => GREATER, + '@' => AT, + '[' => LSQB, + ']' => RSQB, + '^' => CIRCUMFLEX, + '{' => LBRACE, + '|' => VBAR, + '}' => RBRACE, + '~' => TILDE, + _ => OP, + } +} + +/// `_PyToken_TwoChars`. +fn two_chars_token(c1: i32, c2: i32) -> i32 { + match (c1 as u8 as char, c2 as u8 as char) { + ('!', '=') => NOTEQUAL, + ('%', '=') => PERCENTEQUAL, + ('&', '=') => AMPEREQUAL, + ('*', '*') => DOUBLESTAR, + ('*', '=') => STAREQUAL, + ('+', '=') => PLUSEQUAL, + ('-', '=') => MINEQUAL, + ('-', '>') => RARROW, + ('/', '/') => DOUBLESLASH, + ('/', '=') => SLASHEQUAL, + (':', '=') => COLONEQUAL, + ('<', '<') => LEFTSHIFT, + ('<', '=') => LESSEQUAL, + ('<', '>') => NOTEQUAL, + ('=', '=') => EQEQUAL, + ('>', '=') => GREATEREQUAL, + ('>', '>') => RIGHTSHIFT, + ('@', '=') => ATEQUAL, + ('^', '=') => CIRCUMFLEXEQUAL, + ('|', '=') => VBAREQUAL, + _ => OP, + } +} + +/// `_PyToken_ThreeChars`. +fn three_chars_token(c1: i32, c2: i32, c3: i32) -> i32 { + match (c1 as u8 as char, c2 as u8 as char, c3 as u8 as char) { + ('*', '*', '=') => DOUBLESTAREQUAL, + ('.', '.', '.') => ELLIPSIS, + ('/', '/', '=') => DOUBLESLASHEQUAL, + ('<', '<', '=') => LEFTSHIFTEQUAL, + ('>', '>', '=') => RIGHTSHIFTEQUAL, + _ => OP, + } +} + +// ---- tok_state.done codes we distinguish (errcode.h subset) ---- +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Done { + Ok, + /// E_EOF + Eof, + /// E_ERROR — a pending error descriptor is set. + Error, + /// E_DEDENT + Dedent, + /// E_TABSPACE + TabSpace, + /// E_TOODEEP + TooDeep, + /// E_LINECONT + LineCont, + /// E_EOLS / E_EOFS (string-EOF refinements; reported like Error). + Eols, + Eofs, +} + +/// The exception the shim should raise, mirroring what +/// `Python-tokenize.c` leaves in `PyErr`. +struct PendingError { + /// "syntax" | "indent" | "tab" + kind: &'static str, + msg: String, + lineno: i64, + /// character offset (already converted from bytes) + offset: i64, + /// `None` only for the bare-location E_EOF flavour. + text: Option, + end_lineno: Option, + end_offset: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ModeKind { + Regular, + Fstring, +} + +/// `tokenizer_mode` (Parser/lexer/state.h). The `last_expr_*` metadata +/// buffer is omitted: it only feeds `token->metadata` (the f-string +/// debug-text the *parser* consumes), which the tokenize tuples never +/// expose. +struct Mode { + kind: ModeKind, + curly_bracket_depth: i32, + curly_bracket_expr_start_depth: i32, + quote: i32, + quote_size: i32, + raw: bool, + /// buffer index of the f-string opening prefix (`f_string_start`). + start: usize, + /// buffer index of the line the f-string started on. + multi_line_start: usize, + /// line number the f-string started on. + line_start: i64, + in_format_spec: bool, + debug: bool, +} + +impl Mode { + fn regular_top() -> Self { + // state.c zero-initializes the bottom stack slot, so + // `curly_bracket_expr_start_depth` is 0 (not -1) at top level. + Mode { + kind: ModeKind::Regular, + curly_bracket_depth: 0, + curly_bracket_expr_start_depth: 0, + quote: 0, + quote_size: 0, + raw: false, + start: 0, + multi_line_start: 0, + line_start: 0, + in_format_spec: false, + debug: false, + } + } +} + +/// `struct tok_state` for the readline flavour, with C pointers replaced +/// by indices into `buf`. +struct Tok { + buf: Vec, + /// index of the next unread byte. + cur: usize, + /// end of buffered data. + inp: usize, + /// start of the current token (None between tokens). + start: Option, + done: Done, + /// remaining input lines (the shim's slurped readline output). + lines: Vec, + next_line: usize, + indent: usize, + indstack: [i64; MAXINDENT], + altindstack: [i64; MAXINDENT], + atbol: bool, + pendin: i32, + lineno: i64, + first_lineno: i64, + starting_col_offset: i64, + col_offset: i64, + level: usize, + parenstack: [(u8, i64, i64); MAXLEVEL], + cont_line: bool, + /// buffer index of the current line start. + line_start: usize, + /// buffer index of the first line of a multi-line string token. + multi_line_start: usize, + extra_tokens: bool, + comment_newline: bool, + implicit_newline: bool, + modes: Vec, + /// the C `PyErr` slot: set by `syntaxerror()`. + err: Option, +} + +impl Tok { + fn new(lines: Vec, extra_tokens: bool) -> Self { + Tok { + buf: Vec::new(), + cur: 0, + inp: 0, + start: None, + done: Done::Ok, + lines, + next_line: 0, + indent: 0, + indstack: [0; MAXINDENT], + altindstack: [0; MAXINDENT], + atbol: true, + pendin: 0, + lineno: 0, + first_lineno: 0, + starting_col_offset: -1, + col_offset: -1, + level: 0, + parenstack: [(0, 0, 0); MAXLEVEL], + cont_line: false, + line_start: 0, + multi_line_start: 0, + extra_tokens, + comment_newline: false, + implicit_newline: false, + modes: vec![Mode::regular_top()], + err: None, + } + } + + fn inside_fstring(&self) -> bool { + self.modes.len() > 1 + } + + // ---- error construction (Parser/tokenizer/helpers.c) ---- + + /// `_syntaxerror_range`: full-location SyntaxError. `col_offset` / + /// `end_col_offset` of -1 mean "at tok->cur". + fn syntaxerror_range(&mut self, msg: String, col_offset: i64, end_col_offset: i64) -> i32 { + if self.err.is_some() { + return ERRORTOKEN; + } + let upto_cur = String::from_utf8_lossy( + &self.buf[self.line_start.min(self.cur)..self.cur.min(self.inp).max(self.line_start)], + ) + .into_owned(); + let col = if col_offset == -1 { + upto_cur.chars().count() as i64 + } else { + col_offset + }; + let end_col = if end_col_offset == -1 { + col + } else { + end_col_offset + }; + // strcspn(line_start, "\n"): the full physical line for display. + let ls = self.line_start.min(self.inp); + let line_len = self.buf[ls..self.inp] + .iter() + .position(|&b| b == b'\n') + .unwrap_or(self.inp - ls); + let errtext = if line_len != self.cur.saturating_sub(ls) { + String::from_utf8_lossy(&self.buf[ls..ls + line_len]).into_owned() + } else { + upto_cur + }; + self.err = Some(PendingError { + kind: "syntax", + msg, + lineno: self.lineno, + offset: col, + text: Some(errtext), + end_lineno: Some(self.lineno), + end_offset: Some(end_col), + }); + self.done = Done::Error; + ERRORTOKEN + } + + fn syntaxerror(&mut self, msg: impl Into) -> i32 { + self.syntaxerror_range(msg.into(), -1, -1) + } + + /// `_PyTokenizer_indenterror`. + fn indenterror(&mut self) -> i32 { + self.done = Done::TabSpace; + self.cur = self.inp; + ERRORTOKEN + } + + // ---- character stream (lexer.c tok_nextc / tok_backup) ---- + + fn nextc(&mut self) -> i32 { + loop { + if self.cur != self.inp { + self.col_offset += 1; + let c = self.buf[self.cur]; + self.cur += 1; + return i32::from(c); + } + if self.done != Done::Ok { + return EOF; + } + if !self.underflow() { + self.cur = self.inp; + return EOF; + } + self.line_start = self.cur; + if self.buf[self.line_start..self.inp].contains(&0) { + self.syntaxerror("source code cannot contain null bytes"); + self.cur = self.inp; + return EOF; + } + } + } + + fn backup(&mut self, c: i32) { + if c != EOF { + debug_assert!(self.cur > 0); + self.cur -= 1; + debug_assert_eq!(i32::from(self.buf[self.cur]), c); + self.col_offset -= 1; + } + } + + /// `tok_underflow_readline`: pull the next slurped line into the + /// buffer, resetting it first unless a token (or f-string) is in + /// flight. + fn underflow(&mut self) -> bool { + if self.start.is_none() && !self.inside_fstring() { + self.buf.clear(); + self.cur = 0; + self.inp = 0; + } + let line = match self.lines.get(self.next_line) { + Some(l) => { + self.next_line += 1; + l.clone() + } + None => String::new(), + }; + self.buf.extend_from_slice(line.as_bytes()); + self.inp = self.buf.len(); + // tok_readline_string resets line_start even when nothing was + // read (so EOF doesn't leave it dangling past a buffer reset). + self.line_start = self.cur; + if self.inp == self.cur { + self.done = Done::Eof; + return false; + } + self.implicit_newline = false; + if self.buf[self.inp - 1] != b'\n' { + // Last line does not end in \n, fake one. + self.buf.push(b'\n'); + self.inp += 1; + self.implicit_newline = true; + } + // ADVANCE_LINENO() + self.lineno += 1; + self.col_offset = 0; + true + } + + /// `tok_continuation_line`. + fn continuation_line(&mut self) -> i32 { + let mut c = self.nextc(); + if c == '\r' as i32 { + c = self.nextc(); + } + if c != '\n' as i32 { + self.done = Done::LineCont; + return -1; + } + c = self.nextc(); + if c == EOF { + self.done = Done::Eof; + self.cur = self.inp; + return -1; + } + self.backup(c); + c + } + + // ---- number helpers ---- + + /// `tok_decimal_tail`: 0 signals an error was raised. + fn decimal_tail(&mut self) -> i32 { + loop { + let mut c; + loop { + c = self.nextc(); + if !is_digit(c) { + break; + } + } + if c != '_' as i32 { + return c; + } + c = self.nextc(); + if !is_digit(c) { + self.backup(c); + self.syntaxerror("invalid decimal literal"); + return 0; + } + } + } + + /// `lookahead`: does the identifier-ish suffix `test` (followed by a + /// non-identifier char) come next? Restores the stream either way. + fn lookahead(&mut self, test: &str) -> bool { + let pat = test.as_bytes(); + let mut matched: Vec = Vec::new(); + let res; + loop { + let c = self.nextc(); + if matched.len() == pat.len() { + res = !is_potential_identifier_char(c); + self.backup(c); + break; + } + if c == i32::from(pat[matched.len()]) { + matched.push(c); + continue; + } + self.backup(c); + res = false; + break; + } + for &c in matched.iter().rev() { + self.backup(c); + } + res + } + + /// `verify_end_of_number`. WeavePy departure: the SyntaxWarning for + /// keyword-adjacent literals (`0in x`) is not *emitted* (no warnings + /// machinery here), but the control flow — including the char + /// consumption — matches the warn-succeeded path. + fn verify_end_of_number(&mut self, c: i32, kind: &str) -> bool { + if self.extra_tokens { + return true; + } + let mut r = false; + if c == 'a' as i32 { + r = self.lookahead("nd"); + } else if c == 'e' as i32 { + r = self.lookahead("lse"); + } else if c == 'f' as i32 { + r = self.lookahead("or"); + } else if c == 'i' as i32 { + let c2 = self.nextc(); + if c2 == 'f' as i32 || c2 == 'n' as i32 || c2 == 's' as i32 { + r = true; + } + self.backup(c2); + } else if c == 'o' as i32 { + r = self.lookahead("r"); + } else if c == 'n' as i32 { + r = self.lookahead("ot"); + } + if r { + self.backup(c); + // parser_warn(SyntaxWarning, "invalid %s literal") — warning + // suppressed; on the non-raising path the char is re-consumed. + self.nextc(); + } else if c < 128 && is_potential_identifier_char(c) { + self.backup(c); + self.syntaxerror(format!("invalid {kind} literal")); + return false; + } + true + } + + /// `verify_identifier` — PEP 3131 validation of a non-ASCII name. + fn verify_identifier(&mut self) -> bool { + if self.extra_tokens { + return true; + } + let start = self.start.unwrap_or(self.cur); + let text = match std::str::from_utf8(&self.buf[start..self.cur]) { + Ok(t) => t.to_owned(), + Err(_) => { + // Unreachable with str input; treated as E_DECODE in C. + self.syntaxerror("invalid decode"); + return false; + } + }; + let chars: Vec = text.chars().collect(); + debug_assert!(!chars.is_empty()); + let mut invalid = chars.len(); + for (i, &ch) in chars.iter().enumerate() { + let ok = if i == 0 { + ch == '_' || crate::unicode_case::is_xid_start(ch) + } else { + crate::unicode_case::is_xid_continue(ch) + }; + if !ok { + invalid = i; + break; + } + } + if invalid < chars.len() { + let ch = chars[invalid]; + if invalid + 1 < chars.len() { + // Shift tok->cur to just past the offending character so + // the caret lands on it. + let byte_len: usize = chars[..=invalid].iter().map(|c| c.len_utf8()).sum(); + self.cur = start + byte_len; + } + if crate::object::char_is_printable(ch) { + self.syntaxerror(format!("invalid character '{ch}' (U+{:04X})", ch as u32)); + } else { + self.syntaxerror(format!( + "invalid non-printable character U+{:04X}", + ch as u32 + )); + } + return false; + } + true + } +} + +/// Result of one `tok_get`: the token type plus the `p_start`/`p_end` +/// buffer indices (None ↔ C NULL). +struct RawToken { + ty: i32, + start: Option, + end: Option, +} + +impl Tok { + /// `_PyTokenizer_Get` / `tok_get`. + fn get(&mut self) -> RawToken { + if self.modes.last().unwrap().kind == ModeKind::Regular { + self.get_normal_mode() + } else { + self.get_fstring_mode() + } + } + + fn make(&self, ty: i32, start: Option, end: Option) -> RawToken { + RawToken { ty, start, end } + } + + /// `tok_get_normal_mode`. + fn get_normal_mode(&mut self) -> RawToken { + let mut blankline; + + 'nextline: loop { + self.start = None; + self.starting_col_offset = -1; + blankline = false; + + // Get indentation level. + if self.atbol { + let mut col: i64 = 0; + let mut altcol: i64 = 0; + self.atbol = false; + let mut cont_line_col: i64 = 0; + let mut c; + loop { + c = self.nextc(); + if c == ' ' as i32 { + col += 1; + altcol += 1; + } else if c == '\t' as i32 { + col = (col / TABSIZE + 1) * TABSIZE; + altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; + } else if c == 0x0C { + // Control-L (formfeed): for Emacs users. + col = 0; + altcol = 0; + } else if c == '\\' as i32 { + // Indentation cannot be split over multiple + // physical lines with backslashes: the first + // backslash's column wins. + cont_line_col = if cont_line_col != 0 { + cont_line_col + } else { + col + }; + c = self.continuation_line(); + if c == -1 { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + } else if c == EOF && self.err.is_some() { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } else { + break; + } + } + self.backup(c); + if c == '#' as i32 || c == '\n' as i32 || c == '\r' as i32 { + // Whitespace/comment-only lines don't affect + // indentation (no interactive prompt here). + blankline = true; + } + if !blankline && self.level == 0 { + let col = if cont_line_col != 0 { + cont_line_col + } else { + col + }; + let altcol = if cont_line_col != 0 { + cont_line_col + } else { + altcol + }; + if col == self.indstack[self.indent] { + // No change. + if altcol != self.altindstack[self.indent] { + let t = self.indenterror(); + return self.make(t, self.start, Some(self.cur)); + } + } else if col > self.indstack[self.indent] { + // Indent — always one. + if self.indent + 1 >= MAXINDENT { + self.done = Done::TooDeep; + self.cur = self.inp; + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + if altcol <= self.altindstack[self.indent] { + let t = self.indenterror(); + return self.make(t, self.start, Some(self.cur)); + } + self.pendin += 1; + self.indent += 1; + self.indstack[self.indent] = col; + self.altindstack[self.indent] = altcol; + } else { + // Dedent — any number, must be consistent. + while self.indent > 0 && col < self.indstack[self.indent] { + self.pendin -= 1; + self.indent -= 1; + } + if col != self.indstack[self.indent] { + self.done = Done::Dedent; + self.cur = self.inp; + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + if altcol != self.altindstack[self.indent] { + let t = self.indenterror(); + return self.make(t, self.start, Some(self.cur)); + } + } + } + } + + self.start = Some(self.cur); + self.starting_col_offset = self.col_offset; + + // Return pending indents/dedents. + if self.pendin != 0 { + if self.pendin < 0 { + self.pendin += 1; + let (s, e) = if self.extra_tokens { + (Some(self.cur), Some(self.cur)) + } else { + (None, None) + }; + return self.make(DEDENT, s, e); + } + self.pendin -= 1; + let (s, e) = if self.extra_tokens { + (Some(0), Some(self.cur)) + } else { + (None, None) + }; + return self.make(INDENT, s, e); + } + + // Peek ahead at the next character. + let c = self.nextc(); + self.backup(c); + + 'again: loop { + self.start = None; + // Skip spaces. + let mut c; + loop { + c = self.nextc(); + if !(c == ' ' as i32 || c == '\t' as i32 || c == 0x0C) { + break; + } + } + + // Set start of current token. + self.start = if self.cur == 0 { + None + } else { + Some(self.cur - 1) + }; + self.starting_col_offset = self.col_offset - 1; + + // Skip comment (type comments are not requested here). + if c == '#' as i32 { + while c != EOF && c != '\n' as i32 && c != '\r' as i32 { + c = self.nextc(); + } + if self.extra_tokens { + self.backup(c); // don't eat the newline or EOF + let p = self.start; + self.comment_newline = blankline; + return self.make(COMMENT, p, Some(self.cur)); + } + } + + // Check for EOF and errors now. + if c == EOF { + if self.level > 0 { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + let ty = if self.done == Done::Eof { + ENDMARKER + } else { + ERRORTOKEN + }; + return self.make(ty, self.start, Some(self.cur)); + } + + // Identifier (most frequent token!). + let mut nonascii = false; + if is_potential_identifier_start(c) { + // Process the legal combinations of b"", r"", u"", f"". + let (mut saw_b, mut saw_r, mut saw_u, mut saw_f) = (false, false, false, false); + loop { + if !(saw_b || saw_u || saw_f) && (c == 'b' as i32 || c == 'B' as i32) { + saw_b = true; + } else if !(saw_b || saw_u || saw_r || saw_f) + && (c == 'u' as i32 || c == 'U' as i32) + { + saw_u = true; + } else if !(saw_r || saw_u) && (c == 'r' as i32 || c == 'R' as i32) { + saw_r = true; + } else if !(saw_f || saw_b || saw_u) && (c == 'f' as i32 || c == 'F' as i32) + { + saw_f = true; + } else { + break; + } + c = self.nextc(); + if c == '"' as i32 || c == '\'' as i32 { + if saw_f { + return self.f_string_quote(c); + } + return self.letter_quote(c); + } + } + while is_potential_identifier_char(c) { + if c >= 128 { + nonascii = true; + } + c = self.nextc(); + } + self.backup(c); + if nonascii && !self.verify_identifier() { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + return self.make(NAME, self.start, Some(self.cur)); + } + + if c == '\r' as i32 { + c = self.nextc(); + } + + // Newline. + if c == '\n' as i32 { + self.atbol = true; + if blankline || self.level > 0 { + if self.extra_tokens { + if self.comment_newline { + self.comment_newline = false; + } + return self.make(NL, self.start, Some(self.cur)); + } + continue 'nextline; + } + if self.comment_newline && self.extra_tokens { + self.comment_newline = false; + return self.make(NL, self.start, Some(self.cur)); + } + // Leave '\n' out of the string. + let t = self.make(NEWLINE, self.start, Some(self.cur - 1)); + self.cont_line = false; + return t; + } + + // Period or number starting with period? + if c == '.' as i32 { + c = self.nextc(); + if is_digit(c) { + return self.number_fraction(c); + } else if c == '.' as i32 { + c = self.nextc(); + if c == '.' as i32 { + return self.make(ELLIPSIS, self.start, Some(self.cur)); + } + self.backup(c); + self.backup('.' as i32); + } else { + self.backup(c); + } + return self.make(DOT, self.start, Some(self.cur)); + } + + // Number. + if is_digit(c) { + return self.number(c); + } + + if c == '\'' as i32 || c == '"' as i32 { + return self.letter_quote(c); + } + + // Line continuation. + if c == '\\' as i32 { + if self.continuation_line() == -1 { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + self.cont_line = true; + continue 'again; // Read next line. + } + + // Punctuation inside an f-string expression part. + let is_punctuation = + c == ':' as i32 || c == '}' as i32 || c == '!' as i32 || c == '{' as i32; + if is_punctuation && self.inside_fstring() { + let (expr_start_depth, depth, in_format_spec, debug) = { + let m = self.modes.last().unwrap(); + ( + m.curly_bracket_expr_start_depth, + m.curly_bracket_depth, + m.in_format_spec, + m.debug, + ) + }; + if expr_start_depth >= 0 { + // Runs before `{` increments the depth, so adjust + // to test "at the 0th level". + let cursor = depth - i32::from(c != '{' as i32); + let cursor_in_format_with_debug = cursor == 1 && (debug || in_format_spec); + let cursor_valid = cursor == 0 || cursor_in_format_with_debug; + // (update_fstring_expr / set_fstring_expr only feed + // the parser-side metadata — skipped.) + let _ = cursor_valid; + if c == ':' as i32 && cursor == expr_start_depth { + let m = self.modes.last_mut().unwrap(); + m.kind = ModeKind::Fstring; + m.in_format_spec = true; + return self.make(one_char_token(c), self.start, Some(self.cur)); + } + } + } + + // Check for two-character token. + { + let c2 = self.nextc(); + let tok2 = two_chars_token(c, c2); + if tok2 != OP { + let c3 = self.nextc(); + let tok3 = three_chars_token(c, c2, c3); + let current = if tok3 != OP { + tok3 + } else { + self.backup(c3); + tok2 + }; + return self.make(current, self.start, Some(self.cur)); + } + self.backup(c2); + } + + // Keep track of parentheses nesting level. + if c == '(' as i32 || c == '[' as i32 || c == '{' as i32 { + if self.level >= MAXLEVEL { + let t = self.syntaxerror("too many nested parentheses"); + return self.make(t, self.start, Some(self.cur)); + } + self.parenstack[self.level] = ( + c as u8, + self.lineno, + (self.start.unwrap_or(self.cur) as i64) - (self.line_start as i64), + ); + self.level += 1; + if self.inside_fstring() { + self.modes.last_mut().unwrap().curly_bracket_depth += 1; + } + } else if c == ')' as i32 || c == ']' as i32 || c == '}' as i32 { + if self.inside_fstring() + && self.modes.last().unwrap().curly_bracket_depth == 0 + && c == '}' as i32 + { + let t = self.syntaxerror("f-string: single '}' is not allowed"); + return self.make(t, self.start, Some(self.cur)); + } + if !self.extra_tokens && self.level == 0 { + let t = self.syntaxerror(format!("unmatched '{}'", c as u8 as char)); + return self.make(t, self.start, Some(self.cur)); + } + if self.level > 0 { + self.level -= 1; + let (opening, open_lineno, _opencol) = self.parenstack[self.level]; + let matches = (opening == b'(' && c == ')' as i32) + || (opening == b'[' && c == ']' as i32) + || (opening == b'{' && c == '}' as i32); + if !self.extra_tokens && !matches { + // An f-string expression's `{` closed by some + // other bracket reports as unmatched. + if self.inside_fstring() && opening == b'{' { + let m = self.modes.last().unwrap(); + let previous_bracket = m.curly_bracket_depth - 1; + if previous_bracket == m.curly_bracket_expr_start_depth { + let t = self.syntaxerror(format!( + "f-string: unmatched '{}'", + c as u8 as char + )); + return self.make(t, self.start, Some(self.cur)); + } + } + let t = if open_lineno != self.lineno { + self.syntaxerror(format!( + "closing parenthesis '{}' does not match opening parenthesis '{}' on line {}", + c as u8 as char, opening as char, open_lineno + )) + } else { + self.syntaxerror(format!( + "closing parenthesis '{}' does not match opening parenthesis '{}'", + c as u8 as char, opening as char + )) + }; + return self.make(t, self.start, Some(self.cur)); + } + } + if self.inside_fstring() { + let m = self.modes.last_mut().unwrap(); + m.curly_bracket_depth -= 1; + if m.curly_bracket_depth < 0 { + let t = self + .syntaxerror(format!("f-string: unmatched '{}'", c as u8 as char)); + return self.make(t, self.start, Some(self.cur)); + } + if c == '}' as i32 + && m.curly_bracket_depth == m.curly_bracket_expr_start_depth + { + m.curly_bracket_expr_start_depth -= 1; + m.kind = ModeKind::Fstring; + m.in_format_spec = false; + m.debug = false; + } + } + } + + // ASCII control chars (bytes ≥ 128 never reach here — they + // take the identifier path above). + if !(0x20..0x7F).contains(&c) { + let t = self.syntaxerror(format!("invalid non-printable character U+{c:04X}")); + return self.make(t, self.start, Some(self.cur)); + } + + if c == '=' as i32 && self.inside_fstring() { + let m = self.modes.last_mut().unwrap(); + if m.curly_bracket_depth - m.curly_bracket_expr_start_depth == 1 { + m.debug = true; + } + } + + // Punctuation character. + return self.make(one_char_token(c), self.start, Some(self.cur)); + } + } + } + + /// The `f_string_quote:` label — start of an f-string prefix. + fn f_string_quote(&mut self, c: i32) -> RawToken { + let start = self.start.unwrap(); + let first = (self.buf[start] as char).to_ascii_lowercase(); + if !((first == 'f' || first == 'r') && (c == '\'' as i32 || c == '"' as i32)) { + return self.letter_quote(c); + } + let quote = c; + let mut quote_size = 1; + + self.first_lineno = self.lineno; + self.multi_line_start = self.line_start; + + // Find the quote size and start of string. + let after_quote = self.nextc(); + if after_quote == quote { + let after_after_quote = self.nextc(); + if after_after_quote == quote { + quote_size = 3; + } else { + self.backup(after_after_quote); + self.backup(after_quote); + } + } + if after_quote != quote { + self.backup(after_quote); + } + + if self.modes.len() + 1 >= MAXFSTRINGLEVEL { + let t = self.syntaxerror("too many nested f-strings"); + return self.make(t, self.start, Some(self.cur)); + } + let raw = match first { + 'f' => self.buf[start + 1].eq_ignore_ascii_case(&b'r'), + 'r' => true, + _ => unreachable!(), + }; + self.modes.push(Mode { + kind: ModeKind::Fstring, + curly_bracket_depth: 0, + curly_bracket_expr_start_depth: -1, + quote, + quote_size, + raw, + start, + multi_line_start: self.line_start, + line_start: self.lineno, + in_format_spec: false, + debug: false, + }); + self.make(FSTRING_START, self.start, Some(self.cur)) + } + + /// The `letter_quote:` label — an ordinary string literal. + fn letter_quote(&mut self, c: i32) -> RawToken { + if !(c == '\'' as i32 || c == '"' as i32) { + // Fall through in C reaches the operator logic; in practice + // the callers only jump here on a quote. + return self.make(one_char_token(c), self.start, Some(self.cur)); + } + let quote = c; + let mut quote_size = 1; + let mut end_quote_size = 0; + let mut has_escaped_quote = false; + + self.first_lineno = self.lineno; + self.multi_line_start = self.line_start; + + // Find the quote size and start of string. + let mut c = self.nextc(); + if c == quote { + c = self.nextc(); + if c == quote { + quote_size = 3; + } else { + end_quote_size = 1; // empty string found + } + } + if c != quote { + self.backup(c); + } + + // Get rest of string. + while end_quote_size != quote_size { + c = self.nextc(); + if self.done == Done::Error { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + if c == EOF || (quote_size == 1 && c == '\n' as i32) { + // Shift the location to the start of the string and + // report from the initial quote character. + self.cur = self.start.unwrap() + 1; + self.line_start = self.multi_line_start; + let start_line = self.lineno; + self.lineno = self.first_lineno; + + if self.inside_fstring() { + let m = self.modes.last().unwrap(); + if m.quote == quote && m.quote_size == quote_size { + let t = self.syntaxerror("f-string: expecting '}'"); + return self.make(t, self.start, Some(self.cur)); + } + } + + if quote_size == 3 { + self.syntaxerror(format!( + "unterminated triple-quoted string literal (detected at line {start_line})" + )); + if c != '\n' as i32 { + self.done = Done::Eofs; + } + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + if has_escaped_quote { + self.syntaxerror(format!( + "unterminated string literal (detected at line {start_line}); perhaps you escaped the end quote?" + )); + } else { + self.syntaxerror(format!( + "unterminated string literal (detected at line {start_line})" + )); + } + if c != '\n' as i32 { + self.done = Done::Eols; + } + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + if c == quote { + end_quote_size += 1; + } else { + end_quote_size = 0; + if c == '\\' as i32 { + c = self.nextc(); // skip escaped char + if c == quote { + has_escaped_quote = true; + } + if c == '\r' as i32 { + self.nextc(); + } + } + } + } + + self.make(STRING, self.start, Some(self.cur)) + } + + /// `number()`: entered from the main loop with the first digit. + fn number(&mut self, c0: i32) -> RawToken { + let mut c = c0; + if c == '0' as i32 { + // Hex, octal or binary — maybe. + c = self.nextc(); + if c == 'x' as i32 || c == 'X' as i32 { + // Hex. + c = self.nextc(); + loop { + if c == '_' as i32 { + c = self.nextc(); + } + if !is_xdigit(c) { + self.backup(c); + let t = self.syntaxerror("invalid hexadecimal literal"); + return self.make(t, self.start, Some(self.cur)); + } + loop { + c = self.nextc(); + if !is_xdigit(c) { + break; + } + } + if c != '_' as i32 { + break; + } + } + if !self.verify_end_of_number(c, "hexadecimal") { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + } else if c == 'o' as i32 || c == 'O' as i32 { + // Octal. + c = self.nextc(); + loop { + if c == '_' as i32 { + c = self.nextc(); + } + if !('0' as i32..'8' as i32).contains(&c) { + if is_digit(c) { + let t = self.syntaxerror(format!( + "invalid digit '{}' in octal literal", + c as u8 as char + )); + return self.make(t, self.start, Some(self.cur)); + } + self.backup(c); + let t = self.syntaxerror("invalid octal literal"); + return self.make(t, self.start, Some(self.cur)); + } + loop { + c = self.nextc(); + if !('0' as i32..'8' as i32).contains(&c) { + break; + } + } + if c != '_' as i32 { + break; + } + } + if is_digit(c) { + let t = self.syntaxerror(format!( + "invalid digit '{}' in octal literal", + c as u8 as char + )); + return self.make(t, self.start, Some(self.cur)); + } + if !self.verify_end_of_number(c, "octal") { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + } else if c == 'b' as i32 || c == 'B' as i32 { + // Binary. + c = self.nextc(); + loop { + if c == '_' as i32 { + c = self.nextc(); + } + if c != '0' as i32 && c != '1' as i32 { + if is_digit(c) { + let t = self.syntaxerror(format!( + "invalid digit '{}' in binary literal", + c as u8 as char + )); + return self.make(t, self.start, Some(self.cur)); + } + self.backup(c); + let t = self.syntaxerror("invalid binary literal"); + return self.make(t, self.start, Some(self.cur)); + } + loop { + c = self.nextc(); + if c != '0' as i32 && c != '1' as i32 { + break; + } + } + if c != '_' as i32 { + break; + } + } + if is_digit(c) { + let t = self.syntaxerror(format!( + "invalid digit '{}' in binary literal", + c as u8 as char + )); + return self.make(t, self.start, Some(self.cur)); + } + if !self.verify_end_of_number(c, "binary") { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + } else { + // Maybe old-style octal; in any case allow '0' itself. + let mut nonzero = false; + loop { + if c == '_' as i32 { + c = self.nextc(); + if !is_digit(c) { + self.backup(c); + let t = self.syntaxerror("invalid decimal literal"); + return self.make(t, self.start, Some(self.cur)); + } + } + if c != '0' as i32 { + break; + } + c = self.nextc(); + } + let zeros_end = self.cur; + if is_digit(c) { + nonzero = true; + c = self.decimal_tail(); + if c == 0 { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + } + if c == '.' as i32 { + c = self.nextc(); + return self.number_fraction(c); + } else if c == 'e' as i32 || c == 'E' as i32 { + return self.number_exponent(c); + } else if c == 'j' as i32 || c == 'J' as i32 { + return self.number_imaginary(); + } else if nonzero && !self.extra_tokens { + // Old-style octal: now disallowed. + self.backup(c); + let col = (self.start.unwrap() as i64 + 1) - self.line_start as i64; + let end_col = zeros_end as i64 - self.line_start as i64; + let t = self.syntaxerror_range( + "leading zeros in decimal integer literals are not permitted; \ + use an 0o prefix for octal integers" + .to_owned(), + col, + end_col, + ); + return self.make(t, self.start, Some(self.cur)); + } + if !self.verify_end_of_number(c, "decimal") { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + } + } else { + // Decimal. + c = self.decimal_tail(); + if c == 0 { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + // Accept floating-point numbers. + if c == '.' as i32 { + c = self.nextc(); + return self.number_fraction(c); + } + if c == 'e' as i32 || c == 'E' as i32 { + return self.number_exponent(c); + } + if c == 'j' as i32 || c == 'J' as i32 { + return self.number_imaginary(); + } + if !self.verify_end_of_number(c, "decimal") { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + } + self.backup(c); + self.make(NUMBER, self.start, Some(self.cur)) + } + + /// The `fraction:` label — c is the char after the '.'. + fn number_fraction(&mut self, mut c: i32) -> RawToken { + if is_digit(c) { + c = self.decimal_tail(); + if c == 0 { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + } + if c == 'e' as i32 || c == 'E' as i32 { + return self.number_exponent(c); + } + if c == 'j' as i32 || c == 'J' as i32 { + return self.number_imaginary(); + } + if !self.verify_end_of_number(c, "decimal") { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + self.backup(c); + self.make(NUMBER, self.start, Some(self.cur)) + } + + /// The `exponent:` label — e is the 'e'/'E' just consumed. + fn number_exponent(&mut self, e: i32) -> RawToken { + let mut c = self.nextc(); + if c == '+' as i32 || c == '-' as i32 { + c = self.nextc(); + if !is_digit(c) { + self.backup(c); + let t = self.syntaxerror("invalid decimal literal"); + return self.make(t, self.start, Some(self.cur)); + } + } else if !is_digit(c) { + self.backup(c); + if !self.verify_end_of_number(e, "decimal") { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + self.backup(e); + return self.make(NUMBER, self.start, Some(self.cur)); + } + c = self.decimal_tail(); + if c == 0 { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + if c == 'j' as i32 || c == 'J' as i32 { + return self.number_imaginary(); + } + if !self.verify_end_of_number(c, "decimal") { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + self.backup(c); + self.make(NUMBER, self.start, Some(self.cur)) + } + + /// The `imaginary:` label — 'j'/'J' just consumed. + fn number_imaginary(&mut self) -> RawToken { + let c = self.nextc(); + if !self.verify_end_of_number(c, "imaginary") { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + self.backup(c); + self.make(NUMBER, self.start, Some(self.cur)) + } + + /// `tok_get_fstring_mode`. + fn get_fstring_mode(&mut self) -> RawToken { + let mut end_quote_size = 0; + let mut unicode_escape = false; + + self.start = Some(self.cur); + self.first_lineno = self.lineno; + self.starting_col_offset = self.col_offset; + + let (quote, quote_size, raw) = { + let m = self.modes.last().unwrap(); + (m.quote, m.quote_size, m.raw) + }; + + // If we start with a bracket, defer to normal mode: nothing to + // tokenize before it. + let start_char = self.nextc(); + if start_char == '{' as i32 { + let peek1 = self.nextc(); + self.backup(peek1); + self.backup(start_char); + if peek1 != '{' as i32 { + { + let m = self.modes.last_mut().unwrap(); + m.curly_bracket_expr_start_depth += 1; + if m.curly_bracket_expr_start_depth >= MAX_EXPR_NESTING { + let t = self.syntaxerror("f-string: expressions nested too deeply"); + return self.make(t, self.start, Some(self.cur)); + } + m.kind = ModeKind::Regular; + } + return self.get_normal_mode(); + } + } else { + self.backup(start_char); + } + + // Check if we are at the end of the string. On a mismatch C + // backs up only the mismatching char — matched quote chars stay + // consumed and become part of the FSTRING_MIDDLE below. + let mut at_end = true; + for _ in 0..quote_size { + let q = self.nextc(); + if q != quote { + self.backup(q); + at_end = false; + break; + } + } + if at_end { + self.modes.pop(); + return self.make(FSTRING_END, self.start, Some(self.cur)); + } + + self.multi_line_start = self.line_start; + while end_quote_size != quote_size { + let c = self.nextc(); + if self.done == Done::Error { + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + let in_format_spec = { + let m = self.modes.last().unwrap(); + m.in_format_spec && m.curly_bracket_expr_start_depth >= 0 + }; + + if c == EOF || (quote_size == 1 && c == '\n' as i32) { + // A newline ends a format spec for single-quoted + // f-strings (multi-line specs are only legal in + // triple-quoted ones). + if in_format_spec && c == '\n' as i32 { + if quote_size == 1 { + let t = self.syntaxerror( + "f-string: newlines are not allowed in format specifiers for single quoted f-strings", + ); + return self.make(t, self.start, Some(self.cur)); + } + self.backup(c); + let m = self.modes.last_mut().unwrap(); + m.kind = ModeKind::Regular; + m.in_format_spec = false; + return self.make(FSTRING_MIDDLE, self.start, Some(self.cur)); + } + + // Report the error from the initial quote character. + let (fs_start, fs_mls, fs_ls) = { + let m = self.modes.last().unwrap(); + (m.start, m.multi_line_start, m.line_start) + }; + self.cur = fs_start + 1; + self.line_start = fs_mls; + let start_line = self.lineno; + self.lineno = fs_ls; + + if quote_size == 3 { + self.syntaxerror(format!( + "unterminated triple-quoted f-string literal (detected at line {start_line})" + )); + if c != '\n' as i32 { + self.done = Done::Eofs; + } + return self.make(ERRORTOKEN, self.start, Some(self.cur)); + } + let t = self.syntaxerror(format!( + "unterminated f-string literal (detected at line {start_line})" + )); + return self.make(t, self.start, Some(self.cur)); + } + + if c == quote { + end_quote_size += 1; + continue; + } + end_quote_size = 0; + + if c == '{' as i32 { + let peek = self.nextc(); + if peek != '{' as i32 || in_format_spec { + self.backup(peek); + self.backup(c); + { + let m = self.modes.last_mut().unwrap(); + m.curly_bracket_expr_start_depth += 1; + if m.curly_bracket_expr_start_depth >= MAX_EXPR_NESTING { + let t = self.syntaxerror("f-string: expressions nested too deeply"); + return self.make(t, self.start, Some(self.cur)); + } + m.kind = ModeKind::Regular; + m.in_format_spec = false; + } + return self.make(FSTRING_MIDDLE, self.start, Some(self.cur)); + } + return self.make(FSTRING_MIDDLE, self.start, Some(self.cur - 1)); + } else if c == '}' as i32 { + if unicode_escape { + return self.make(FSTRING_MIDDLE, self.start, Some(self.cur)); + } + let peek = self.nextc(); + // Format specs can't legally use double brackets, so `}}` + // at bracket-depth 0 outside a spec is a literal brace. + let cursor = self.modes.last().unwrap().curly_bracket_depth; + if peek == '}' as i32 && !in_format_spec && cursor == 0 { + return self.make(FSTRING_MIDDLE, self.start, Some(self.cur - 1)); + } + self.backup(peek); + self.backup(c); + { + let m = self.modes.last_mut().unwrap(); + m.kind = ModeKind::Regular; + m.in_format_spec = false; + } + return self.make(FSTRING_MIDDLE, self.start, Some(self.cur)); + } else if c == '\\' as i32 { + let mut peek = self.nextc(); + #[allow(unused_assignments)] + if peek == '\r' as i32 { + peek = self.nextc(); + } + // A backslash right before a curly brace: restore and + // let the loop handle the brace itself. (The invalid + // escape SyntaxWarning is suppressed here, like the + // number-literal one.) + if peek == '{' as i32 || peek == '}' as i32 { + self.backup(peek); + continue; + } + if !raw { + if peek == 'N' as i32 { + // Handle named unicode escapes (\N{BULLET}). + peek = self.nextc(); + if peek == '{' as i32 { + unicode_escape = true; + } else { + self.backup(peek); + } + } + } + // else: skip the escaped character. + } + } + + // Backup the quotes: emit a final FSTRING_MIDDLE; the quotes + // become the FSTRING_END on the next iteration. + for _ in 0..quote_size { + self.backup(quote); + } + self.make(FSTRING_MIDDLE, self.start, Some(self.cur)) + } +} + +fn is_string_lit(ty: i32) -> bool { + ty == STRING || ty == FSTRING_MIDDLE +} + +fn chars_of(bytes: &[u8]) -> i64 { + String::from_utf8_lossy(bytes).chars().count() as i64 +} + +/// The driver: `tokenizeriter_next` in a loop, producing the finished +/// 5-tuples plus an optional structured error. +fn run(lines: Vec, extra_tokens: bool) -> (Vec, Option) { + let mut tok = Tok::new(lines, extra_tokens); + let mut out: Vec = Vec::new(); + + loop { + let raw = tok.get(); + let mut ty = raw.ty; + if ty == ERRORTOKEN { + let err = tok.err.take().unwrap_or_else(|| tokenizer_error(&tok)); + return (out, Some(error_object(err))); + } + + let mut string: String = match (raw.start, raw.end) { + (Some(s), Some(e)) if s <= e && e <= tok.buf.len() => { + String::from_utf8_lossy(&tok.buf[s..e]).into_owned() + } + _ => String::new(), + }; + + let is_trailing_token = ty == ENDMARKER || (ty == DEDENT && tok.done == Done::Eof); + + let line_start = if is_string_lit(ty) { + tok.multi_line_start + } else { + tok.line_start + }; + let line: String = if tok.extra_tokens && is_trailing_token { + String::new() + } else { + let ls = line_start.min(tok.inp); + let mut size = tok.inp - ls; + if size >= 1 && tok.implicit_newline { + size -= 1; + } + String::from_utf8_lossy(&tok.buf[ls..ls + size]).into_owned() + }; + + let mut lineno = if is_string_lit(ty) { + tok.first_lineno + } else { + tok.lineno + }; + let mut end_lineno = tok.lineno; + let mut col_offset: i64 = -1; + let mut end_col_offset: i64 = -1; + if let Some(s) = raw.start { + if s >= line_start { + col_offset = chars_of(&tok.buf[line_start..s]); + } + } + if let Some(e) = raw.end { + if e >= tok.line_start { + if lineno == end_lineno { + // Same line: chars from the (string-lit adjusted) + // line start. + end_col_offset = col_offset.max(0) + + chars_of(&tok.buf[raw.start.unwrap_or(e).max(line_start)..e]); + } else { + end_col_offset = chars_of(&tok.buf[tok.line_start..e]); + } + } + } + + if tok.extra_tokens { + if is_trailing_token { + lineno += 1; + end_lineno = lineno; + col_offset = 0; + end_col_offset = 0; + } + // Match the original Python tokenize implementation. + if ty > DEDENT && ty < OP { + ty = OP; + } else if ty == NEWLINE { + string = if tok.implicit_newline { + String::new() + } else if raw.start.is_some_and(|s| tok.buf.get(s) == Some(&b'\r')) { + "\r\n".to_owned() + } else { + "\n".to_owned() + }; + end_col_offset += 1; + } else if ty == NL && tok.implicit_newline { + string = String::new(); + } + } + + out.push(Object::new_tuple(vec![ + Object::Int(i64::from(ty)), + Object::from_str(string), + Object::new_tuple(vec![Object::Int(lineno), Object::Int(col_offset)]), + Object::new_tuple(vec![Object::Int(end_lineno), Object::Int(end_col_offset)]), + Object::from_str(line), + ])); + + if ty == ENDMARKER { + return (out, None); + } + } +} + +/// `_tokenizer_error` — build the exception from `tok->done` when the +/// lexer returned ERRORTOKEN without setting one itself. +fn tokenizer_error(tok: &Tok) -> PendingError { + let (kind, msg): (&'static str, &str) = match tok.done { + Done::Eof => { + // PyErr_SyntaxLocationObject flavour: bare location, no text. + return PendingError { + kind: "syntax", + msg: "unexpected EOF in multi-line statement".to_owned(), + lineno: tok.lineno, + offset: tok.inp as i64, + text: None, + end_lineno: None, + end_offset: None, + }; + } + Done::Dedent => ( + "indent", + "unindent does not match any outer indentation level", + ), + Done::TabSpace => ("tab", "inconsistent use of tabs and spaces in indentation"), + Done::TooDeep => ("indent", "too many levels of indentation"), + Done::LineCont => ( + "syntax", + "unexpected character after line continuation character", + ), + _ => ("syntax", "unknown tokenization error"), + }; + // error_line = the whole buffer minus its trailing newline; offset = + // char offset of tok->inp (one past the end — the C conversion reads + // the NUL terminator as one extra char). + let size = tok.inp.saturating_sub(1); + let error_line = String::from_utf8_lossy(&tok.buf[..size]).into_owned(); + let offset = error_line.chars().count() as i64 + 1; + PendingError { + kind, + msg: msg.to_owned(), + lineno: tok.lineno, + offset, + text: Some(error_line), + end_lineno: None, + end_offset: None, + } +} + +fn error_object(e: PendingError) -> Object { + Object::new_tuple(vec![ + Object::from_static(match e.kind { + "indent" => "indent", + "tab" => "tab", + _ => "syntax", + }), + Object::from_str(e.msg), + Object::Int(e.lineno), + Object::Int(e.offset), + match e.text { + Some(t) => Object::from_str(t), + None => Object::None, + }, + match e.end_lineno { + Some(l) => Object::Int(l), + None => Object::None, + }, + match e.end_offset { + Some(o) => Object::Int(o), + None => Object::None, + }, + ]) +} + +/// `_tokenize_core.tokens(lines, extra_tokens)` → +/// `(list[token-5-tuple], error-descriptor | None)`. +fn tokens_fn(args: &[Object]) -> Result { + let lines_obj = args + .first() + .ok_or_else(|| type_error("tokens() missing required argument 'lines'"))?; + let extra_tokens = matches!(args.get(1), Some(Object::Bool(true)) | Some(Object::Int(1))); + let mut lines: Vec = Vec::new(); + match lines_obj { + Object::List(l) => { + for item in l.borrow().iter() { + match item { + Object::Str(s) => lines.push(s.to_string()), + _ => return Err(type_error("tokens() lines must be a list of str")), + } + } + } + _ => return Err(type_error("tokens() lines must be a list of str")), + } + let (toks, err) = run(lines, extra_tokens); + Ok(Object::new_tuple(vec![ + Object::new_list(toks), + err.unwrap_or(Object::None), + ])) +} + +pub fn build(_cache: &ModuleCache) -> Rc { + let dict = Rc::new(RefCell::new(DictData::default())); + { + let mut d = dict.borrow_mut(); + d.insert( + DictKey(Object::from_static("__name__")), + Object::from_static("_tokenize_core"), + ); + d.insert( + DictKey(Object::from_static("__doc__")), + Object::from_static( + "WeavePy native tokenizer core (RFC 0052) — CPython 3.13 lexer port.", + ), + ); + let f = Object::Builtin(Rc::new(BuiltinFn::new("tokens", tokens_fn))); + crate::descr_registry::register_module(&f, "_tokenize_core"); + d.insert(DictKey(Object::from_static("tokens")), f); + } + Rc::new(PyModule { + name: "_tokenize_core".to_owned(), + filename: None, + dict, + }) +} diff --git a/crates/weavepy/src/lib.rs b/crates/weavepy/src/lib.rs index 4e14aad..b2ca1df 100644 --- a/crates/weavepy/src/lib.rs +++ b/crates/weavepy/src/lib.rs @@ -283,7 +283,17 @@ pub fn run_source_with_options(source: &str, opts: &RunOptions) -> Result<(), Er // once the interpreter is up, just before the module body runs. let (module_res, escape_warnings) = parser::parse_module_with_warnings(source_ref); let module = module_res?; - let code = compiler::compile_module_with_source(&module, source_ref, &opts.filename)?; + // `-O`/`-OO` applies to the main module too (assert/docstring + // stripping, `__debug__` folding) — RFC 0052. + let code = compiler::compile_module_with_options( + &module, + source_ref, + &opts.filename, + compiler::CompileOptions { + flags: 0, + optimize: opts.flags.optimize, + }, + )?; let mut interpreter = vm::Interpreter::default(); interpreter.apply_run_options(&opts.flags); if !opts.flags.safe_path { @@ -477,6 +487,23 @@ fn format_compile_error(source: &str, filename: &str, err: &compiler::CompileErr } fn format_lex_error(source: &str, filename: &str, err: &lexer::LexError) -> String { + // A line continuation at EOF with nothing before it on the line: + // CPython's *file* tokenizer reports column 0 for this, and the + // C-level error printer then omits the caret line entirely + // (test_eof's bpo-2180 from-file cases assert on that shape). + if let lexer::LexError::UnexpectedEofParsing { + pos, + line_had_tokens: false, + } = err + { + let loc = SourceLocation::from_byte(source, *pos); + let rtext = loc.line_text.trim_end_matches('\n'); + let ltext = rtext.trim_start_matches([' ', '\n', '\x0c']); + return format!( + " File \"{filename}\", line {}\n {ltext}\nSyntaxError: {err}\n", + loc.line + ); + } let byte = err.byte_offset(); format_syntax_error_span(source, filename, byte, byte, &err.to_string()) } diff --git a/docs/rfcs/0052-compiler-front-end-fidelity.md b/docs/rfcs/0052-compiler-front-end-fidelity.md new file mode 100644 index 0000000..849110a --- /dev/null +++ b/docs/rfcs/0052-compiler-front-end-fidelity.md @@ -0,0 +1,386 @@ +# RFC 0052: Conformance wave 7 — compiler front-end fidelity: real `compile()`, pegen-exact syntax errors, `tokenize`/`symtable`, and patchable builtins + +- **Status**: Accepted +- **Authors**: WeavePy authors +- **Created**: 2026-07-13 +- **Tracking issue**: TBD +- **Builds on**: RFC 0051 (wave 6 — verbatim `typing` + core-language + burn-down), RFC 0049 (measured whole-suite baseline protocol), + RFC 0033 (CPython-faithful code objects, `_ast`/`dis`/`symtable` + cores), RFC 0005 (pegen-exact f-string errors — the pattern this + wave generalizes). + +## Summary + +Wave 6 left the sweep at 254 of 427 vendored CPython 3.13 `Lib/test/` +labels passing. Reading the remaining red rows, the largest coherent +cluster is the **compiler front end**: `compile()` accepts no keyword +arguments and cannot compile an AST object (it re-parses stashed +source text), `PyCF_*`/`CO_FUTURE_*` flags don't exist, `optimize` +levels are ignored, a dozen suites fail on non-pegen `SyntaxError` +messages (`test_syntax`, `test_eof`, `test_global`, `test_flufl`, +`test_future_stmt`, …), `tokenize` still emits pre-PEP-701 f-string +tokens, and `symtable` lacks the PEP 695/annotation block types that +3.13's tests probe first. + +Wave 7 makes the front end *real*: + +1. **`compile()` grows its full CPython signature** — `flags`, + `dont_inherit`, `optimize`, keyword acceptance, `PyCF_ONLY_AST`, + `PyCF_ALLOW_TOP_LEVEL_AWAIT`, `PyCF_DONT_IMPLY_DEDENT`, + `PyCF_ALLOW_INCOMPLETE_INPUT`, `PyCF_TYPE_COMMENTS` (accepted), + and `CO_FUTURE_*` threading — over a new **Python-AST → Rust-AST + converter** so `compile(tree, …)` compiles the tree the caller + built (pytest assertion rewriting's exact shape), not a stashed + copy of the original text. +2. **Syntax errors go pegen-exact beyond f-strings** — the + unterminated-string family ("detected at line N"), `unexpected EOF + while parsing`, `global`/`nonlocal` declaration-ordering errors + from a symtable-fidelity pass, `barry_as_FLUFL` grammar switching, + and future-feature diagnostics. +3. **`tokenize` becomes 3.13-faithful** — a native `_tokenize` core + over `weavepy-lexer` emitting PEP 701 `FSTRING_START` / + `FSTRING_MIDDLE` / `FSTRING_END` triples, exact token types, and + the `_generate_tokens_from_c_tokenizer` internal the test suite + drives. +4. **`symtable` completes** — PEP 695 `type alias` / `type + parameters` / `TypeVar` bound blocks and annotation scopes, plus + `filename`/`compile_type` wiring. +5. **Builtins become patchable** — the interpreter's private builtins + dict and `sys.modules['builtins'].__dict__` unify into one shared + dict, `LOAD_GLOBAL` resolves builtins through the frame's + `__builtins__`, and the inline caches gain version guards, so + `unittest.mock.patch('builtins.open')` behaves like CPython. + +As with every wave since RFC 0036, the deliverable is *measured*: the +full sweep is re-run, `tests/regrtest/expectations.toml` is rewritten +from evidence, and every remaining red carries an actionable +first-failure reason. + +## Motivation + +1. **`compile()`-from-AST is the drop-in gate for test tooling.** + pytest's assertion rewriting is `ast.parse` → `NodeTransformer` → + `compile(tree, path, "exec", dont_inherit=True)`. Today that + compiles the *original* source (the stashed-text hack), silently + dropping the rewrite — assertions "pass" without introspection. + coverage.py, hypothesis, numba, attrs, and the standard library's + own `codeop`/`doctest` also feed flags or trees into `compile()`. + The measured `test_compile` row fails ~60 of ~190 tests on exactly + this surface. +2. **Syntax-error fidelity is cheap conformance with a proven + pattern.** RFC 0005 already made f-string sub-parse errors + pegen-exact; the same architecture extends to the remaining + message families. Six labels fail *first* on message shape alone + (`test_eof`, `test_global`, `test_flufl`, `test_future_stmt`, + `test_syntax`, `test_source_encoding`), and `test_grammar`'s + measured reason names top-level await. +3. **`tokenize` is load-bearing for the tooling ecosystem.** inspect, + doctest, IPython, coverage, black, and 2to3-era tools consume it. + The frozen pre-PEP-701 port mis-tokenizes every f-string in + 3.12+ style code; `test_tokenize` (3,243 lines) is skipped with a + stale reason. +4. **Patchable builtins close a semantic hole, not a test hack.** + CPython resolves global-scope misses through the frame's + `f_builtins`, which is `builtins.__dict__` — one namespace, user + mutable. WeavePy's two-dict scheme diverges the moment anything + writes through a path the mirror doesn't cover (`mock.patch`, + `dict.__setitem__`, `exec` with custom `__builtins__`). It is the + measured blocker on `test_argparse` and a residual in the mock + cluster (`test_unittest`, `test_ensurepip`, `test_mimetypes` had + to work around it). +5. **Cost of inaction.** The README's drop-in claim is graded by the + sweep *and* by "clone a project, run its pytest suite". Both are + currently capped by the front end: the interpreter runs the code + but cannot ingest the ecosystem's compile-time metaprogramming. + +## CPython reference + +- `Python/pythonrun.c`, `Python/compile.c`, `Python/ast.c` — + `compile()` semantics: flag validation (`PyCF_MASK`, + `PyCF_MASK_OBSOLETE`), `dont_inherit`, `optimize` (-1/0/1/2), + AST-object input via `PyAST_obj2mod` (mode/node-type agreement, + recursive field validation, exact `TypeError`/`ValueError` shapes). +- `Include/cpython/compile.h` + `Lib/ast.py` — the `PyCF_*` constant + values (`PyCF_ONLY_AST` 0x400, `PyCF_TYPE_COMMENTS` 0x1000, + `PyCF_ALLOW_TOP_LEVEL_AWAIT` 0x2000, `PyCF_OPTIMIZED_AST` + 0x400|0x8000, `PyCF_DONT_IMPLY_DEDENT` 0x200, + `PyCF_ALLOW_INCOMPLETE_INPUT` 0x4000) and `Lib/__future__.py` — + `CO_FUTURE_*` bits. +- `Parser/pegen_errors.c`, `Parser/tokenizer/helpers.c` — the + unterminated-string / EOF message family + (`"unterminated string literal (detected at line %d)"`, + `"unterminated triple-quoted string literal (detected at line %d)"`, + `"unexpected EOF while parsing"`) and error-location rules. +- `Python/symtable.c` — `"name '%s' is assigned to before global + declaration"`, `"name '%s' is used prior to global declaration"` + (and the `nonlocal` twins), block types (`TypeAliasBlock`, + `TypeParametersBlock`, `TypeVariableBlock`, `AnnotationBlock`), + and `Modules/symtablemodule.c` for the `_symtable` surface. +- `Parser/tokenizer/*.c` + `Lib/tokenize.py` (3.13) — PEP 701 + f-string tokens, `TokenizerIter`, `detect_encoding`, + `_generate_tokens_from_c_tokenizer`. +- `Grammar/python.gram` — `barry_as_FLUFL` (`'<>' { … barry_as_flufl + … }`) and `invalid_*` rules for message text. +- `Python/ceval.c` `_PyEval_GetBuiltin` / `frameobject.c` + `f_builtins` — single-namespace builtins resolution; + `Python/specialize.c` `LOAD_GLOBAL` version guards. +- Acceptance tests: `Lib/test/test_compile.py`, `test_syntax.py`, + `test_eof.py`, `test_global.py`, `test_flufl.py`, + `test_future_stmt/`, `test_source_encoding.py`, `test_tokenize.py`, + `test_symtable.py`, `test_type_comments.py`, `test_grammar.py`, + `test_codeop.py`, `test_code_module.py`, `test_unparse.py`, + `test_argparse.py`, and the mock-cluster residuals. + +## Detailed design + +### WS1 — `compile()` full surface + AST-object lowering + +**Signature.** `do_compile_call` gains CPython's exact signature +(`source, filename, mode, flags=0, dont_inherit=False, optimize=-1`, +all usable as keywords via a `call_kw` binding), with CPython's +validation order and error shapes: unknown flag bits → +`ValueError("compile(): unrecognised flags")`, `optimize` outside +{-1, 0, 1, 2} → `ValueError("compile(): invalid optimize value")`, +oversized ints → `OverflowError`, `filename` accepting `str`, `bytes`, +and `os.PathLike` with embedded-NUL rejection. + +**Constants.** A new frozen-`ast`-visible constant surface: +`ast.PyCF_ONLY_AST`, `PyCF_TYPE_COMMENTS`, `PyCF_ALLOW_TOP_LEVEL_AWAIT`, +`PyCF_OPTIMIZED_AST` on `ast` and `_ast`; `PyCF_DONT_IMPLY_DEDENT` and +`PyCF_ALLOW_INCOMPLETE_INPUT` honored where `codeop` already defines +them. `__future__.CO_FUTURE_*` values become real: the compiler +records active futures into `co_flags`, and `dont_inherit=False` +inherits the *calling frame's* future bits like CPython. + +**AST-object input.** The stashed-source hack is replaced by a real +converter in `ast_mod.rs`: `obj2ast` walks a Python AST instance +(any object exposing `_fields`, matching `PyAST_obj2mod`'s duck +typing) and rebuilds the `weavepy_parser::ast` tree, validating node +types, field arity, position attributes (with CPython's +missing-`lineno` error text), and mode/root-node agreement +(`exec`→`Module`, `eval`→`Expression`, `single`→`Interactive`). +`compile(tree, …)` then flows through the normal compiler. The +`_weavepy_source` stash is deleted. `PyCF_ONLY_AST` returns the parse +tree (built by the existing Rust→Python builder) without compiling; +`ast.parse` becomes literally `compile(source, filename, mode, +PyCF_ONLY_AST | extra_flags)`. + +**`optimize` levels.** Threaded through `Compiler::new`: +level ≥ 1 folds `__debug__` to `False` and strips `assert` +statements; level 2 additionally drops docstrings (module, class, +function — `co_consts[0]` shape matching CPython). The CLI's +`-O`/`-OO` set the interpreter default so `sys.flags.optimize`, +`__debug__`, and bare `compile()` agree. + +**Top-level await.** `PyCF_ALLOW_TOP_LEVEL_AWAIT` compiles module +code with `CO_COROUTINE`, permitting `await`/`async for`/`async with` +at top level (the `asyncio` REPL contract). Without the flag the +existing `'await' outside function` error stands. + +### WS2 — pegen-exact syntax errors beyond f-strings + +Following the RFC 0005 layering (lexer emits structured errors, +parser maps them to CPython message families, VM shapes the final +`SyntaxError`): + +- **Unterminated strings.** `LexError::UnterminatedString` splits + into single-line and triple-quoted variants carrying the *detection + line*; messages become `unterminated string literal (detected at + line N)` / `unterminated triple-quoted string literal (detected at + line N)` with the opening-quote offset and stripped-`\n` `.text`, + matching `test_eof` byte-for-byte (including the latin-1-cookie and + BOM re-lining cases). +- **EOF continuation.** A trailing `\` before EOF raises + `unexpected EOF while parsing` with CPython's offset (end of line, + `.text` keeping the backslash + `\n`). +- **Declaration ordering.** The `validate.rs` symtable pass tracks + per-scope first-use/first-assignment/first-annotation positions and + raises `name 'x' is assigned to before global declaration`, + `… is used prior to global declaration`, `… is parameter and + global`, and the `nonlocal` twins, at the *directive's* position + (`test_global` asserts lineno/offset of the `global` statement). +- **FLUFL.** The lexer learns `<>` as a token; the parser accepts it + as `!=` only when `barry_as_FLUFL` is active (from a `__future__` + import seen earlier in the token stream, or `CO_FUTURE_BARRY_AS_BDFL` + in `compile()` flags) and then *rejects* `!=` with `with Barry as + BDFL, use '<>' instead of '!='`; inactive `<>` stays bare + `invalid syntax` at CPython's offset. +- **Future features.** `test_future_stmt`'s message set: `future + feature X is not defined`, placement errors, and `not a chance` + keep their current text but gain CPython's positions; new: + `__future__` imports set `CO_FUTURE_*` bits observable on + `co_flags` (WS1). +- **`test_syntax` burn-down.** The suite is doctest-driven message + comparison; work the measured diffs (the `'invalid syntax'` + substring family, `cannot assign to …` positions, `expected ':'` + hints) until the residual is enumerable, recording what remains. + +### WS3 — 3.13-faithful `tokenize` + +A native `_tokenize` module (new `tokenize_mod.rs`) exposes +`TokenizerIter` over `weavepy-lexer`: + +- Emits CPython 3.13 token streams: exact-type mapping + (`OP` exact types via `EXACT_TOKEN_TYPES`), `NL` vs `NEWLINE`, + `INDENT`/`DEDENT`, `COMMENT`, and PEP 701 `FSTRING_START` / + `FSTRING_MIDDLE` / `FSTRING_END` with CPython's interior-token + re-tokenization of replacement fields. The lexer already scans + f-string fields structurally (RFC 0005); the iterator re-projects + those spans as token triples rather than one `STRING`. +- `(line, col)` positions computed the way CPython reports them + (character columns, `''`-line synthetic tokens at EOF). +- Frozen `tokenize.py` is replaced by CPython 3.13's file, verbatim + per the adoption policy, over the native core + (`_tokenize.TokenizerIter`), keeping `detect_encoding` behavior and + the `_generate_tokens_from_c_tokenizer` internal the tests import. +- `test_tokenize`'s skip row is retired; the suite is measured. + +### WS4 — `symtable` completion + +`symtable_mod.rs` grows the 3.13 block model: + +- PEP 695 constructs produce their dedicated blocks: `type X = …` → + `TypeAliasBlock`, generic params on `def`/`class` → + `TypeParametersBlock`, TypeVar bounds/defaults → + `TypeVariableBlock`; annotations under `from __future__ import + annotations` or in stubs produce `AnnotationBlock` where CPython + does. +- `filename` threads into parse errors; `compile_type` selects + exec/eval/single parsing like `_symtable.symtable`. +- Wrapper `symtable.py` re-syncs with 3.13 (`Class.get_methods()` + deprecation shape, `SymbolTableType` enum values). + +### WS5 — patchable builtins + +Per the RFC 0024/0031 engine-work conventions: + +- **One namespace.** `sys.modules['builtins'].dict` *is* the + interpreter's builtins `Rc` — the frozen `builtins.py` copy loop + and the `store_attr` mirroring are deleted. `PyFrame.builtins` and + `function.__builtins__` observe the same dict. +- **Frame-scoped resolution.** `LOAD_GLOBAL`'s slow path and the + specializer resolve builtins via `globals['__builtins__']` (module + or dict, falling back to the interpreter dict), matching + `_PyEval_GetBuiltin`, so `exec(code, {'__builtins__': {...}})` + behaves. +- **Cache correctness.** The `LoadGlobalModule`/`LoadGlobalBuiltin` + inline caches gain the dict version guards their `bytecode.rs` + comment already promises (a per-dict `Cell` bumped on any + structural change or builtins write), so a patched `open` deopts + the cache instead of serving the stale slot. Call-site intercepts + in `dispatch_call` only trigger when the loaded object still *is* + the original builtin. + +### WS6 — re-measure and re-baseline + +Per the RFC 0049 protocol: two full sweeps +(`weavepy-conformance regrtest --all-cpython --mode subprocess +--jobs 8`), cross-checked; `expectations.toml` rewritten so every +row is measured; bundled fixtures stay green; new bundled regrtests +land for the novel surfaces (compile-from-AST round-trips incl. a +pytest-shaped assert rewrite, `PyCF_ONLY_AST`, optimize levels, +FLUFL, unterminated-string messages, PEP 701 tokenize streams, +PEP 695 symtable blocks, builtins patching through `mock.patch` and +raw dict mutation). + +### Acceptance criteria + +1. `compile(ast_tree, file, mode)` compiles the *given* tree: + a mutated-AST fixture (pytest-style assert rewrite) observably + executes the rewritten code; `compile(src, f, m, PyCF_ONLY_AST)` + returns an `ast.Module`. +2. `cpython/Lib/test/test_eof.py`, `test_global.py`, `test_flufl.py`, + and `test_future_stmt` flip to measured `pass`; `test_compile.py` + and `test_syntax.py` flip to `pass` or to measured rows whose + residuals are enumerated and small (they are the two largest + suites in the cluster). +3. `test_tokenize.py` is unskipped and measured; PEP 701 f-string + token triples match CPython on the bundled fixtures. +4. `test_symtable.py` flips to measured `pass`. +5. `mock.patch('builtins.open')` affects `open()` observed via + `LOAD_GLOBAL` in freshly compiled and *already-specialized* code; + `test_argparse.py`'s builtins-patching error clears (measured). +6. At least 10 net labels flip red→green on the full sweep versus the + wave-6 baseline. +7. `cargo fmt` / `clippy -D warnings` / `cargo test --workspace` / + `regrtest --check` all green. + +## Drawbacks + +- **The AST converter is a big, fiddly surface** (~100 node types × + field validation with exact error shapes). Mitigated by driving it + from the same node table the Rust→Python builder already encodes, + and by `test_compile`/`test_ast` grading both directions. +- **Unifying the builtins dict touches interpreter startup order.** + The frozen `builtins` module must exist before arbitrary imports; + regressions here break everything at once. Mitigated by keeping the + interpreter's dict as the single source and pointing the module at + it (not the reverse), so pre-module-load lookups are unchanged. +- **Dict version guards add a branch to hot lookups.** The guard is a + `u64` load+compare on the specialized path — the same cost CPython + pays; the perf RFCs' benchmarks gate regressions. +- **PEP 701 tokenize re-projection duplicates f-string structure + knowledge** between the scanner and the token iterator. Accepted: + the scanner already owns field spans; the iterator is a projection, + not a second tokenizer. + +## Alternatives + +- **Keep the stashed-source hack and special-case pytest** (teach the + rewriter to hand back source): rejected — every AST-mutating tool + would need its own hack, and CPython's `TypeError`/`ValueError` + validation surface would stay unimplementable. +- **Unparse the Python AST to text and re-parse** instead of a real + converter: rejected — loses exact positions (PEP 657 columns in + tracebacks would lie about rewritten code), can't represent + synthetic trees with deliberate positions, and diverges from + CPython's validation error shapes. +- **Port CPython's C tokenizer wholesale** for WS3: rejected — + `weavepy-lexer` is already conformance-graded against the CPython + oracle; a projection layer is ~10× smaller and keeps one lexer. +- **Extend `store_attr` mirroring instead of unifying the dicts**: + rejected — `dict.__setitem__`, `dict.update`, and mock internals + bypass attribute stores; the two-dict scheme is unfixable by + patching sync points (the current bug is the proof). + +## Prior art + +- **CPython** is the spec throughout; `PyAST_obj2mod` + + `ast_for_*` validation define the converter's contract, and 3.12's + `tokenize`-over-C-iterator rewrite (gh-102856) defines WS3's shape. +- **PyPy** compiles from `ast` objects natively (its compiler ingests + the app-level AST); pytest assertion rewriting has worked on PyPy + for a decade — evidence the converter approach, not the unparse + shortcut, is the durable one. +- **RustPython** exposes `compile(tree, …)` via an AST-to-bytecode + path over rustpython-ast and hit the same mode/node-agreement + error-shape details; their issue tracker documents the long tail + this RFC's validation matrix covers up front. +- **RFC 0005 (f-strings)** proved the layered pegen-message pattern + inside this codebase; WS2 is its generalization. + +## Unresolved questions + +- Whether `test_compile`'s optimizer-shape classes + (`TestStackSizeStability`, `TestInstructionSequence`, peephole + assertions) can pass without adopting CPython's exact block-layout + optimizer, or land as an enumerated residual. Acceptance + criterion 2 allows either. +- Whether `PyCF_TYPE_COMMENTS` gets real `# type:` parsing this wave + or validated-but-inert acceptance with `test_type_comments` + measured red on the parsing arc (the flag plumbing lands either + way). +- How far the `test_syntax` doctest matrix (2,723 lines of message + comparisons) converges inside the wave budget. + +## Future work + +- A CPython-shaped control-flow-graph optimizer pass (would retire + the `test_compile` optimizer residual and most of + `test_compiler_codegen`). +- `PyCF_OPTIMIZED_AST` constant folding on the returned tree. +- Wave 8 candidates unblocked here: `doctest`/`unittest` residuals + (need patchable builtins + compile flags), `codeop`/REPL parity + (needs `PyCF_ALLOW_INCOMPLETE_INPUT`), coverage.py support + (needs `tokenize` + trace fidelity). diff --git a/tests/regrtest/expectations.toml b/tests/regrtest/expectations.toml index 72236b2..3c91cdd 100644 --- a/tests/regrtest/expectations.toml +++ b/tests/regrtest/expectations.toml @@ -82,8 +82,10 @@ timeout_seconds = 180 reason = "measured (RFC 0049): fails in ~60s idle; near-boundary under load, budget raised" [tests."cpython/Lib/test/test_ast.py"] -status = "timeout" -reason = "measured (RFC 0049 wave-5 full-suite baseline): killed after 60s" +status = "fail" +reason = "measured (RFC 0052): suite now completes (~18s standalone, previously killed after 60s) via the compile()-from-AST path; 169 failures / 80 errors remain across full AST validation, node-construction, and position fidelity" +# ~18s standalone but sits near the 60s budget on a loaded 8-job sweep. +timeout_seconds = 180 [tests."cpython/Lib/test/test_asyncgen.py"] status = "pass" @@ -222,8 +224,8 @@ timeout_seconds = 120 reason = "RFC 0050 WS2+WS3: passes end-to-end (284 run, 13 skip). WS2 delivered error-handler unification, UTF-7, stateful UTF-16/32 decoders, the stream layer, unicode-escape handler protocol, 57 vendored charmap codepages, and the IDNA/punycode/nameprep surface. WS3 added the stateful CJK escape codecs (_codec_cjk_ext: hz, iso2022_jp/_1/_2/_2004/_3/_ext, iso2022_kr, johab, shift_jis_2004/shift_jisx0213) plus the euc_jisx0213 2000-emulation variant, ported from Modules/cjkcodecs and differentially verified against CPython 3.13." [tests."cpython/Lib/test/test_codeop.py"] -status = "fail" -reason = "measured (RFC 0049 wave-5 full-suite baseline): AssertionError: 1 != 2" +status = "pass" +reason = "RFC 0052: compile() flags (PyCF_DONT_IMPLY_DEDENT / PyCF_ALLOW_INCOMPLETE_INPUT) + pegen-exact incomplete-input diagnostics" [tests."cpython/Lib/test/test_collections.py"] status = "pass" @@ -231,6 +233,9 @@ reason = "measured: passes end-to-end in ~7s under -j8 (RFC 0037 WS8 — CPython [tests."cpython/Lib/test/test_compile.py"] status = "fail" +# ~30-55s under -j8; observed >60s on a loaded host (RFC 0052 +# re-measure) — verdict is stable, raise headroom only. +timeout_seconds = 180 reason = "compile builtin: PyCF_* flags + AST input handling" [tests."cpython/Lib/test/test_compiler_assemble.py"] @@ -247,7 +252,9 @@ reason = "RFC 0037 WS3: passes end-to-end (35 tests). The complex() constructor [tests."cpython/Lib/test/test_concurrent_futures.py"] status = "pass" -timeout_seconds = 300 +# ~120-250s under -j8; observed >300s on a loaded/thermally-throttled host +# (RFC 0052 re-measure) — verdict is stable, so raise headroom only. +timeout_seconds = 600 reason = "RFC 0040 WS6: passes as a graded unit — 267 run, 0 fail, 0 err, 20 skip (release build). The whole `concurrent.futures` matrix is green: ThreadPool + ProcessPool (fork/forkserver/spawn) across test_init/test_future/test_as_completed/test_wait/test_thread_pool/test_process_pool/test_shutdown/test_deadlock. Two fixes landed this wave. (1) A VM deterministic-finalization bug: a comprehension's anonymous ``/``/``/`` is GC-tracked when it captures cells, so when emitted by `MakeFunction` and consumed by the very next `Call` a plain `Rc` drop left it pinned by its own GC handle — leaking the captured locals until the next cycle collection. `ThreadPoolExecutor.map`'s `result_iterator` closes over the listcomp's `self`, so every `map` leaked one ref to the executor; `del executor` then never hit refcount 0, the idle-worker wakeup weakref-callback never fired, and `test_shutdown`/`test_del_shutdown` hung. `reap_call_receiver` now routes a uniquely-held call-temporary `Function` through the same prompt-reap cascade as closure-function locals (the leak also resolved the test_future 18→20 and test_as_completed 19→20 residuals). (2) Added the native `faulthandler` module: `test_deadlock` fires `faulthandler._sigsegv()` inside a pool worker to force a hard crash and assert `BrokenProcessPool` recovery; without the module `import faulthandler` raised in the worker, so the crash never happened and every recovery case errored or hung to `LONG_TIMEOUT`. The crash primitives genuinely `raise(3)` the signal, and ProcessPool broken-worker detection (verified independently for SIGSEGV/SIGABRT/os._exit) recovers promptly." [tests."cpython/Lib/test/test_configparser.py"] @@ -360,6 +367,9 @@ reason = "measured (RFC 0049 wave-5 full-suite baseline): AttributeError: 'Dynam [tests."cpython/Lib/test/test_email.py"] status = "fail" +# ~35-55s under -j8; observed >60s on a loaded host (RFC 0052 +# re-measure) — verdict is stable, raise headroom only. +timeout_seconds = 180 reason = "email: policy.utf8 + EmailMessage.iter_attachments" [tests."cpython/Lib/test/test_embed.py"] @@ -379,8 +389,8 @@ status = "pass" reason = "measured (RFC 0051 wave-6): enumerate() accepts its documented keyword arguments (iterable=/start=) — the wave-5 first-failure. ~6-9s under -j8 (pickle matrix)." [tests."cpython/Lib/test/test_eof.py"] -status = "fail" -reason = "measured (RFC 0049 wave-5 full-suite baseline): AssertionError: 'unterminated string literal at byte 5 (, line 1)' != 'unterminated triple-quoted string literal (detected at[23 chars]e 1)'" +status = "pass" +reason = "RFC 0052: CPython-exact unterminated-string / EOF SyntaxError wording ('detected at line N')" [tests."cpython/Lib/test/test_exception_group.py"] status = "fail" @@ -423,8 +433,8 @@ status = "pass" reason = "RFC 0037 WS3: passes end-to-end (skipped=3). float()/math coercion now routes user objects through __float__/__index__ (coerce_f64_opt) and int->float raises OverflowError when the magnitude exceeds the f64 range instead of silently yielding inf; NaN orderings are unordered (Lt/LtE/Gt/GtE are all False against a NaN); float.hex/fromhex roundtrip and the unbound float methods resolve." [tests."cpython/Lib/test/test_flufl.py"] -status = "fail" -reason = "measured (RFC 0049 wave-5 full-suite baseline): SyntaxError: unexpected token in expression: Greater" +status = "pass" +reason = "RFC 0052: PEP 401 barry_as_FLUFL future flag — lexer `<>` toggle threaded through compile()/eval and the REPL" [tests."cpython/Lib/test/test_fnmatch.py"] status = "pass" @@ -467,8 +477,8 @@ status = "pass" reason = "measured: passes end-to-end (RFC 0037 WS8 — partial/lru_cache/singledispatch + the classmethod(partial)/bound-method descriptor-redispatch fix) against the vendored CPython 3.13 Lib/test." [tests."cpython/Lib/test/test_future_stmt.py"] -status = "fail" -reason = "measured (RFC 0049 wave-5 full-suite baseline): SyntaxError: future feature spam is not defined" +status = "pass" +reason = "RFC 0052: CO_FUTURE_* threading through compile(flags=/dont_inherit=), future-feature validation, and __future__ module parity" [tests."cpython/Lib/test/test_gc.py"] status = "pass" @@ -499,8 +509,8 @@ status = "fail" reason = "measured (RFC 0049 wave-5 full-suite baseline): FileNotFoundError: [Errno 2] No such file or directory: '/Users/owencarey/Documents/weavefoundry/weavepy/vendor/cpython/Modules/getpath.py'" [tests."cpython/Lib/test/test_gettext.py"] -status = "fail" -reason = "measured: `gettext` now ships (frozen verbatim) and imports — previously `ModuleNotFoundError` — so the suite runs (71/78); 7 residual errors around the C-accelerated catalog/plural-eval edges remain a follow-up." +status = "pass" +reason = "RFC 0052: the 7 residual catalog/plural-eval errors cleared once tokenize/compile fidelity landed (gettext's plural-form parser tokenizes the expression)" [tests."cpython/Lib/test/test_glob.py"] status = "pass" @@ -508,12 +518,12 @@ timeout_seconds = 180 reason = "correct but slow (~51s idle); budget raised for parallel-load headroom" [tests."cpython/Lib/test/test_global.py"] -status = "fail" -reason = "measured (RFC 0049 wave-5 full-suite baseline): AssertionError: SyntaxError not raised" +status = "pass" +reason = "RFC 0052: symtable-stage 'name assigned/used before global declaration' SyntaxErrors" [tests."cpython/Lib/test/test_grammar.py"] -status = "fail" -reason = "exercises features still in flight (top-level await, complex f-strings)" +status = "pass" +reason = "RFC 0052: pegen-exact diagnostics, PyCF_ALLOW_TOP_LEVEL_AWAIT, and f-string fidelity closed the remaining grammar gaps" [tests."cpython/Lib/test/test_gzip.py"] status = "pass" @@ -899,6 +909,10 @@ reason = "measured (RFC 0049 wave-5 full-suite baseline): ImportError: cannot im [tests."cpython/Lib/test/test_repl.py"] status = "fail" +# Spawns subprocess REPLs; sits near the 60s budget under -j8 load +# (observed 45-70s). Raised so the stable `fail` verdict doesn't flip +# to a spurious `timeout`. +timeout_seconds = 180 reason = "measured (RFC 0049 wave-5 full-suite baseline): File '/Users/owencarey/Documents/weavefoundry/weavepy/ve…[truncated]" [tests."cpython/Lib/test/test_reprlib.py"] @@ -1062,12 +1076,12 @@ status = "fail" reason = "measured (RFC 0049 wave-5 full-suite baseline): EOFError: EOF when reading a line" [tests."cpython/Lib/test/test_symtable.py"] -status = "fail" -reason = "measured (RFC 0049 wave-5 full-suite baseline): AttributeError: 'NoneType' object has no attribute 'get_children'" +status = "pass" +reason = "RFC 0052: PEP 695 block types (type alias / type parameters / type variable / annotation) in _symtable, plus compile_type + filename threading" [tests."cpython/Lib/test/test_syntax.py"] -status = "fail" -reason = "measured (RFC 0049 wave-5 full-suite baseline): AssertionError: SyntaxError did not contain 'invalid syntax'" +status = "pass" +reason = "RFC 0052: pegen-exact SyntaxError messages, spans, and exception classes across the invalid_* rule family" [tests."cpython/Lib/test/test_sys.py"] status = "fail" @@ -1086,8 +1100,8 @@ status = "fail" reason = "measured (RFC 0049 wave-5 full-suite baseline): ImportError: cannot import name '_INSTALL_SCHEMES' from 'sysconfig'" [tests."cpython/Lib/test/test_tabnanny.py"] -status = "fail" -reason = "measured (RFC 0049 wave-5 full-suite baseline): AssertionError: '/var[26 chars]9tc1swkk3w0000gn/T/tmp_hhh2us3.py 3 \\'\\\\tprint('world')\\\\n\\'\\n' != '/var[26 chars]9tc1swkk3w0000gn/T/tmp_hhh2us3.py 3 \\'\\\\tprint(" +status = "pass" +reason = "RFC 0052: CPython 3.13-faithful tokenize (token text/positions) feeds tabnanny's whitespace analysis" [tests."cpython/Lib/test/test_tarfile.py"] status = "pass" @@ -1113,7 +1127,7 @@ reason = "measured: passes end-to-end after wiring `open(opener=...)` + file-des [tests."cpython/Lib/test/test_threading.py"] status = "pass" timeout_seconds = 600 -reason = "RFC 0039: real OS threads with cooperative GIL hand-off, faithful Lock/RLock (subclassable `_thread.RLock`), thread-death thread-local cleanup (foreign-thread `_DummyThread` reaping), and prompt refcount reclamation of acyclic garbage — including the cross-thread `Thread.run` teardown cycle (`test_no_refcycle_through_target`) swept on `join()`. All 213 tests pass (29 skips for CPython-only knobs). 600s: the suite runs ~32s standalone and passes under moderate load, but with the main thread participating in GIL hand-offs it has been observed exceeding even 300s under full parallel sweep load (RFC 0050 WS6 re-measure); the raised budget keeps the verdict about correctness, not scheduler contention." +reason = "RFC 0039: real OS threads with cooperative GIL hand-off, faithful Lock/RLock (subclassable `_thread.RLock`), thread-death thread-local cleanup (foreign-thread `_DummyThread` reaping), and prompt refcount reclamation of acyclic garbage — including the cross-thread `Thread.run` teardown cycle (`test_no_refcycle_through_target`) swept on `join()`. All 213 tests pass (29 skips for CPython-only knobs). 600s: the suite runs ~32s standalone and passes under moderate load, but with the main thread participating in GIL hand-offs it has been observed exceeding even 300s under full parallel sweep load (RFC 0050 WS6 re-measure); the raised budget keeps the verdict about correctness, not scheduler contention. KNOWN FLAKE (pre-existing, reproduces on unmodified main): `test_no_refcycle_through_target` intermittently fails when concurrent threads race the prompt-reap deadness check at `del self._kwargs` — the kwargs dict stays pinned by its GC handle, keeping the target instance's weakref live. Deterministic repro: busy daemon threads + gc.disable() flips it to 30/30 failures on main; the suite's own daemon-thread pressure makes it ~50% under load." [tests."cpython/Lib/test/test_threading_local.py"] status = "fail" @@ -1132,8 +1146,9 @@ status = "fail" reason = "measured (RFC 0049 wave-5 full-suite baseline): ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^…[truncated]" [tests."cpython/Lib/test/test_tokenize.py"] -status = "skip" -reason = "imports CPython-only internals (tokenize.tokenize, encoding cookies)" +status = "pass" +timeout_seconds = 600 +reason = "measured (RFC 0052): native _tokenize_core lexer port; the roundtrip sweeps over the vendored stdlib are correct but slow" [tests."cpython/Lib/test/test_tomllib.py"] status = "fail" diff --git a/tests/regrtest/test_builtins_patching.py b/tests/regrtest/test_builtins_patching.py new file mode 100644 index 0000000..50a8077 --- /dev/null +++ b/tests/regrtest/test_builtins_patching.py @@ -0,0 +1,99 @@ +"""RFC 0052 WS5 — patchable builtins. + +`sys.modules['builtins'].__dict__` and the interpreter's ambient +lookup namespace are one dict: attribute writes, raw dict mutation, +and `unittest.mock.patch` must all be observed by `LOAD_GLOBAL` — +including in *already-specialized* code — and sandboxed `exec` +namespaces must stay sealed. +""" + +import builtins +import sys +from unittest import mock + +# --- module identity --------------------------------------------------- +assert sys.modules['builtins'] is builtins +assert builtins.__dict__ is vars(sys.modules['builtins']) +assert sys._getframe(0).f_builtins is builtins.__dict__ +assert builtins.__name__ == 'builtins' + +# --- raw dict mutation is live for name resolution --------------------- +def use_len(x): + return len(x) + +assert use_len([1, 2, 3]) == 3 +orig_len = builtins.__dict__['len'] +builtins.__dict__['len'] = lambda x: 42 +try: + assert use_len([1, 2, 3]) == 42 +finally: + builtins.__dict__['len'] = orig_len +assert use_len([1, 2, 3]) == 3 + +# --- attribute writes and deletes -------------------------------------- +orig_abs = builtins.abs +builtins.abs = lambda x: 'patched' +try: + assert abs(-5) == 'patched' +finally: + builtins.abs = orig_abs +assert abs(-5) == 5 + +builtins.__dict__['_weave_tmp'] = 7 +assert _weave_tmp == 7 # noqa: F821 +del builtins.__dict__['_weave_tmp'] +try: + _weave_tmp # noqa: F821 +except NameError: + pass +else: + raise AssertionError('expected NameError after dict delete') + +# --- mock.patch deopts already-specialized LOAD_GLOBAL ------------------ +def hot(): + return len([1, 2]) + +for _ in range(300): # warm the inline cache + hot() +with mock.patch('builtins.len', lambda x: 'patched'): + assert hot() == 'patched' +assert hot() == 2 + +with mock.patch('builtins.open', mock.mock_open(read_data='data')): + with open('/nonexistent') as fh: + assert fh.read() == 'data' + +# --- func_builtins snapshots at function creation (CPython) ------------- +# Rebinding globals()['__builtins__'] between calls must not change an +# existing function's resolution (test_dynamic's +# test_cannot_replace_builtins_dict_between_calls). +saved = globals()['__builtins__'] +globals()['__builtins__'] = {'len': lambda x: 7} +try: + assert use_len([1, 2, 3]) == 3 +finally: + globals()['__builtins__'] = saved + +# --- sandboxed exec namespaces stay sealed ------------------------------ +ns = {'__builtins__': {'len': lambda x: -1}} +exec("r = len([1,2,3])", ns) +assert ns['r'] == -1 + +try: + exec("print('hi')", {'__builtins__': {}}) +except NameError: + pass +else: + raise AssertionError('expected NameError from sealed builtins') + +# A function *defined inside* the sandbox inherits the sandbox. +ns2 = {'__builtins__': {'len': lambda x: -2}} +exec("def f():\n return len([1])", ns2) +assert ns2['f']() == -2 + +# `__builtins__` may also be the module object itself (CPython allows both). +ns3 = {'__builtins__': builtins} +exec("r = len([1,2])", ns3) +assert ns3['r'] == 2 + +print('ok') diff --git a/tests/regrtest/test_doctest_machinery.py b/tests/regrtest/test_doctest_machinery.py index d0bee4d..fe44c6d 100644 --- a/tests/regrtest/test_doctest_machinery.py +++ b/tests/regrtest/test_doctest_machinery.py @@ -137,11 +137,17 @@ def test_single_compile_echoes_repr(self): import io import contextlib buf = io.StringIO() - code = compile("1 + 1\n'hi'\nNone\n", "", "single") + # "single" mode accepts one statement at a time (a multi-statement + # source is a SyntaxError, matching CPython) — compile each + # example the way doctest does. + ns = {} with contextlib.redirect_stdout(buf): - exec(code, {}) + for src in ("1 + 1\n", "'hi'\n", "None\n"): + exec(compile(src, "", "single"), ns) # 1+1 -> "2", 'hi' -> "'hi'", None is suppressed by displayhook. self.assertEqual(buf.getvalue(), "2\n'hi'\n") + with self.assertRaises(SyntaxError): + compile("1 + 1\n'hi'\nNone\n", "", "single") class BridgeTests(unittest.TestCase):