diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index b852d71b80082..4f1594d02a82e 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -714,7 +714,7 @@ impl LayoutCalculator { }, fields: FieldsShape::Arbitrary { offsets: [niche_offset].into(), - memory_index: [0].into(), + in_memory_order: [FieldIdx::new(0)].into(), }, backend_repr: abi, largest_niche, @@ -1008,8 +1008,8 @@ impl LayoutCalculator { let pair = LayoutData::::scalar_pair(&self.cx, tag, prim_scalar); let pair_offsets = match pair.fields { - FieldsShape::Arbitrary { ref offsets, ref memory_index } => { - assert_eq!(memory_index.raw, [0, 1]); + FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => { + assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]); offsets } _ => panic!("encountered a non-arbitrary layout during enum layout"), @@ -1061,7 +1061,7 @@ impl LayoutCalculator { }, fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), - memory_index: [0].into(), + in_memory_order: [FieldIdx::new(0)].into(), }, largest_niche, uninhabited, @@ -1110,10 +1110,10 @@ impl LayoutCalculator { let pack = repr.pack; let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align }; let mut max_repr_align = repr.align; - let mut inverse_memory_index: IndexVec = fields.indices().collect(); + let mut in_memory_order: IndexVec = fields.indices().collect(); let optimize_field_order = !repr.inhibit_struct_field_reordering(); let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() }; - let optimizing = &mut inverse_memory_index.raw[..end]; + let optimizing = &mut in_memory_order.raw[..end]; let fields_excluding_tail = &fields.raw[..end]; // unsizable tail fields are excluded so that we use the same seed for the sized and unsized layouts. let field_seed = fields_excluding_tail @@ -1248,12 +1248,10 @@ impl LayoutCalculator { // regardless of the status of `-Z randomize-layout` } } - // inverse_memory_index holds field indices by increasing memory offset. - // That is, if field 5 has offset 0, the first element of inverse_memory_index is 5. + // in_memory_order holds field indices by increasing memory offset. + // That is, if field 5 has offset 0, the first element of in_memory_order is 5. // We now write field offsets to the corresponding offset slot; // field 5 with offset 0 puts 0 in offsets[5]. - // At the bottom of this function, we invert `inverse_memory_index` to - // produce `memory_index` (see `invert_mapping`). let mut unsized_field = None::<&F>; let mut offsets = IndexVec::from_elem(Size::ZERO, fields); let mut offset = Size::ZERO; @@ -1265,7 +1263,7 @@ impl LayoutCalculator { align = align.max(prefix_align); offset = prefix_size.align_to(prefix_align); } - for &i in &inverse_memory_index { + for &i in &in_memory_order { let field = &fields[i]; if let Some(unsized_field) = unsized_field { return Err(LayoutCalculatorError::UnexpectedUnsized(*unsized_field)); @@ -1322,18 +1320,6 @@ impl LayoutCalculator { debug!("univariant min_size: {:?}", offset); let min_size = offset; - // As stated above, inverse_memory_index holds field indices by increasing offset. - // This makes it an already-sorted view of the offsets vec. - // To invert it, consider: - // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0. - // Field 5 would be the first element, so memory_index is i: - // Note: if we didn't optimize, it's already right. - let memory_index = if optimize_field_order { - inverse_memory_index.invert_bijective_mapping() - } else { - debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices())); - inverse_memory_index.into_iter().map(|it| it.index() as u32).collect() - }; let size = min_size.align_to(align); // FIXME(oli-obk): deduplicate and harden these checks if size.bytes() >= dl.obj_size_bound() { @@ -1389,8 +1375,11 @@ impl LayoutCalculator { let pair = LayoutData::::scalar_pair(&self.cx, a, b); let pair_offsets = match pair.fields { - FieldsShape::Arbitrary { ref offsets, ref memory_index } => { - assert_eq!(memory_index.raw, [0, 1]); + FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => { + assert_eq!( + in_memory_order.raw, + [FieldIdx::new(0), FieldIdx::new(1)] + ); offsets } FieldsShape::Primitive @@ -1434,7 +1423,7 @@ impl LayoutCalculator { Ok(LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, - fields: FieldsShape::Arbitrary { offsets, memory_index }, + fields: FieldsShape::Arbitrary { offsets, in_memory_order }, backend_repr: abi, largest_niche, uninhabited, @@ -1530,7 +1519,10 @@ where Ok(LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, - fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), memory_index: [0].into() }, + fields: FieldsShape::Arbitrary { + offsets: [Size::ZERO].into(), + in_memory_order: [FieldIdx::new(0)].into(), + }, backend_repr: repr, largest_niche: elt.largest_niche, uninhabited: false, diff --git a/compiler/rustc_abi/src/layout/coroutine.rs b/compiler/rustc_abi/src/layout/coroutine.rs index 2b22276d4aed7..815cf1e28a08c 100644 --- a/compiler/rustc_abi/src/layout/coroutine.rs +++ b/compiler/rustc_abi/src/layout/coroutine.rs @@ -182,33 +182,29 @@ pub(super) fn layout< // CoroutineLayout. debug!("prefix = {:#?}", prefix); let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields { - FieldsShape::Arbitrary { mut offsets, memory_index } => { - let mut inverse_memory_index = memory_index.invert_bijective_mapping(); - + FieldsShape::Arbitrary { mut offsets, in_memory_order } => { // "a" (`0..b_start`) and "b" (`b_start..`) correspond to // "outer" and "promoted" fields respectively. let b_start = tag_index.plus(1); let offsets_b = IndexVec::from_raw(offsets.raw.split_off(b_start.index())); let offsets_a = offsets; - // Disentangle the "a" and "b" components of `inverse_memory_index` + // Disentangle the "a" and "b" components of `in_memory_order` // by preserving the order but keeping only one disjoint "half" each. // FIXME(eddyb) build a better abstraction for permutations, if possible. - let inverse_memory_index_b: IndexVec = inverse_memory_index - .iter() - .filter_map(|&i| i.index().checked_sub(b_start.index()).map(FieldIdx::new)) - .collect(); - inverse_memory_index.raw.retain(|&i| i.index() < b_start.index()); - let inverse_memory_index_a = inverse_memory_index; - - // Since `inverse_memory_index_{a,b}` each only refer to their - // respective fields, they can be safely inverted - let memory_index_a = inverse_memory_index_a.invert_bijective_mapping(); - let memory_index_b = inverse_memory_index_b.invert_bijective_mapping(); + let mut in_memory_order_a = IndexVec::::new(); + let mut in_memory_order_b = IndexVec::::new(); + for i in in_memory_order { + if let Some(j) = i.index().checked_sub(b_start.index()) { + in_memory_order_b.push(FieldIdx::new(j)); + } else { + in_memory_order_a.push(i); + } + } let outer_fields = - FieldsShape::Arbitrary { offsets: offsets_a, memory_index: memory_index_a }; - (outer_fields, offsets_b, memory_index_b) + FieldsShape::Arbitrary { offsets: offsets_a, in_memory_order: in_memory_order_a }; + (outer_fields, offsets_b, in_memory_order_b.invert_bijective_mapping()) } _ => unreachable!(), }; @@ -236,7 +232,7 @@ pub(super) fn layout< )?; variant.variants = Variants::Single { index }; - let FieldsShape::Arbitrary { offsets, memory_index } = variant.fields else { + let FieldsShape::Arbitrary { offsets, in_memory_order } = variant.fields else { unreachable!(); }; @@ -249,8 +245,9 @@ pub(super) fn layout< // promoted fields were being used, but leave the elements not in the // subset as `invalid_field_idx`, which we can filter out later to // obtain a valid (bijective) mapping. + let memory_index = in_memory_order.invert_bijective_mapping(); let invalid_field_idx = promoted_memory_index.len() + memory_index.len(); - let mut combined_inverse_memory_index = + let mut combined_in_memory_order = IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx); let mut offsets_and_memory_index = iter::zip(offsets, memory_index); @@ -268,19 +265,18 @@ pub(super) fn layout< (promoted_offsets[field_idx], promoted_memory_index[field_idx]) } }; - combined_inverse_memory_index[memory_index] = i; + combined_in_memory_order[memory_index] = i; offset }) .collect(); - // Remove the unused slots and invert the mapping to obtain the - // combined `memory_index` (also see previous comment). - combined_inverse_memory_index.raw.retain(|&i| i.index() != invalid_field_idx); - let combined_memory_index = combined_inverse_memory_index.invert_bijective_mapping(); + // Remove the unused slots to obtain the combined `in_memory_order` + // (also see previous comment). + combined_in_memory_order.raw.retain(|&i| i.index() != invalid_field_idx); variant.fields = FieldsShape::Arbitrary { offsets: combined_offsets, - memory_index: combined_memory_index, + in_memory_order: combined_in_memory_order, }; size = size.max(variant.size); diff --git a/compiler/rustc_abi/src/layout/simple.rs b/compiler/rustc_abi/src/layout/simple.rs index b3807c8727396..3784611b157be 100644 --- a/compiler/rustc_abi/src/layout/simple.rs +++ b/compiler/rustc_abi/src/layout/simple.rs @@ -16,7 +16,7 @@ impl LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Arbitrary { offsets: IndexVec::new(), - memory_index: IndexVec::new(), + in_memory_order: IndexVec::new(), }, backend_repr: BackendRepr::Memory { sized }, largest_niche: None, @@ -108,7 +108,7 @@ impl LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Arbitrary { offsets: [Size::ZERO, b_offset].into(), - memory_index: [0, 1].into(), + in_memory_order: [FieldIdx::new(0), FieldIdx::new(1)].into(), }, backend_repr: BackendRepr::ScalarPair(a, b), largest_niche, @@ -133,7 +133,7 @@ impl LayoutData { Some(fields) => FieldsShape::Union(fields), None => FieldsShape::Arbitrary { offsets: IndexVec::new(), - memory_index: IndexVec::new(), + in_memory_order: IndexVec::new(), }, }, backend_repr: BackendRepr::Memory { sized: true }, diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 53b2c3f36dfc9..061ad8617893c 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -1636,19 +1636,14 @@ pub enum FieldsShape { // FIXME(eddyb) use small vector optimization for the common case. offsets: IndexVec, - /// Maps source order field indices to memory order indices, + /// Maps memory order field indices to source order indices, /// depending on how the fields were reordered (if at all). /// This is a permutation, with both the source order and the /// memory order using the same (0..n) index ranges. /// - /// Note that during computation of `memory_index`, sometimes - /// it is easier to operate on the inverse mapping (that is, - /// from memory order to source order), and that is usually - /// named `inverse_memory_index`. - /// // FIXME(eddyb) build a better abstraction for permutations, if possible. // FIXME(camlorn) also consider small vector optimization here. - memory_index: IndexVec, + in_memory_order: IndexVec, }, } @@ -1682,51 +1677,17 @@ impl FieldsShape { } } - #[inline] - pub fn memory_index(&self, i: usize) -> usize { - match *self { - FieldsShape::Primitive => { - unreachable!("FieldsShape::memory_index: `Primitive`s have no fields") - } - FieldsShape::Union(_) | FieldsShape::Array { .. } => i, - FieldsShape::Arbitrary { ref memory_index, .. } => { - memory_index[FieldIdx::new(i)].try_into().unwrap() - } - } - } - /// Gets source indices of the fields by increasing offsets. #[inline] pub fn index_by_increasing_offset(&self) -> impl ExactSizeIterator { - let mut inverse_small = [0u8; 64]; - let mut inverse_big = IndexVec::new(); - let use_small = self.count() <= inverse_small.len(); - - // We have to write this logic twice in order to keep the array small. - if let FieldsShape::Arbitrary { ref memory_index, .. } = *self { - if use_small { - for (field_idx, &mem_idx) in memory_index.iter_enumerated() { - inverse_small[mem_idx as usize] = field_idx.index() as u8; - } - } else { - inverse_big = memory_index.invert_bijective_mapping(); - } - } - // Primitives don't really have fields in the way that structs do, // but having this return an empty iterator for them is unhelpful // since that makes them look kinda like ZSTs, which they're not. let pseudofield_count = if let FieldsShape::Primitive = self { 1 } else { self.count() }; - (0..pseudofield_count).map(move |i| match *self { + (0..pseudofield_count).map(move |i| match self { FieldsShape::Primitive | FieldsShape::Union(_) | FieldsShape::Array { .. } => i, - FieldsShape::Arbitrary { .. } => { - if use_small { - inverse_small[i] as usize - } else { - inverse_big[i as u32].index() - } - } + FieldsShape::Arbitrary { in_memory_order, .. } => in_memory_order[i as u32].index(), }) } } diff --git a/compiler/rustc_builtin_macros/src/eii.rs b/compiler/rustc_builtin_macros/src/eii.rs index b97cb1daec534..e29b29fc6ccb0 100644 --- a/compiler/rustc_builtin_macros/src/eii.rs +++ b/compiler/rustc_builtin_macros/src/eii.rs @@ -1,11 +1,12 @@ use rustc_ast::token::{Delimiter, TokenKind}; use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_ast::{ - DUMMY_NODE_ID, EiiExternTarget, EiiImpl, ItemKind, Stmt, StmtKind, ast, token, tokenstream, + Attribute, DUMMY_NODE_ID, EiiExternTarget, EiiImpl, ItemKind, MetaItem, Path, Stmt, StmtKind, + Visibility, ast, }; use rustc_ast_pretty::pprust::path_to_string; use rustc_expand::base::{Annotatable, ExtCtxt}; -use rustc_span::{Ident, Span, kw, sym}; +use rustc_span::{ErrorGuaranteed, Ident, Span, kw, sym}; use thin_vec::{ThinVec, thin_vec}; use crate::errors::{ @@ -52,12 +53,12 @@ pub(crate) fn unsafe_eii( fn eii_( ecx: &mut ExtCtxt<'_>, - span: Span, + eii_attr_span: Span, meta_item: &ast::MetaItem, item: Annotatable, impl_unsafe: bool, ) -> Vec { - let span = ecx.with_def_site_ctxt(span); + let eii_attr_span = ecx.with_def_site_ctxt(eii_attr_span); let (item, stmt) = if let Annotatable::Item(item) = item { (item, false) @@ -67,7 +68,7 @@ fn eii_( (item.clone(), true) } else { ecx.dcx().emit_err(EiiSharedMacroExpectedFunction { - span, + span: eii_attr_span, name: path_to_string(&meta_item.path), }); return vec![item]; @@ -77,113 +78,218 @@ fn eii_( let item = *item; - let ast::Item { attrs, id: _, span: item_span, vis, kind: ItemKind::Fn(mut func), tokens: _ } = - item - else { + let ast::Item { attrs, id: _, span: _, vis, kind: ItemKind::Fn(func), tokens: _ } = item else { ecx.dcx().emit_err(EiiSharedMacroExpectedFunction { - span, + span: eii_attr_span, name: path_to_string(&meta_item.path), }); return vec![Annotatable::Item(Box::new(item))]; }; - // Detect when this is the *second* eii attribute on an item. - let mut new_attrs = ThinVec::new(); - for i in attrs { - if i.has_name(sym::eii) { - ecx.dcx().emit_err(EiiOnlyOnce { - span: i.span, - first_span: span, - name: path_to_string(&meta_item.path), - }); - } else { - new_attrs.push(i); - } + let attrs_from_decl = + filter_attrs_for_multiple_eii_attr(ecx, attrs, eii_attr_span, &meta_item.path); + + let Ok(macro_name) = name_for_impl_macro(ecx, &func, &meta_item) else { + return vec![Annotatable::Item(orig_item)]; + }; + + // span of the declaring item without attributes + let item_span = func.sig.span; + // span of the eii attribute and the item below it, i.e. the full declaration + let decl_span = eii_attr_span.to(item_span); + let foreign_item_name = func.ident; + + let mut return_items = Vec::new(); + + if func.body.is_some() { + return_items.push(Box::new(generate_default_impl( + &func, + impl_unsafe, + macro_name, + eii_attr_span, + item_span, + ))) + } + + return_items.push(Box::new(generate_foreign_item( + ecx, + eii_attr_span, + item_span, + *func, + vis, + &attrs_from_decl, + ))); + return_items.push(Box::new(generate_attribute_macro_to_implement( + ecx, + eii_attr_span, + macro_name, + foreign_item_name, + impl_unsafe, + decl_span, + ))); + + if stmt { + return_items + .into_iter() + .map(|i| { + Annotatable::Stmt(Box::new(Stmt { + id: DUMMY_NODE_ID, + kind: StmtKind::Item(i), + span: eii_attr_span, + })) + }) + .collect() + } else { + return_items.into_iter().map(|i| Annotatable::Item(i)).collect() } - let attrs = new_attrs; +} - let macro_name = if meta_item.is_word() { - func.ident +/// Decide on the name of the macro that can be used to implement the EII. +/// This is either an explicitly given name, or the name of the item in the +/// declaration of the EII. +fn name_for_impl_macro( + ecx: &mut ExtCtxt<'_>, + func: &ast::Fn, + meta_item: &MetaItem, +) -> Result { + if meta_item.is_word() { + Ok(func.ident) } else if let Some([first]) = meta_item.meta_item_list() && let Some(m) = first.meta_item() && m.path.segments.len() == 1 { - m.path.segments[0].ident + Ok(m.path.segments[0].ident) } else { - ecx.dcx().emit_err(EiiMacroExpectedMaxOneArgument { + Err(ecx.dcx().emit_err(EiiMacroExpectedMaxOneArgument { span: meta_item.span, name: path_to_string(&meta_item.path), - }); - return vec![Annotatable::Item(orig_item)]; - }; + })) + } +} - let mut return_items = Vec::new(); +/// Ensure that in the list of attrs, there's only a single `eii` attribute. +fn filter_attrs_for_multiple_eii_attr( + ecx: &mut ExtCtxt<'_>, + attrs: ThinVec, + eii_attr_span: Span, + eii_attr_path: &Path, +) -> ThinVec { + attrs + .into_iter() + .filter(|i| { + if i.has_name(sym::eii) { + ecx.dcx().emit_err(EiiOnlyOnce { + span: i.span, + first_span: eii_attr_span, + name: path_to_string(eii_attr_path), + }); + false + } else { + true + } + }) + .collect() +} - if func.body.is_some() { - let mut default_func = func.clone(); - func.body = None; - default_func.eii_impls.push(ast::EiiImpl { - node_id: DUMMY_NODE_ID, - eii_macro_path: ast::Path::from_ident(macro_name), - impl_safety: if impl_unsafe { ast::Safety::Unsafe(span) } else { ast::Safety::Default }, - span, - inner_span: macro_name.span, - is_default: true, // important! - }); +fn generate_default_impl( + func: &ast::Fn, + impl_unsafe: bool, + macro_name: Ident, + eii_attr_span: Span, + item_span: Span, +) -> ast::Item { + // FIXME: re-add some original attrs + let attrs = ThinVec::new(); + + let mut default_func = func.clone(); + default_func.eii_impls.push(EiiImpl { + node_id: DUMMY_NODE_ID, + inner_span: macro_name.span, + eii_macro_path: ast::Path::from_ident(macro_name), + impl_safety: if impl_unsafe { + ast::Safety::Unsafe(eii_attr_span) + } else { + ast::Safety::Default + }, + span: eii_attr_span, + is_default: true, + }); - return_items.push(Box::new(ast::Item { - attrs: ThinVec::new(), - id: ast::DUMMY_NODE_ID, - span, - vis: ast::Visibility { span, kind: ast::VisibilityKind::Inherited, tokens: None }, - kind: ast::ItemKind::Const(Box::new(ast::ConstItem { - ident: Ident { name: kw::Underscore, span }, - defaultness: ast::Defaultness::Final, - generics: ast::Generics::default(), - ty: Box::new(ast::Ty { - id: DUMMY_NODE_ID, - kind: ast::TyKind::Tup(ThinVec::new()), - span, - tokens: None, - }), - rhs: Some(ast::ConstItemRhs::Body(Box::new(ast::Expr { - id: DUMMY_NODE_ID, - kind: ast::ExprKind::Block( - Box::new(ast::Block { - stmts: thin_vec![ast::Stmt { - id: DUMMY_NODE_ID, - kind: ast::StmtKind::Item(Box::new(ast::Item { - attrs: thin_vec![], // FIXME: re-add some original attrs - id: DUMMY_NODE_ID, - span: item_span, - vis: ast::Visibility { - span, - kind: ast::VisibilityKind::Inherited, - tokens: None - }, - kind: ItemKind::Fn(default_func), - tokens: None, - })), - span - }], - id: DUMMY_NODE_ID, - rules: ast::BlockCheckMode::Default, - span, - tokens: None, - }), - None, - ), - span, - attrs: ThinVec::new(), - tokens: None, - }))), - define_opaque: None, - })), + ast::Item { + attrs: ThinVec::new(), + id: ast::DUMMY_NODE_ID, + span: eii_attr_span, + vis: ast::Visibility { + span: eii_attr_span, + kind: ast::VisibilityKind::Inherited, tokens: None, - })) + }, + kind: ast::ItemKind::Const(Box::new(ast::ConstItem { + ident: Ident { name: kw::Underscore, span: eii_attr_span }, + defaultness: ast::Defaultness::Final, + generics: ast::Generics::default(), + ty: Box::new(ast::Ty { + id: DUMMY_NODE_ID, + kind: ast::TyKind::Tup(ThinVec::new()), + span: eii_attr_span, + tokens: None, + }), + rhs: Some(ast::ConstItemRhs::Body(Box::new(ast::Expr { + id: DUMMY_NODE_ID, + kind: ast::ExprKind::Block( + Box::new(ast::Block { + stmts: thin_vec![ast::Stmt { + id: DUMMY_NODE_ID, + kind: ast::StmtKind::Item(Box::new(ast::Item { + attrs, + id: DUMMY_NODE_ID, + span: item_span, + vis: ast::Visibility { + span: eii_attr_span, + kind: ast::VisibilityKind::Inherited, + tokens: None + }, + kind: ItemKind::Fn(Box::new(default_func)), + tokens: None, + })), + span: eii_attr_span + }], + id: DUMMY_NODE_ID, + rules: ast::BlockCheckMode::Default, + span: eii_attr_span, + tokens: None, + }), + None, + ), + span: eii_attr_span, + attrs: ThinVec::new(), + tokens: None, + }))), + define_opaque: None, + })), + tokens: None, } +} - let decl_span = span.to(func.sig.span); +/// Generates a foreign item, like +/// +/// ```rust, ignore +/// extern "…" { safe fn item(); } +/// ``` +fn generate_foreign_item( + ecx: &mut ExtCtxt<'_>, + eii_attr_span: Span, + item_span: Span, + mut func: ast::Fn, + vis: Visibility, + attrs_from_decl: &[Attribute], +) -> ast::Item { + let mut foreign_item_attrs = ThinVec::new(); + foreign_item_attrs.extend_from_slice(attrs_from_decl); + + // Add the rustc_eii_extern_item on the foreign item. Usually, foreign items are mangled. + // This attribute makes sure that we later know that this foreign item's symbol should not be. + foreign_item_attrs.push(ecx.attr_word(sym::rustc_eii_extern_item, eii_attr_span)); let abi = match func.sig.header.ext { // extern "X" fn => extern "X" {} @@ -196,85 +302,69 @@ fn eii_( suffix: None, symbol_unescaped: sym::Rust, style: ast::StrStyle::Cooked, - span, + span: eii_attr_span, }), }; // ABI has been moved to the extern {} block, so we remove it from the fn item. func.sig.header.ext = ast::Extern::None; + func.body = None; // And mark safe functions explicitly as `safe fn`. if func.sig.header.safety == ast::Safety::Default { func.sig.header.safety = ast::Safety::Safe(func.sig.span); } - // extern "…" { safe fn item(); } - let mut extern_item_attrs = attrs.clone(); - extern_item_attrs.push(ast::Attribute { - kind: ast::AttrKind::Normal(Box::new(ast::NormalAttr { - item: ast::AttrItem { - unsafety: ast::Safety::Default, - // Add the rustc_eii_extern_item on the foreign item. Usually, foreign items are mangled. - // This attribute makes sure that we later know that this foreign item's symbol should not be. - path: ast::Path::from_ident(Ident::new(sym::rustc_eii_extern_item, span)), - args: ast::AttrArgs::Empty, - tokens: None, - }, - tokens: None, - })), - id: ecx.sess.psess.attr_id_generator.mk_attr_id(), - style: ast::AttrStyle::Outer, - span, - }); - - let extern_block = Box::new(ast::Item { + ast::Item { attrs: ast::AttrVec::default(), id: ast::DUMMY_NODE_ID, - span, - vis: ast::Visibility { span, kind: ast::VisibilityKind::Inherited, tokens: None }, + span: eii_attr_span, + vis: ast::Visibility { + span: eii_attr_span, + kind: ast::VisibilityKind::Inherited, + tokens: None, + }, kind: ast::ItemKind::ForeignMod(ast::ForeignMod { - extern_span: span, - safety: ast::Safety::Unsafe(span), + extern_span: eii_attr_span, + safety: ast::Safety::Unsafe(eii_attr_span), abi, items: From::from([Box::new(ast::ForeignItem { - attrs: extern_item_attrs, + attrs: foreign_item_attrs, id: ast::DUMMY_NODE_ID, span: item_span, vis, - kind: ast::ForeignItemKind::Fn(func.clone()), + kind: ast::ForeignItemKind::Fn(Box::new(func.clone())), tokens: None, })]), }), tokens: None, - }); + } +} - let mut macro_attrs = attrs.clone(); - macro_attrs.push( - // #[builtin_macro(eii_shared_macro)] - ast::Attribute { - kind: ast::AttrKind::Normal(Box::new(ast::NormalAttr { - item: ast::AttrItem { - unsafety: ast::Safety::Default, - path: ast::Path::from_ident(Ident::new(sym::rustc_builtin_macro, span)), - args: ast::AttrArgs::Delimited(ast::DelimArgs { - dspan: DelimSpan::from_single(span), - delim: Delimiter::Parenthesis, - tokens: TokenStream::new(vec![tokenstream::TokenTree::token_alone( - token::TokenKind::Ident(sym::eii_shared_macro, token::IdentIsRaw::No), - span, - )]), - }), - tokens: None, - }, - tokens: None, - })), - id: ecx.sess.psess.attr_id_generator.mk_attr_id(), - style: ast::AttrStyle::Outer, - span, - }, - ); +/// Generate a stub macro (a bit like in core) that will roughly look like: +/// +/// ```rust, ignore, example +/// // Since this a stub macro, the actual code that expands it lives in the compiler. +/// // This attribute tells the compiler that +/// #[builtin_macro(eii_shared_macro)] +/// // the metadata to link this macro to the generated foreign item. +/// #[eii_extern_target()] +/// macro macro_name { () => {} } +/// ``` +fn generate_attribute_macro_to_implement( + ecx: &mut ExtCtxt<'_>, + span: Span, + macro_name: Ident, + foreign_item_name: Ident, + impl_unsafe: bool, + decl_span: Span, +) -> ast::Item { + let mut macro_attrs = ThinVec::new(); + + // #[builtin_macro(eii_shared_macro)] + macro_attrs.push(ecx.attr_nested_word(sym::rustc_builtin_macro, sym::eii_shared_macro, span)); - let macro_def = Box::new(ast::Item { + ast::Item { attrs: macro_attrs, id: ast::DUMMY_NODE_ID, span, @@ -305,33 +395,15 @@ fn eii_( ]), }), macro_rules: false, - // #[eii_extern_target(func.ident)] + // #[eii_extern_target(foreign_item_ident)] eii_extern_target: Some(ast::EiiExternTarget { - extern_item_path: ast::Path::from_ident(func.ident), + extern_item_path: ast::Path::from_ident(foreign_item_name), impl_unsafe, span: decl_span, }), }, ), tokens: None, - }); - - return_items.push(extern_block); - return_items.push(macro_def); - - if stmt { - return_items - .into_iter() - .map(|i| { - Annotatable::Stmt(Box::new(Stmt { - id: DUMMY_NODE_ID, - kind: StmtKind::Item(i), - span, - })) - }) - .collect() - } else { - return_items.into_iter().map(|i| Annotatable::Item(i)).collect() } } @@ -436,10 +508,10 @@ pub(crate) fn eii_shared_macro( f.eii_impls.push(EiiImpl { node_id: DUMMY_NODE_ID, + inner_span: meta_item.path.span, eii_macro_path: meta_item.path.clone(), impl_safety: meta_item.unsafety, span, - inner_span: meta_item.path.span, is_default, }); diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs index 2878fa7aa298a..65779b38ad1c0 100644 --- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs +++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs @@ -10,7 +10,7 @@ //! function u0:22(i64) -> i8, i8 system_v { //! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd361e9f5c3d1c4deE //! ; instance Instance { def: Item(DefId(0:42 ~ example[3895]::{impl#0}::call_once)), args: ['{erased}, '{erased}] } -//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } +//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], in_memory_order: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], in_memory_order: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } //! //! ; kind loc.idx param pass mode ty //! ; ssa _0 (u8, u8) 2b 1 var=(0, 1) @@ -41,7 +41,7 @@ //! ; //! ; _0 = >::call_mut(move _3, copy _2) //! v2 = stack_load.i64 ss0 -//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } +//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], in_memory_order: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } //! v3, v4 = call fn0(v1, v2) ; v1 = 1 //! v5 -> v3 //! v6 -> v4 diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 18da945f5a318..1afb087178f1b 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -707,8 +707,7 @@ pub(crate) unsafe fn llvm_optimize( if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Enable) { let cx = SimpleCx::new(module.module_llvm.llmod(), module.module_llvm.llcx, cgcx.pointer_size); - // For now we only support up to 10 kernels named kernel_0 ... kernel_9, a follow-up PR is - // introducing a proper offload intrinsic to solve this limitation. + for func in cx.get_functions() { let offload_kernel = "offload-kernel"; if attributes::has_string_attr(func, offload_kernel) { diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 6cbddfec46318..16455b4c79cd6 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -23,13 +23,14 @@ use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrs, SanitizerFnAttrs}; use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::TyCtxt; -use rustc_session::config::DebugInfo; +use rustc_session::config::{DebugInfo, Offload}; use rustc_span::Symbol; use rustc_target::spec::SanitizerSet; use super::ModuleLlvm; use crate::attributes; use crate::builder::Builder; +use crate::builder::gpu_offload::OffloadGlobals; use crate::context::CodegenCx; use crate::llvm::{self, Value}; @@ -85,6 +86,19 @@ pub(crate) fn compile_codegen_unit( let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str()); { let mut cx = CodegenCx::new(tcx, cgu, &llvm_module); + + // Declare and store globals shared by all offload kernels + // + // These globals are left in the LLVM-IR host module so all kernels can access them. + // They are necessary for correct offload execution. We do this here to simplify the + // `offload` intrinsic, avoiding the need for tracking whether it's the first + // intrinsic call or not. + if cx.sess().opts.unstable_opts.offload.contains(&Offload::Enable) + && !cx.sess().target.is_like_gpu + { + cx.offload_globals.replace(Some(OffloadGlobals::declare(&cx))); + } + let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); for &(mono_item, data) in &mono_items { mono_item.predefine::>( diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index 5d1ddd057d88a..046501d08c482 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -2,17 +2,76 @@ use std::ffi::CString; use llvm::Linkage::*; use rustc_abi::Align; -use rustc_codegen_ssa::traits::BaseTypeCodegenMethods; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; use rustc_middle::ty::offload_meta::OffloadMetadata; -use crate::builder::SBuilder; +use crate::builder::Builder; +use crate::common::CodegenCx; use crate::llvm::AttributePlace::Function; -use crate::llvm::{self, BasicBlock, Linkage, Type, Value}; +use crate::llvm::{self, Linkage, Type, Value}; use crate::{SimpleCx, attributes}; +// LLVM kernel-independent globals required for offloading +pub(crate) struct OffloadGlobals<'ll> { + pub launcher_fn: &'ll llvm::Value, + pub launcher_ty: &'ll llvm::Type, + + pub bin_desc: &'ll llvm::Type, + + pub kernel_args_ty: &'ll llvm::Type, + + pub offload_entry_ty: &'ll llvm::Type, + + pub begin_mapper: &'ll llvm::Value, + pub end_mapper: &'ll llvm::Value, + pub mapper_fn_ty: &'ll llvm::Type, + + pub ident_t_global: &'ll llvm::Value, + + pub register_lib: &'ll llvm::Value, + pub unregister_lib: &'ll llvm::Value, + pub init_rtls: &'ll llvm::Value, +} + +impl<'ll> OffloadGlobals<'ll> { + pub(crate) fn declare(cx: &CodegenCx<'ll, '_>) -> Self { + let (launcher_fn, launcher_ty) = generate_launcher(cx); + let kernel_args_ty = KernelArgsTy::new_decl(cx); + let offload_entry_ty = TgtOffloadEntry::new_decl(cx); + let (begin_mapper, _, end_mapper, mapper_fn_ty) = gen_tgt_data_mappers(cx); + let ident_t_global = generate_at_one(cx); + + let tptr = cx.type_ptr(); + let ti32 = cx.type_i32(); + let tgt_bin_desc_ty = vec![ti32, tptr, tptr, tptr]; + let bin_desc = cx.type_named_struct("struct.__tgt_bin_desc"); + cx.set_struct_body(bin_desc, &tgt_bin_desc_ty, false); + + let register_lib = declare_offload_fn(&cx, "__tgt_register_lib", mapper_fn_ty); + let unregister_lib = declare_offload_fn(&cx, "__tgt_unregister_lib", mapper_fn_ty); + let init_ty = cx.type_func(&[], cx.type_void()); + let init_rtls = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty); + + OffloadGlobals { + launcher_fn, + launcher_ty, + bin_desc, + kernel_args_ty, + offload_entry_ty, + begin_mapper, + end_mapper, + mapper_fn_ty, + ident_t_global, + register_lib, + unregister_lib, + init_rtls, + } + } +} + // ; Function Attrs: nounwind // declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr) #2 -fn generate_launcher<'ll>(cx: &'ll SimpleCx<'_>) -> (&'ll llvm::Value, &'ll llvm::Type) { +fn generate_launcher<'ll>(cx: &CodegenCx<'ll, '_>) -> (&'ll llvm::Value, &'ll llvm::Type) { let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); let ti32 = cx.type_i32(); @@ -30,7 +89,7 @@ fn generate_launcher<'ll>(cx: &'ll SimpleCx<'_>) -> (&'ll llvm::Value, &'ll llvm // @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @0 }, align 8 // FIXME(offload): @0 should include the file name (e.g. lib.rs) in which the function to be // offloaded was defined. -fn generate_at_one<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Value { +pub(crate) fn generate_at_one<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll llvm::Value { let unknown_txt = ";unknown;unknown;0;0;;"; let c_entry_name = CString::new(unknown_txt).unwrap(); let c_val = c_entry_name.as_bytes_with_nul(); @@ -68,7 +127,7 @@ pub(crate) struct TgtOffloadEntry { } impl TgtOffloadEntry { - pub(crate) fn new_decl<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Type { + pub(crate) fn new_decl<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll llvm::Type { let offload_entry_ty = cx.type_named_struct("struct.__tgt_offload_entry"); let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); @@ -82,7 +141,7 @@ impl TgtOffloadEntry { } fn new<'ll>( - cx: &'ll SimpleCx<'_>, + cx: &CodegenCx<'ll, '_>, region_id: &'ll Value, llglobal: &'ll Value, ) -> [&'ll Value; 9] { @@ -126,7 +185,7 @@ impl KernelArgsTy { const OFFLOAD_VERSION: u64 = 3; const FLAGS: u64 = 0; const TRIPCOUNT: u64 = 0; - fn new_decl<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll Type { + fn new_decl<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Type { let kernel_arguments_ty = cx.type_named_struct("struct.__tgt_kernel_arguments"); let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); @@ -140,8 +199,8 @@ impl KernelArgsTy { kernel_arguments_ty } - fn new<'ll>( - cx: &'ll SimpleCx<'_>, + fn new<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, num_args: u64, memtransfer_types: &'ll Value, geps: [&'ll Value; 3], @@ -171,7 +230,8 @@ impl KernelArgsTy { } // Contains LLVM values needed to manage offloading for a single kernel. -pub(crate) struct OffloadKernelData<'ll> { +#[derive(Copy, Clone)] +pub(crate) struct OffloadKernelGlobals<'ll> { pub offload_sizes: &'ll llvm::Value, pub memtransfer_types: &'ll llvm::Value, pub region_id: &'ll llvm::Value, @@ -179,7 +239,7 @@ pub(crate) struct OffloadKernelData<'ll> { } fn gen_tgt_data_mappers<'ll>( - cx: &'ll SimpleCx<'_>, + cx: &CodegenCx<'ll, '_>, ) -> (&'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Type) { let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); @@ -241,12 +301,18 @@ pub(crate) fn add_global<'ll>( // mapped to/from the gpu. It also returns a region_id with the name of this kernel, to be // concatenated into the list of region_ids. pub(crate) fn gen_define_handling<'ll>( - cx: &SimpleCx<'ll>, - offload_entry_ty: &'ll llvm::Type, + cx: &CodegenCx<'ll, '_>, metadata: &[OffloadMetadata], - types: &[&Type], - symbol: &str, -) -> OffloadKernelData<'ll> { + types: &[&'ll Type], + symbol: String, + offload_globals: &OffloadGlobals<'ll>, +) -> OffloadKernelGlobals<'ll> { + if let Some(entry) = cx.offload_kernel_cache.borrow().get(&symbol) { + return *entry; + } + + let offload_entry_ty = offload_globals.offload_entry_ty; + // It seems like non-pointer values are automatically mapped. So here, we focus on pointer (or // reference) types. let ptr_meta = types.iter().zip(metadata).filter_map(|(&x, meta)| match cx.type_kind(x) { @@ -272,9 +338,9 @@ pub(crate) fn gen_define_handling<'ll>( let name = format!(".{symbol}.region_id"); let initializer = cx.get_const_i8(0); - let region_id = add_unnamed_global(&cx, &name, initializer, WeakAnyLinkage); + let region_id = add_global(&cx, &name, initializer, WeakAnyLinkage); - let c_entry_name = CString::new(symbol).unwrap(); + let c_entry_name = CString::new(symbol.clone()).unwrap(); let c_val = c_entry_name.as_bytes_with_nul(); let offload_entry_name = format!(".offloading.entry_name.{symbol}"); @@ -298,11 +364,16 @@ pub(crate) fn gen_define_handling<'ll>( let c_section_name = CString::new("llvm_offload_entries").unwrap(); llvm::set_section(offload_entry, &c_section_name); - OffloadKernelData { offload_sizes, memtransfer_types, region_id, offload_entry } + let result = + OffloadKernelGlobals { offload_sizes, memtransfer_types, region_id, offload_entry }; + + cx.offload_kernel_cache.borrow_mut().insert(symbol, result); + + result } fn declare_offload_fn<'ll>( - cx: &'ll SimpleCx<'_>, + cx: &CodegenCx<'ll, '_>, name: &str, ty: &'ll llvm::Type, ) -> &'ll llvm::Value { @@ -335,28 +406,28 @@ fn declare_offload_fn<'ll>( // 4. set insert point after kernel call. // 5. generate all the GEPS and stores, to be used in 6) // 6. generate __tgt_target_data_end calls to move data from the GPU -pub(crate) fn gen_call_handling<'ll>( - cx: &SimpleCx<'ll>, - bb: &BasicBlock, - offload_data: &OffloadKernelData<'ll>, +pub(crate) fn gen_call_handling<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, + offload_data: &OffloadKernelGlobals<'ll>, args: &[&'ll Value], types: &[&Type], metadata: &[OffloadMetadata], + offload_globals: &OffloadGlobals<'ll>, ) { - let OffloadKernelData { offload_sizes, offload_entry, memtransfer_types, region_id } = + let cx = builder.cx; + let OffloadKernelGlobals { offload_sizes, offload_entry, memtransfer_types, region_id } = offload_data; - let (tgt_decl, tgt_target_kernel_ty) = generate_launcher(&cx); - // %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } - let tptr = cx.type_ptr(); - let ti32 = cx.type_i32(); - let tgt_bin_desc_ty = vec![ti32, tptr, tptr, tptr]; - let tgt_bin_desc = cx.type_named_struct("struct.__tgt_bin_desc"); - cx.set_struct_body(tgt_bin_desc, &tgt_bin_desc_ty, false); - let tgt_kernel_decl = KernelArgsTy::new_decl(&cx); - let (begin_mapper_decl, _, end_mapper_decl, fn_ty) = gen_tgt_data_mappers(&cx); + let tgt_decl = offload_globals.launcher_fn; + let tgt_target_kernel_ty = offload_globals.launcher_ty; - let mut builder = SBuilder::build(cx, bb); + // %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } + let tgt_bin_desc = offload_globals.bin_desc; + + let tgt_kernel_decl = offload_globals.kernel_args_ty; + let begin_mapper_decl = offload_globals.begin_mapper; + let end_mapper_decl = offload_globals.end_mapper; + let fn_ty = offload_globals.mapper_fn_ty; let num_args = types.len() as u64; let ip = unsafe { llvm::LLVMRustGetInsertPoint(&builder.llbuilder) }; @@ -378,9 +449,8 @@ pub(crate) fn gen_call_handling<'ll>( // Step 0) // %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } // %6 = alloca %struct.__tgt_bin_desc, align 8 - let llfn = unsafe { llvm::LLVMGetBasicBlockParent(bb) }; unsafe { - llvm::LLVMRustPositionBuilderPastAllocas(&builder.llbuilder, llfn); + llvm::LLVMRustPositionBuilderPastAllocas(&builder.llbuilder, builder.llfn()); } let tgt_bin_desc_alloca = builder.direct_alloca(tgt_bin_desc, Align::EIGHT, "EmptyDesc"); @@ -413,16 +483,16 @@ pub(crate) fn gen_call_handling<'ll>( } let mapper_fn_ty = cx.type_func(&[cx.type_ptr()], cx.type_void()); - let register_lib_decl = declare_offload_fn(&cx, "__tgt_register_lib", mapper_fn_ty); - let unregister_lib_decl = declare_offload_fn(&cx, "__tgt_unregister_lib", mapper_fn_ty); + let register_lib_decl = offload_globals.register_lib; + let unregister_lib_decl = offload_globals.unregister_lib; let init_ty = cx.type_func(&[], cx.type_void()); - let init_rtls_decl = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty); + let init_rtls_decl = offload_globals.init_rtls; // FIXME(offload): Later we want to add them to the wrapper code, rather than our main function. // call void @__tgt_register_lib(ptr noundef %6) - builder.call(mapper_fn_ty, register_lib_decl, &[tgt_bin_desc_alloca], None); + builder.call(mapper_fn_ty, None, None, register_lib_decl, &[tgt_bin_desc_alloca], None, None); // call void @__tgt_init_all_rtls() - builder.call(init_ty, init_rtls_decl, &[], None); + builder.call(init_ty, None, None, init_rtls_decl, &[], None, None); for i in 0..num_args { let idx = cx.get_const_i32(i); @@ -437,15 +507,15 @@ pub(crate) fn gen_call_handling<'ll>( // For now we have a very simplistic indexing scheme into our // offload_{baseptrs,ptrs,sizes}. We will probably improve this along with our gpu frontend pr. - fn get_geps<'a, 'll>( - builder: &mut SBuilder<'a, 'll>, - cx: &'ll SimpleCx<'ll>, + fn get_geps<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, ty: &'ll Type, ty2: &'ll Type, a1: &'ll Value, a2: &'ll Value, a4: &'ll Value, ) -> [&'ll Value; 3] { + let cx = builder.cx; let i32_0 = cx.get_const_i32(0); let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, i32_0]); @@ -454,9 +524,8 @@ pub(crate) fn gen_call_handling<'ll>( [gep1, gep2, gep3] } - fn generate_mapper_call<'a, 'll>( - builder: &mut SBuilder<'a, 'll>, - cx: &'ll SimpleCx<'ll>, + fn generate_mapper_call<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, geps: [&'ll Value; 3], o_type: &'ll Value, fn_to_call: &'ll Value, @@ -464,20 +533,20 @@ pub(crate) fn gen_call_handling<'ll>( num_args: u64, s_ident_t: &'ll Value, ) { + let cx = builder.cx; let nullptr = cx.const_null(cx.type_ptr()); let i64_max = cx.get_const_i64(u64::MAX); let num_args = cx.get_const_i32(num_args); let args = vec![s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type, nullptr, nullptr]; - builder.call(fn_ty, fn_to_call, &args, None); + builder.call(fn_ty, None, None, fn_to_call, &args, None, None); } // Step 2) - let s_ident_t = generate_at_one(&cx); - let geps = get_geps(&mut builder, &cx, ty, ty2, a1, a2, a4); + let s_ident_t = offload_globals.ident_t_global; + let geps = get_geps(builder, ty, ty2, a1, a2, a4); generate_mapper_call( - &mut builder, - &cx, + builder, geps, memtransfer_types, begin_mapper_decl, @@ -504,14 +573,13 @@ pub(crate) fn gen_call_handling<'ll>( region_id, a5, ]; - builder.call(tgt_target_kernel_ty, tgt_decl, &args, None); + builder.call(tgt_target_kernel_ty, None, None, tgt_decl, &args, None, None); // %41 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args) // Step 4) - let geps = get_geps(&mut builder, &cx, ty, ty2, a1, a2, a4); + let geps = get_geps(builder, ty, ty2, a1, a2, a4); generate_mapper_call( - &mut builder, - &cx, + builder, geps, memtransfer_types, end_mapper_decl, @@ -520,7 +588,5 @@ pub(crate) fn gen_call_handling<'ll>( s_ident_t, ); - builder.call(mapper_fn_ty, unregister_lib_decl, &[tgt_bin_desc_alloca], None); - - drop(builder); + builder.call(mapper_fn_ty, None, None, unregister_lib_decl, &[tgt_bin_desc_alloca], None, None); } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 6caf60e3cc41e..03da70d48f7d6 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -35,6 +35,7 @@ use smallvec::SmallVec; use crate::abi::to_llvm_calling_convention; use crate::back::write::to_llvm_code_model; +use crate::builder::gpu_offload::{OffloadGlobals, OffloadKernelGlobals}; use crate::callee::get_fn; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; use crate::llvm::{self, Metadata, MetadataKindId, Module, Type, Value}; @@ -156,6 +157,12 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// Cache of Objective-C selector references pub objc_selrefs: RefCell>, + + /// Globals shared by the offloading runtime + pub offload_globals: RefCell>>, + + /// Cache of kernel-specific globals + pub offload_kernel_cache: RefCell>>, } fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { @@ -639,6 +646,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { objc_class_t: Cell::new(None), objc_classrefs: Default::default(), objc_selrefs: Default::default(), + offload_globals: Default::default(), + offload_kernel_cache: Default::default(), }, PhantomData, ) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 82615d4a160ed..3bc890310cc87 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -26,7 +26,7 @@ use tracing::debug; use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; -use crate::builder::gpu_offload::TgtOffloadEntry; +use crate::builder::gpu_offload::{gen_call_handling, gen_define_handling}; use crate::context::CodegenCx; use crate::errors::{ AutoDiffWithoutEnable, AutoDiffWithoutLto, OffloadWithoutEnable, OffloadWithoutFatLTO, @@ -1295,8 +1295,6 @@ fn codegen_offload<'ll, 'tcx>( let args = get_args_from_tuple(bx, args[1], fn_target); let target_symbol = symbol_name_for_instance_in_crate(tcx, fn_target, LOCAL_CRATE); - let offload_entry_ty = TgtOffloadEntry::new_decl(&cx); - let sig = tcx.fn_sig(fn_target.def_id()).skip_binder().skip_binder(); let inputs = sig.inputs(); @@ -1304,17 +1302,16 @@ fn codegen_offload<'ll, 'tcx>( let types = inputs.iter().map(|ty| cx.layout_of(*ty).llvm_type(cx)).collect::>(); - let offload_data = crate::builder::gpu_offload::gen_define_handling( - cx, - offload_entry_ty, - &metadata, - &types, - &target_symbol, - ); - - // FIXME(Sa4dUs): pass the original builder once we separate kernel launch logic from globals - let bb = unsafe { llvm::LLVMGetInsertBlock(bx.llbuilder) }; - crate::builder::gpu_offload::gen_call_handling(cx, bb, &offload_data, &args, &types, &metadata); + let offload_globals_ref = cx.offload_globals.borrow(); + let offload_globals = match offload_globals_ref.as_ref() { + Some(globals) => globals, + None => { + // Offload is not initialized, cannot continue + return; + } + }; + let offload_data = gen_define_handling(&cx, &metadata, &types, target_symbol, offload_globals); + gen_call_handling(bx, &offload_data, &args, &types, &metadata, offload_globals); } fn get_args_from_tuple<'ll, 'tcx>( diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs index b5de10c7dcd11..95106c7c143b3 100644 --- a/compiler/rustc_const_eval/src/interpret/visitor.rs +++ b/compiler/rustc_const_eval/src/interpret/visitor.rs @@ -4,7 +4,7 @@ use std::num::NonZero; use rustc_abi::{FieldIdx, FieldsShape, VariantIdx, Variants}; -use rustc_index::IndexVec; +use rustc_index::{Idx as _, IndexVec}; use rustc_middle::mir::interpret::InterpResult; use rustc_middle::ty::{self, Ty}; use tracing::trace; @@ -27,13 +27,15 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { /// This function provides the chance to reorder the order in which fields are visited for /// `FieldsShape::Aggregate`. /// - /// The default means we iterate in source declaration order; alternatively this can do some - /// work with `memory_index` to iterate in memory order. + /// The default means we iterate in source declaration order; alternatively this can use + /// `in_memory_order` to iterate in memory order. #[inline(always)] fn aggregate_field_iter( - memory_index: &IndexVec, - ) -> impl Iterator + 'static { - memory_index.indices() + in_memory_order: &IndexVec, + ) -> impl Iterator { + // Allow the optimizer to elide the bounds checking when creating each index. + let _ = FieldIdx::new(in_memory_order.len()); + (0..in_memory_order.len()).map(FieldIdx::new) } // Recursive actions, ready to be overloaded. @@ -168,8 +170,8 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { &FieldsShape::Union(fields) => { self.visit_union(v, fields)?; } - FieldsShape::Arbitrary { memory_index, .. } => { - for idx in Self::aggregate_field_iter(memory_index) { + FieldsShape::Arbitrary { in_memory_order, .. } => { + for idx in Self::aggregate_field_iter(in_memory_order) { let field = self.ecx().project_field(v, idx)?; self.visit_field(v, idx.as_usize(), &field)?; } diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index 0b930dc342f26..a7e8515e415f0 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -1123,7 +1123,7 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ template!(Word, List: &[r#""...""#]), DuplicatesOk, EncodeCrossCrate::Yes, ), - rustc_attr!( + rustc_attr!( rustc_offload_kernel, Normal, template!(Word), DuplicatesOk, EncodeCrossCrate::Yes, diff --git a/compiler/rustc_index/src/slice.rs b/compiler/rustc_index/src/slice.rs index d2702bdb0571d..415fe370b702c 100644 --- a/compiler/rustc_index/src/slice.rs +++ b/compiler/rustc_index/src/slice.rs @@ -181,9 +181,6 @@ impl IndexSlice { /// Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`, /// assuming the values in `self` are a permutation of `0..self.len()`. /// - /// This is used to go between `memory_index` (source field order to memory order) - /// and `inverse_memory_index` (memory order to source field order). - /// See also `FieldsShape::Arbitrary::memory_index` for more details. // FIXME(eddyb) build a better abstraction for permutations, if possible. pub fn invert_bijective_mapping(&self) -> IndexVec { debug_assert_eq!( diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index 49b2e0c1ff1a7..3d641905d3251 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -241,12 +241,14 @@ bitflags::bitflags! { } impl Encodable for RemapPathScopeComponents { + #[inline] fn encode(&self, s: &mut E) { s.emit_u8(self.bits()); } } impl Decodable for RemapPathScopeComponents { + #[inline] fn decode(s: &mut D) -> RemapPathScopeComponents { RemapPathScopeComponents::from_bits(s.read_u8()) .expect("invalid bits for RemapPathScopeComponents") @@ -308,12 +310,13 @@ struct InnerRealFileName { } impl Hash for RealFileName { + #[inline] fn hash(&self, state: &mut H) { // To prevent #70924 from happening again we should only hash the // remapped path if that exists. This is because remapped paths to // sysroot crates (/rust/$hash or /rust/$version) remain stable even // if the corresponding local path changes. - if !self.scopes.is_all() { + if !self.was_fully_remapped() { self.local.hash(state); } self.maybe_remapped.hash(state); @@ -327,6 +330,7 @@ impl RealFileName { /// ## Panic /// /// Only one scope components can be given to this function. + #[inline] pub fn path(&self, scope: RemapPathScopeComponents) -> &Path { assert!( scope.bits().count_ones() == 1, @@ -351,6 +355,7 @@ impl RealFileName { /// ## Panic /// /// Only one scope components can be given to this function. + #[inline] pub fn embeddable_name(&self, scope: RemapPathScopeComponents) -> (&Path, &Path) { assert!( scope.bits().count_ones() == 1, @@ -369,26 +374,58 @@ impl RealFileName { /// if this information exists. /// /// May not exists if the filename was imported from another crate. + /// + /// Avoid embedding this in build artifacts; prefer `path()` or `embeddable_name()`. + #[inline] pub fn local_path(&self) -> Option<&Path> { - self.local.as_ref().map(|lp| lp.name.as_ref()) + if self.was_not_remapped() { + Some(&self.maybe_remapped.name) + } else if let Some(local) = &self.local { + Some(&local.name) + } else { + None + } } /// Returns the path suitable for reading from the file system on the local host, /// if this information exists. /// /// May not exists if the filename was imported from another crate. + /// + /// Avoid embedding this in build artifacts; prefer `path()` or `embeddable_name()`. + #[inline] pub fn into_local_path(self) -> Option { - self.local.map(|lp| lp.name) + if self.was_not_remapped() { + Some(self.maybe_remapped.name) + } else if let Some(local) = self.local { + Some(local.name) + } else { + None + } } /// Returns whenever the filename was remapped. + #[inline] pub(crate) fn was_remapped(&self) -> bool { !self.scopes.is_empty() } + /// Returns whenever the filename was fully remapped. + #[inline] + fn was_fully_remapped(&self) -> bool { + self.scopes.is_all() + } + + /// Returns whenever the filename was not remapped. + #[inline] + fn was_not_remapped(&self) -> bool { + self.scopes.is_empty() + } + /// Returns an empty `RealFileName` /// /// Useful as the working directory input to `SourceMap::to_real_filename`. + #[inline] pub fn empty() -> RealFileName { RealFileName { local: Some(InnerRealFileName { @@ -420,9 +457,14 @@ impl RealFileName { /// Update the filename for encoding in the crate metadata. /// /// Currently it's about removing the local part when the filename - /// is fully remapped. + /// is either fully remapped or not remapped at all. + #[inline] pub fn update_for_crate_metadata(&mut self) { - if self.scopes.is_all() { + if self.was_fully_remapped() || self.was_not_remapped() { + // NOTE: This works because when the filename is fully + // remapped, we don't care about the `local` part, + // and when the filename is not remapped at all, + // `maybe_remapped` and `local` are equal. self.local = None; } } @@ -529,6 +571,7 @@ impl FileName { /// if this information exists. /// /// Avoid embedding this in build artifacts. Prefer using the `display` method. + #[inline] pub fn prefer_remapped_unconditionally(&self) -> FileNameDisplay<'_> { FileNameDisplay { inner: self, display_pref: FileNameDisplayPreference::Remapped } } @@ -537,16 +580,19 @@ impl FileName { /// if this information exists. /// /// Avoid embedding this in build artifacts. Prefer using the `display` method. + #[inline] pub fn prefer_local_unconditionally(&self) -> FileNameDisplay<'_> { FileNameDisplay { inner: self, display_pref: FileNameDisplayPreference::Local } } /// Returns a short (either the filename or an empty string). + #[inline] pub fn short(&self) -> FileNameDisplay<'_> { FileNameDisplay { inner: self, display_pref: FileNameDisplayPreference::Short } } /// Returns a `Display`-able path for the given scope. + #[inline] pub fn display(&self, scope: RemapPathScopeComponents) -> FileNameDisplay<'_> { FileNameDisplay { inner: self, display_pref: FileNameDisplayPreference::Scope(scope) } } diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs index 05b24d7109481..0078866ab9503 100644 --- a/compiler/rustc_target/src/asm/mod.rs +++ b/compiler/rustc_target/src/asm/mod.rs @@ -997,8 +997,8 @@ impl InlineAsmClobberAbi { _ => Err(&["C", "system"]), }, InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => match name { - "C" | "system" => Ok(InlineAsmClobberAbi::LoongArch), - _ => Err(&["C", "system"]), + "C" | "system" | "efiapi" => Ok(InlineAsmClobberAbi::LoongArch), + _ => Err(&["C", "system", "efiapi"]), }, InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => match name { "C" | "system" => Ok(if target.abi == Abi::Spe { diff --git a/compiler/rustc_target/src/spec/abi_map.rs b/compiler/rustc_target/src/spec/abi_map.rs index 8126227d80e48..d7fc18cd3761e 100644 --- a/compiler/rustc_target/src/spec/abi_map.rs +++ b/compiler/rustc_target/src/spec/abi_map.rs @@ -56,6 +56,7 @@ impl AbiMap { ArmVer::Other }), Arch::Avr => ArchKind::Avr, + Arch::LoongArch32 | Arch::LoongArch64 => ArchKind::LoongArch, Arch::Msp430 => ArchKind::Msp430, Arch::Nvptx64 => ArchKind::Nvptx, Arch::RiscV32 | Arch::RiscV64 => ArchKind::Riscv, @@ -108,7 +109,10 @@ impl AbiMap { (ExternAbi::EfiApi, ArchKind::Arm(..)) => CanonAbi::Arm(ArmCall::Aapcs), (ExternAbi::EfiApi, ArchKind::X86_64) => CanonAbi::X86(X86Call::Win64), - (ExternAbi::EfiApi, ArchKind::Aarch64 | ArchKind::Riscv | ArchKind::X86) => CanonAbi::C, + ( + ExternAbi::EfiApi, + ArchKind::Aarch64 | ArchKind::LoongArch | ArchKind::Riscv | ArchKind::X86, + ) => CanonAbi::C, (ExternAbi::EfiApi, _) => return AbiMapping::Invalid, /* arm */ @@ -196,6 +200,7 @@ enum ArchKind { Amdgpu, Arm(ArmVer), Avr, + LoongArch, Msp430, Nvptx, Riscv, diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs index 69a65e6b0f024..47775f9684008 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs @@ -15,7 +15,7 @@ pub(crate) fn target() -> Target { data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: Arch::AArch64, options: TargetOptions { - features: "+v8a".into(), + features: "+v8a,+outline-atomics".into(), max_atomic_width: Some(128), stack_probes: StackProbeType::Inline, supported_sanitizers: SanitizerSet::ADDRESS diff --git a/compiler/rustc_transmute/src/layout/tree.rs b/compiler/rustc_transmute/src/layout/tree.rs index d9ebdadf3b79c..1202ed2384315 100644 --- a/compiler/rustc_transmute/src/layout/tree.rs +++ b/compiler/rustc_transmute/src/layout/tree.rs @@ -491,7 +491,7 @@ pub(crate) mod rustc { ) -> Result { // This constructor does not support non-`FieldsShape::Arbitrary` // layouts. - let FieldsShape::Arbitrary { offsets, memory_index } = layout.fields() else { + let FieldsShape::Arbitrary { offsets, in_memory_order } = layout.fields() else { return Err(Err::NotYetSupported); }; @@ -519,8 +519,7 @@ pub(crate) mod rustc { } // Append the fields, in memory order, to the layout. - let inverse_memory_index = memory_index.invert_bijective_mapping(); - for &field_idx in inverse_memory_index.iter() { + for &field_idx in in_memory_order.iter() { // Add interfield padding. let padding_needed = offsets[field_idx] - size; let padding = Self::padding(padding_needed.bytes_usize()); diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 6962eeddf1370..62f3667ad7f4f 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -9,7 +9,7 @@ use rustc_abi::{ use rustc_hashes::Hash64; use rustc_hir::attrs::AttributeKind; use rustc_hir::find_attr; -use rustc_index::IndexVec; +use rustc_index::{Idx as _, IndexVec}; use rustc_middle::bug; use rustc_middle::query::Providers; use rustc_middle::traits::ObligationCause; @@ -374,7 +374,7 @@ fn layout_of_uncached<'tcx>( // specifically care about pattern types will have to handle it. layout.fields = FieldsShape::Arbitrary { offsets: [Size::ZERO].into_iter().collect(), - memory_index: [0].into_iter().collect(), + in_memory_order: [FieldIdx::new(0)].into_iter().collect(), }; tcx.mk_layout(layout) } diff --git a/src/tools/miri/src/helpers.rs b/src/tools/miri/src/helpers.rs index f11860f4a1adf..218e4ffe5e2f7 100644 --- a/src/tools/miri/src/helpers.rs +++ b/src/tools/miri/src/helpers.rs @@ -584,10 +584,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } fn aggregate_field_iter( - memory_index: &IndexVec, - ) -> impl Iterator + 'static { - let inverse_memory_index = memory_index.invert_bijective_mapping(); - inverse_memory_index.into_iter() + in_memory_order: &IndexVec, + ) -> impl Iterator { + in_memory_order.iter().copied() } // Hook to detect `UnsafeCell`. diff --git a/tests/codegen-llvm/gpu_offload/gpu_host.rs b/tests/codegen-llvm/gpu_offload/gpu_host.rs index b0f83c825705f..ac179a65828d7 100644 --- a/tests/codegen-llvm/gpu_offload/gpu_host.rs +++ b/tests/codegen-llvm/gpu_offload/gpu_host.rs @@ -11,6 +11,7 @@ // when inside of a function called main. This, too, is a temporary workaround for not having a // frontend. +#![feature(rustc_attrs)] #![feature(core_intrinsics)] #![no_main] @@ -21,29 +22,31 @@ fn main() { core::hint::black_box(&x); } -// CHECK: %struct.__tgt_offload_entry = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr } // CHECK: %struct.ident_t = type { i32, i32, i32, i32, ptr } +// CHECK: %struct.__tgt_offload_entry = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr } // CHECK: %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } // CHECK: %struct.__tgt_kernel_arguments = type { i32, i32, ptr, ptr, ptr, ptr, ptr, ptr, i64, i64, [3 x i32], [3 x i32], i32 } +// CHECK: @anon.{{.*}}.0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1 +// CHECK: @anon.{{.*}}.1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @anon.{{.*}}.0 }, align 8 + // CHECK: @.offload_sizes._kernel_1 = private unnamed_addr constant [1 x i64] [i64 1024] // CHECK: @.offload_maptypes._kernel_1 = private unnamed_addr constant [1 x i64] [i64 35] -// CHECK: @._kernel_1.region_id = internal unnamed_addr constant i8 0 +// CHECK: @._kernel_1.region_id = internal constant i8 0 // CHECK: @.offloading.entry_name._kernel_1 = internal unnamed_addr constant [10 x i8] c"_kernel_1\00", section ".llvm.rodata.offloading", align 1 // CHECK: @.offloading.entry._kernel_1 = internal constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @._kernel_1.region_id, ptr @.offloading.entry_name._kernel_1, i64 0, i64 0, ptr null }, section "llvm_offload_entries", align 8 -// CHECK: @anon.{{.*}}.0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1 -// CHECK: @anon.{{.*}}.1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @anon.{{.*}}.0 }, align 8 +// CHECK: Function Attrs: nounwind +// CHECK: declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr) -// CHECK: Function Attrs: -// CHECK-NEXT: define{{( dso_local)?}} void @main() +// CHECK: define{{( dso_local)?}} void @main() // CHECK-NEXT: start: // CHECK-NEXT: %0 = alloca [8 x i8], align 8 // CHECK-NEXT: %x = alloca [1024 x i8], align 16 // CHECK: call void @kernel_1(ptr noalias noundef nonnull align 4 dereferenceable(1024) %x) // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %0) // CHECK-NEXT: store ptr %x, ptr %0, align 8 -// CHECK-NEXT: call void asm sideeffect "", "r,~{memory}"(ptr nonnull %0) #4, !srcloc !4 +// CHECK-NEXT: call void asm sideeffect "", "r,~{memory}"(ptr nonnull %0) // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %0) // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %x) // CHECK-NEXT: ret void @@ -92,9 +95,6 @@ fn main() { // CHECK-NEXT: ret void // CHECK-NEXT: } -// CHECK: Function Attrs: nounwind -// CHECK: declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr) - #[unsafe(no_mangle)] #[inline(never)] pub fn kernel_1(x: &mut [f32; 256]) { diff --git a/tests/ui/abi/c-zst.aarch64-darwin.stderr b/tests/ui/abi/c-zst.aarch64-darwin.stderr index d050bfcbaa173..6d2ac90c0c975 100644 --- a/tests/ui/abi/c-zst.aarch64-darwin.stderr +++ b/tests/ui/abi/c-zst.aarch64-darwin.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -41,7 +41,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/c-zst.powerpc-linux.stderr b/tests/ui/abi/c-zst.powerpc-linux.stderr index 816addd795761..f297aa984dd2e 100644 --- a/tests/ui/abi/c-zst.powerpc-linux.stderr +++ b/tests/ui/abi/c-zst.powerpc-linux.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/c-zst.s390x-linux.stderr b/tests/ui/abi/c-zst.s390x-linux.stderr index 816addd795761..f297aa984dd2e 100644 --- a/tests/ui/abi/c-zst.s390x-linux.stderr +++ b/tests/ui/abi/c-zst.s390x-linux.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/c-zst.sparc64-linux.stderr b/tests/ui/abi/c-zst.sparc64-linux.stderr index 816addd795761..f297aa984dd2e 100644 --- a/tests/ui/abi/c-zst.sparc64-linux.stderr +++ b/tests/ui/abi/c-zst.sparc64-linux.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/c-zst.x86_64-linux.stderr b/tests/ui/abi/c-zst.x86_64-linux.stderr index d050bfcbaa173..6d2ac90c0c975 100644 --- a/tests/ui/abi/c-zst.x86_64-linux.stderr +++ b/tests/ui/abi/c-zst.x86_64-linux.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -41,7 +41,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr index 816addd795761..f297aa984dd2e 100644 --- a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr +++ b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/debug.generic.stderr b/tests/ui/abi/debug.generic.stderr index 04d6f50872a39..8a031b79780a8 100644 --- a/tests/ui/abi/debug.generic.stderr +++ b/tests/ui/abi/debug.generic.stderr @@ -244,7 +244,7 @@ error: fn_abi_of(test_generic) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -328,7 +328,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -400,7 +400,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -479,7 +479,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -552,7 +552,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -629,7 +629,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -701,7 +701,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -779,7 +779,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -851,7 +851,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -962,7 +962,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/debug.loongarch64.stderr b/tests/ui/abi/debug.loongarch64.stderr index 85c888c4fae08..00bd3febde4ec 100644 --- a/tests/ui/abi/debug.loongarch64.stderr +++ b/tests/ui/abi/debug.loongarch64.stderr @@ -244,7 +244,7 @@ error: fn_abi_of(test_generic) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -328,7 +328,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -400,7 +400,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -479,7 +479,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -552,7 +552,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -629,7 +629,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -701,7 +701,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -779,7 +779,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -851,7 +851,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -962,7 +962,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/debug.riscv64.stderr b/tests/ui/abi/debug.riscv64.stderr index 85c888c4fae08..00bd3febde4ec 100644 --- a/tests/ui/abi/debug.riscv64.stderr +++ b/tests/ui/abi/debug.riscv64.stderr @@ -244,7 +244,7 @@ error: fn_abi_of(test_generic) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -328,7 +328,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -400,7 +400,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -479,7 +479,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -552,7 +552,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -629,7 +629,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -701,7 +701,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -779,7 +779,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -851,7 +851,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -962,7 +962,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/pass-indirectly-attr.stderr b/tests/ui/abi/pass-indirectly-attr.stderr index a93982dacfa1b..03fe9ea46a4b7 100644 --- a/tests/ui/abi/pass-indirectly-attr.stderr +++ b/tests/ui/abi/pass-indirectly-attr.stderr @@ -24,7 +24,7 @@ error: fn_abi_of(extern_c) = FnAbi { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -65,7 +65,7 @@ error: fn_abi_of(extern_c) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -106,7 +106,7 @@ error: fn_abi_of(extern_rust) = FnAbi { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -165,7 +165,7 @@ error: fn_abi_of(extern_rust) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/sysv64-zst.stderr b/tests/ui/abi/sysv64-zst.stderr index 2233e8e4f623e..82d3793c35328 100644 --- a/tests/ui/abi/sysv64-zst.stderr +++ b/tests/ui/abi/sysv64-zst.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -41,7 +41,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr index a86b28d98ecc6..584416f58f861 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr +++ b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(take_va_list) = FnAbi { }, fields: Arbitrary { offsets: $OFFSETS, - memory_index: $MEMORY_INDEX, + in_memory_order: $MEMORY_INDEX, }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(take_va_list) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/c-variadic/pass-by-value-abi.rs b/tests/ui/c-variadic/pass-by-value-abi.rs index 1e3935067cc27..a7cf37295a74a 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.rs +++ b/tests/ui/c-variadic/pass-by-value-abi.rs @@ -1,7 +1,7 @@ //@ check-fail //@ normalize-stderr: "randomization_seed: \d+" -> "randomization_seed: $$SEED" //@ normalize-stderr: "valid_range: 0\.\.=\d+" -> "valid_range: 0..=$$MAX" -//@ normalize-stderr: "memory_index: \[[^\]]+\]" -> "memory_index: $$MEMORY_INDEX" +//@ normalize-stderr: "in_memory_order: \[[^\]]+\]" -> "in_memory_order: $$MEMORY_INDEX" //@ normalize-stderr: "offsets: \[[^\]]+\]" -> "offsets: $$OFFSETS" //@ revisions: x86_64 aarch64 win //@ compile-flags: -O diff --git a/tests/ui/c-variadic/pass-by-value-abi.win.stderr b/tests/ui/c-variadic/pass-by-value-abi.win.stderr index 8ff93961f6678..b8e3f699b30e8 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.win.stderr +++ b/tests/ui/c-variadic/pass-by-value-abi.win.stderr @@ -20,7 +20,7 @@ error: fn_abi_of(take_va_list) = FnAbi { ), fields: Arbitrary { offsets: $OFFSETS, - memory_index: $MEMORY_INDEX, + in_memory_order: $MEMORY_INDEX, }, largest_niche: None, uninhabited: false, @@ -55,7 +55,7 @@ error: fn_abi_of(take_va_list) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr index aafe89d59e333..b5e0e8589af16 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr +++ b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(take_va_list) = FnAbi { }, fields: Arbitrary { offsets: $OFFSETS, - memory_index: $MEMORY_INDEX, + in_memory_order: $MEMORY_INDEX, }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(take_va_list) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -91,7 +91,7 @@ error: fn_abi_of(take_va_list_sysv64) = FnAbi { }, fields: Arbitrary { offsets: $OFFSETS, - memory_index: $MEMORY_INDEX, + in_memory_order: $MEMORY_INDEX, }, largest_niche: None, uninhabited: false, @@ -130,7 +130,7 @@ error: fn_abi_of(take_va_list_sysv64) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -171,7 +171,7 @@ error: fn_abi_of(take_va_list_win64) = FnAbi { }, fields: Arbitrary { offsets: $OFFSETS, - memory_index: $MEMORY_INDEX, + in_memory_order: $MEMORY_INDEX, }, largest_niche: None, uninhabited: false, @@ -210,7 +210,7 @@ error: fn_abi_of(take_va_list_win64) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/enum-discriminant/wrapping_niche.stderr b/tests/ui/enum-discriminant/wrapping_niche.stderr index e3e1755e14dd4..9b97ad4aeac7e 100644 --- a/tests/ui/enum-discriminant/wrapping_niche.stderr +++ b/tests/ui/enum-discriminant/wrapping_niche.stderr @@ -16,7 +16,7 @@ error: layout_of(UnsignedAroundZero) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -52,7 +52,7 @@ error: layout_of(UnsignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -73,7 +73,7 @@ error: layout_of(UnsignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -94,7 +94,7 @@ error: layout_of(UnsignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -134,7 +134,7 @@ error: layout_of(SignedAroundZero) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -170,7 +170,7 @@ error: layout_of(SignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -191,7 +191,7 @@ error: layout_of(SignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -212,7 +212,7 @@ error: layout_of(SignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/layout/debug.stderr b/tests/ui/layout/debug.stderr index b2ce6385ab654..79ce21eb532b0 100644 --- a/tests/ui/layout/debug.stderr +++ b/tests/ui/layout/debug.stderr @@ -16,7 +16,7 @@ error: layout_of(E) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -52,7 +52,7 @@ error: layout_of(E) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -77,7 +77,7 @@ error: layout_of(E) = Layout { Size(4 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, 2, @@ -130,7 +130,7 @@ error: layout_of(S) = Layout { Size(8 bytes), Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, 2, 1, @@ -200,7 +200,7 @@ error: layout_of(Result) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -251,7 +251,7 @@ error: layout_of(Result) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -289,7 +289,7 @@ error: layout_of(Result) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/layout/hexagon-enum.stderr b/tests/ui/layout/hexagon-enum.stderr index 659013c8a6f1a..20e0a8642a66f 100644 --- a/tests/ui/layout/hexagon-enum.stderr +++ b/tests/ui/layout/hexagon-enum.stderr @@ -16,7 +16,7 @@ error: layout_of(A) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -52,7 +52,7 @@ error: layout_of(A) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -92,7 +92,7 @@ error: layout_of(B) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -128,7 +128,7 @@ error: layout_of(B) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -168,7 +168,7 @@ error: layout_of(C) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -204,7 +204,7 @@ error: layout_of(C) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -244,7 +244,7 @@ error: layout_of(P) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -280,7 +280,7 @@ error: layout_of(P) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -320,7 +320,7 @@ error: layout_of(T) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -356,7 +356,7 @@ error: layout_of(T) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr b/tests/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr index 2087fedeb19bc..61cfcbdc07f75 100644 --- a/tests/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr +++ b/tests/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr @@ -22,7 +22,7 @@ error: layout_of(MissingPayloadField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -72,7 +72,7 @@ error: layout_of(MissingPayloadField) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -95,7 +95,7 @@ error: layout_of(MissingPayloadField) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -142,7 +142,7 @@ error: layout_of(CommonPayloadField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -193,7 +193,7 @@ error: layout_of(CommonPayloadField) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(CommonPayloadField) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -279,7 +279,7 @@ error: layout_of(CommonPayloadFieldIsMaybeUninit) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -329,7 +329,7 @@ error: layout_of(CommonPayloadFieldIsMaybeUninit) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -366,7 +366,7 @@ error: layout_of(CommonPayloadFieldIsMaybeUninit) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -414,7 +414,7 @@ error: layout_of(NicheFirst) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -470,7 +470,7 @@ error: layout_of(NicheFirst) = Layout { Size(0 bytes), Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -503,7 +503,7 @@ error: layout_of(NicheFirst) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -524,7 +524,7 @@ error: layout_of(NicheFirst) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -570,7 +570,7 @@ error: layout_of(NicheSecond) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -626,7 +626,7 @@ error: layout_of(NicheSecond) = Layout { Size(1 bytes), Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 1, 0, ], @@ -659,7 +659,7 @@ error: layout_of(NicheSecond) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -680,7 +680,7 @@ error: layout_of(NicheSecond) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/layout/issue-96185-overaligned-enum.stderr b/tests/ui/layout/issue-96185-overaligned-enum.stderr index 6bcc5b4906b50..64e2f42c042f1 100644 --- a/tests/ui/layout/issue-96185-overaligned-enum.stderr +++ b/tests/ui/layout/issue-96185-overaligned-enum.stderr @@ -10,7 +10,7 @@ error: layout_of(Aligned1) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -46,7 +46,7 @@ error: layout_of(Aligned1) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -69,7 +69,7 @@ error: layout_of(Aligned1) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -113,7 +113,7 @@ error: layout_of(Aligned2) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -149,7 +149,7 @@ error: layout_of(Aligned2) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -172,7 +172,7 @@ error: layout_of(Aligned2) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/layout/thumb-enum.stderr b/tests/ui/layout/thumb-enum.stderr index 1ef22daf2a9d6..a6e603652123b 100644 --- a/tests/ui/layout/thumb-enum.stderr +++ b/tests/ui/layout/thumb-enum.stderr @@ -16,7 +16,7 @@ error: layout_of(A) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -52,7 +52,7 @@ error: layout_of(A) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -92,7 +92,7 @@ error: layout_of(B) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -128,7 +128,7 @@ error: layout_of(B) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -168,7 +168,7 @@ error: layout_of(C) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -204,7 +204,7 @@ error: layout_of(C) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -244,7 +244,7 @@ error: layout_of(P) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -280,7 +280,7 @@ error: layout_of(P) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -320,7 +320,7 @@ error: layout_of(T) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -356,7 +356,7 @@ error: layout_of(T) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/layout/zero-sized-array-enum-niche.stderr b/tests/ui/layout/zero-sized-array-enum-niche.stderr index 1707b8aff81cf..23f9092778085 100644 --- a/tests/ui/layout/zero-sized-array-enum-niche.stderr +++ b/tests/ui/layout/zero-sized-array-enum-niche.stderr @@ -10,7 +10,7 @@ error: layout_of(Result<[u32; 0], bool>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -48,7 +48,7 @@ error: layout_of(Result<[u32; 0], bool>) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -73,7 +73,7 @@ error: layout_of(Result<[u32; 0], bool>) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -118,7 +118,7 @@ error: layout_of(MultipleAlignments) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -156,7 +156,7 @@ error: layout_of(MultipleAlignments) = Layout { offsets: [ Size(2 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -181,7 +181,7 @@ error: layout_of(MultipleAlignments) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -206,7 +206,7 @@ error: layout_of(MultipleAlignments) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -251,7 +251,7 @@ error: layout_of(Result<[u32; 0], Packed>>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -289,7 +289,7 @@ error: layout_of(Result<[u32; 0], Packed>>) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -314,7 +314,7 @@ error: layout_of(Result<[u32; 0], Packed>>) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -359,7 +359,7 @@ error: layout_of(Result<[u32; 0], Packed>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -401,7 +401,7 @@ error: layout_of(Result<[u32; 0], Packed>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -426,7 +426,7 @@ error: layout_of(Result<[u32; 0], Packed>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/repr/repr-c-dead-variants.aarch64-unknown-linux-gnu.stderr b/tests/ui/repr/repr-c-dead-variants.aarch64-unknown-linux-gnu.stderr index 1a1929c530ff4..62f6ec92d493c 100644 --- a/tests/ui/repr/repr-c-dead-variants.aarch64-unknown-linux-gnu.stderr +++ b/tests/ui/repr/repr-c-dead-variants.aarch64-unknown-linux-gnu.stderr @@ -16,7 +16,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -60,7 +60,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -108,7 +108,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -158,7 +158,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -195,7 +195,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -270,7 +270,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { Size(8 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -298,7 +298,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/repr/repr-c-dead-variants.armebv7r-none-eabi.stderr b/tests/ui/repr/repr-c-dead-variants.armebv7r-none-eabi.stderr index 67729e9fcef9c..3e0efad974cd2 100644 --- a/tests/ui/repr/repr-c-dead-variants.armebv7r-none-eabi.stderr +++ b/tests/ui/repr/repr-c-dead-variants.armebv7r-none-eabi.stderr @@ -16,7 +16,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -60,7 +60,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -108,7 +108,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -158,7 +158,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -195,7 +195,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -270,7 +270,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { Size(8 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -298,7 +298,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/repr/repr-c-dead-variants.i686-pc-windows-msvc.stderr b/tests/ui/repr/repr-c-dead-variants.i686-pc-windows-msvc.stderr index 1a1929c530ff4..62f6ec92d493c 100644 --- a/tests/ui/repr/repr-c-dead-variants.i686-pc-windows-msvc.stderr +++ b/tests/ui/repr/repr-c-dead-variants.i686-pc-windows-msvc.stderr @@ -16,7 +16,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -60,7 +60,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -108,7 +108,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -158,7 +158,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -195,7 +195,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -270,7 +270,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { Size(8 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -298,7 +298,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/repr/repr-c-dead-variants.x86_64-unknown-linux-gnu.stderr b/tests/ui/repr/repr-c-dead-variants.x86_64-unknown-linux-gnu.stderr index 1a1929c530ff4..62f6ec92d493c 100644 --- a/tests/ui/repr/repr-c-dead-variants.x86_64-unknown-linux-gnu.stderr +++ b/tests/ui/repr/repr-c-dead-variants.x86_64-unknown-linux-gnu.stderr @@ -16,7 +16,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -60,7 +60,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -108,7 +108,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -158,7 +158,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -195,7 +195,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -270,7 +270,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { Size(8 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -298,7 +298,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/repr/repr-c-int-dead-variants.stderr b/tests/ui/repr/repr-c-int-dead-variants.stderr index d88a842f88482..b25e4a9b7b6f5 100644 --- a/tests/ui/repr/repr-c-int-dead-variants.stderr +++ b/tests/ui/repr/repr-c-int-dead-variants.stderr @@ -16,7 +16,7 @@ error: layout_of(UnivariantU8) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -60,7 +60,7 @@ error: layout_of(UnivariantU8) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -108,7 +108,7 @@ error: layout_of(TwoVariantsU8) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -158,7 +158,7 @@ error: layout_of(TwoVariantsU8) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -195,7 +195,7 @@ error: layout_of(TwoVariantsU8) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(DeadBranchHasOtherFieldU8) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -270,7 +270,7 @@ error: layout_of(DeadBranchHasOtherFieldU8) = Layout { Size(8 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -298,7 +298,7 @@ error: layout_of(DeadBranchHasOtherFieldU8) = Layout { offsets: [ Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/type/pattern_types/non_null.stderr b/tests/ui/type/pattern_types/non_null.stderr index ad61e9a591473..0847af2d086fa 100644 --- a/tests/ui/type/pattern_types/non_null.stderr +++ b/tests/ui/type/pattern_types/non_null.stderr @@ -17,7 +17,7 @@ error: layout_of((*const T) is !null) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -64,7 +64,7 @@ error: layout_of(Option<(*const ()) is !null>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -96,7 +96,7 @@ error: layout_of(Option<(*const ()) is !null>) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -126,7 +126,7 @@ error: layout_of(Option<(*const ()) is !null>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -186,7 +186,7 @@ error: layout_of((*const [u8]) is !null) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/type/pattern_types/or_patterns.stderr b/tests/ui/type/pattern_types/or_patterns.stderr index adc7320829b95..7206c570187b4 100644 --- a/tests/ui/type/pattern_types/or_patterns.stderr +++ b/tests/ui/type/pattern_types/or_patterns.stderr @@ -57,7 +57,7 @@ error: layout_of((i8) is (i8::MIN..=-1 | 1..)) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -102,7 +102,7 @@ error: layout_of((i8) is (i8::MIN..=-2 | 0..)) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/type/pattern_types/range_patterns.stderr b/tests/ui/type/pattern_types/range_patterns.stderr index c9a846e22f581..6c9acac7eb7b8 100644 --- a/tests/ui/type/pattern_types/range_patterns.stderr +++ b/tests/ui/type/pattern_types/range_patterns.stderr @@ -16,7 +16,7 @@ error: layout_of(NonZero) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -61,7 +61,7 @@ error: layout_of((u32) is 1..) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -106,7 +106,7 @@ error: layout_of(Option<(u32) is 1..>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -137,7 +137,7 @@ error: layout_of(Option<(u32) is 1..>) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -166,7 +166,7 @@ error: layout_of(Option<(u32) is 1..>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -217,7 +217,7 @@ error: layout_of(Option>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -248,7 +248,7 @@ error: layout_of(Option>) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -277,7 +277,7 @@ error: layout_of(Option>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -328,7 +328,7 @@ error: layout_of(NonZeroU32New) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -401,7 +401,7 @@ error: layout_of((i8) is -10..=10) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -446,7 +446,7 @@ error: layout_of((i8) is i8::MIN..=0) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], },