diff --git a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs index 7e5ea6bf3c22c..885ad6071d91c 100644 --- a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs +++ b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs @@ -65,9 +65,17 @@ fn do_check_simd_vector_abi<'tcx>( let size = arg_abi.layout.size; match passes_vectors_by_value(&arg_abi.mode, &arg_abi.layout.backend_repr) { UsesVectorRegisters::FixedVector => { + // Some targets use homogeneous aggregates, where the unit size counts. + let unit_size = match &arg_abi.mode { + PassMode::Cast { pad_i32_count: _, cast } if cast.prefix.is_empty() => { + cast.rest.unit.size + } + _ => size, + }; + let feature_def = tcx.sess.target.features_for_correct_fixed_length_vector_abi(); // Find the first feature that provides at least this vector size. - let feature = match feature_def.iter().find(|(bits, _)| size.bits() <= *bits) { + let feature = match feature_def.iter().find(|(bits, _)| unit_size.bits() <= *bits) { Some((_, feature)) => feature, None => { let (span, _hir_id) = loc(); diff --git a/compiler/rustc_target/src/callconv/aarch64.rs b/compiler/rustc_target/src/callconv/aarch64.rs index 0162aa838cb6b..09187836ee65a 100644 --- a/compiler/rustc_target/src/callconv/aarch64.rs +++ b/compiler/rustc_target/src/callconv/aarch64.rs @@ -35,7 +35,7 @@ where // The softfloat ABI treats floats like integers, so they // do not get homogeneous aggregate treatment. RegKind::Float => cx.target_spec().rustc_abi != Some(RustcAbi::Softfloat), - RegKind::Vector { .. } => size.bits() == 64 || size.bits() == 128, + RegKind::Vector { .. } => unit.size.bits() == 64 || unit.size.bits() == 128, }; valid_unit.then_some(Uniform::consecutive(unit, size)) diff --git a/compiler/rustc_target/src/callconv/arm.rs b/compiler/rustc_target/src/callconv/arm.rs index 66f0ded3874f9..615bd4f540068 100644 --- a/compiler/rustc_target/src/callconv/arm.rs +++ b/compiler/rustc_target/src/callconv/arm.rs @@ -26,7 +26,7 @@ where let valid_unit = match unit.kind { RegKind::Integer => false, RegKind::Float => true, - RegKind::Vector { .. } => size.bits() == 64 || size.bits() == 128, + RegKind::Vector { .. } => unit.size.bits() == 64 || unit.size.bits() == 128, }; valid_unit.then_some(Uniform::consecutive(unit, size)) diff --git a/compiler/rustc_target/src/callconv/powerpc64.rs b/compiler/rustc_target/src/callconv/powerpc64.rs index 3eb40abe90f33..075e69f7d74d8 100644 --- a/compiler/rustc_target/src/callconv/powerpc64.rs +++ b/compiler/rustc_target/src/callconv/powerpc64.rs @@ -36,7 +36,7 @@ where let valid_unit = match unit.kind { RegKind::Integer => false, RegKind::Float => true, - RegKind::Vector { .. } => arg.layout.size.bits() == 128, + RegKind::Vector { .. } => unit.size.bits() == 128, }; valid_unit.then_some(Uniform::consecutive(unit, arg.layout.size)) diff --git a/tests/auxiliary/minicore.rs b/tests/auxiliary/minicore.rs index 04564049dbed2..2b7eb0b9afbcf 100644 --- a/tests/auxiliary/minicore.rs +++ b/tests/auxiliary/minicore.rs @@ -496,4 +496,5 @@ pub mod simd { pub type i64x8 = Simd; pub type u8x16 = Simd; + pub type u64x2 = Simd; } diff --git a/tests/codegen-llvm/aarch64-abi/homogeneous-aggregate.rs b/tests/codegen-llvm/aarch64-abi/homogeneous-aggregate.rs new file mode 100644 index 0000000000000..3d5ec10f2b2f7 --- /dev/null +++ b/tests/codegen-llvm/aarch64-abi/homogeneous-aggregate.rs @@ -0,0 +1,84 @@ +//@ add-minicore +//@ compile-flags: -Cno-prepopulate-passes -Copt-level=0 +// +//@ revisions: linux win +//@[linux] compile-flags: --target aarch64-unknown-linux-gnu +//@[win] compile-flags: --target aarch64-pc-windows-msvc +// +//@ needs-llvm-components: aarch64 + +// Test that homogeneous aggregates are passed and returned with the correct ABI. + +#![feature(no_core, lang_items)] +#![crate_type = "lib"] +#![no_core] + +extern crate minicore; +use minicore::simd::*; +use minicore::*; + +// A homogeneous float aggregate. +#[repr(C)] +pub struct Hfa { + pub a: f32, + pub b: f32, +} +impl Copy for Hfa {} + +// CHECK: define void @test_hfa([2 x float] %0) +#[unsafe(no_mangle)] +pub extern "C" fn test_hfa(a: Hfa) { + hint::black_box(a); +} + +// Fields can be vectors too. +#[repr(C)] +pub struct Hfa2V2F64 { + pub a: f64x2, + pub b: f64x2, +} + +// CHECK: define void @test_hfa_2_f64x2([2 x <2 x double>] %0) +#[unsafe(no_mangle)] +pub extern "C" fn test_hfa_2_f64x2(a: Hfa2V2F64) { + hint::black_box(a); +} + +#[repr(C)] +pub struct Hfa2V2U64 { + pub a: u64x2, + pub b: u64x2, +} + +// CHECK: define void @test_hfa_2_u64x2([2 x <16 x i8>] %0) +#[unsafe(no_mangle)] +pub extern "C" fn test_hfa_2_u64x2(a: Hfa2V2U64) { + hint::black_box(a); +} + +#[repr(C)] +pub struct Hfa2V2F32 { + pub a: f32x2, + pub b: f32x2, +} + +// CHECK: define void @test_hfa_2_f32x2([2 x <2 x float>] %0) +#[unsafe(no_mangle)] +pub extern "C" fn test_hfa_2_f32x2(a: Hfa2V2F32) { + hint::black_box(a); +} + +#[repr(C)] +pub struct Hfa4V2F64 { + pub a: f64x2, + pub b: f64x2, + pub c: f64x2, + pub d: f64x2, +} + +// CHECK: define void @test_hfa_4_f64x2([4 x <2 x double>] %0) +#[unsafe(no_mangle)] +#[target_feature(enable = "neon")] +pub extern "C" fn test_hfa_4_f64x2(a: Hfa4V2F64) { + hint::black_box(a); +} diff --git a/tests/codegen-llvm/arm-abi/homogeneous-aggregate.rs b/tests/codegen-llvm/arm-abi/homogeneous-aggregate.rs index c44dc4fa56f5f..272eb419494b0 100644 --- a/tests/codegen-llvm/arm-abi/homogeneous-aggregate.rs +++ b/tests/codegen-llvm/arm-abi/homogeneous-aggregate.rs @@ -11,10 +11,12 @@ // Test that homogeneous aggregates are passed and returned with the correct ABI on 32-bit arm. #![feature(no_core, lang_items)] +#![feature(arm_target_feature)] #![crate_type = "lib"] #![no_core] extern crate minicore; +use minicore::simd::*; use minicore::*; // A homogeneous float aggregate, which a hard-float ABI passes in VFP registers. @@ -68,6 +70,69 @@ pub extern "C" fn test_hfa_4_f64(a: Hfa4F64) { hint::black_box(a); } +// Fields can be vectors too. +#[repr(C)] +pub struct Hfa2V2F64 { + pub a: f64x2, + pub b: f64x2, +} + +// linux: define void @test_hfa_2_f64x2([2 x <2 x double>] %0) +// eabi: define dso_local void @test_hfa_2_f64x2([4 x i64] %0) +// watchos: define void @test_hfa_2_f64x2([2 x <2 x double>] %0) +#[unsafe(no_mangle)] +#[target_feature(enable = "neon")] +pub extern "C" fn test_hfa_2_f64x2(a: Hfa2V2F64) { + hint::black_box(a); +} + +#[repr(C)] +pub struct Hfa2V2U64 { + pub a: u64x2, + pub b: u64x2, +} + +// linux: define void @test_hfa_2_u64x2([2 x <16 x i8>] %0) +// eabi: define dso_local void @test_hfa_2_u64x2([4 x i64] %0) +// watchos: define void @test_hfa_2_u64x2([2 x <16 x i8>] %0) +#[unsafe(no_mangle)] +#[target_feature(enable = "neon")] +pub extern "C" fn test_hfa_2_u64x2(a: Hfa2V2U64) { + hint::black_box(a); +} + +#[repr(C)] +pub struct Hfa2V2F32 { + pub a: f32x2, + pub b: f32x2, +} + +// linux: define void @test_hfa_2_f32x2([2 x <2 x float>] %0) +// eabi: define dso_local void @test_hfa_2_f32x2([2 x i64] %0) +// watchos: define void @test_hfa_2_f32x2([2 x <2 x float>] %0) +#[unsafe(no_mangle)] +#[target_feature(enable = "neon")] +pub extern "C" fn test_hfa_2_f32x2(a: Hfa2V2F32) { + hint::black_box(a); +} + +#[repr(C)] +pub struct Hfa4V2F64 { + pub a: f64x2, + pub b: f64x2, + pub c: f64x2, + pub d: f64x2, +} + +// linux: define void @test_hfa_4_f64x2([4 x <2 x double>] %0) +// eabi: define dso_local void @test_hfa_4_f64x2([8 x i64] %0) +// watchos: define void @test_hfa_4_f64x2([4 x <2 x double>] %0) +#[unsafe(no_mangle)] +#[target_feature(enable = "neon")] +pub extern "C" fn test_hfa_4_f64x2(a: Hfa4V2F64) { + hint::black_box(a); +} + // A homogeneous aggregate can have at most 4 fields, so this does not qualify. #[repr(C)] pub struct Floats5 { diff --git a/tests/codegen-llvm/powerpc64-abi/homogeneous-aggregate.rs b/tests/codegen-llvm/powerpc64-abi/homogeneous-aggregate.rs new file mode 100644 index 0000000000000..dd053411aa4e5 --- /dev/null +++ b/tests/codegen-llvm/powerpc64-abi/homogeneous-aggregate.rs @@ -0,0 +1,96 @@ +//@ add-minicore +//@ compile-flags: -Cno-prepopulate-passes -Copt-level=0 +// +//@ revisions: ppc64 ppc64_vsx ppc64le +//@[ppc64] compile-flags: --target powerpc64-unknown-linux-gnu +//@[ppc64_vsx] compile-flags: --target powerpc64-unknown-linux-gnu -Ctarget-feature=+vsx +//@[ppc64le] compile-flags: --target powerpc64le-unknown-linux-gnu +// +//@ needs-llvm-components: powerpc + +// Test that homogeneous aggregates are passed and returned with the correct ABI. + +#![feature(no_core, lang_items)] +#![crate_type = "lib"] +#![no_core] + +extern crate minicore; +use minicore::simd::*; +use minicore::*; + +// A homogeneous float aggregate. +#[repr(C)] +pub struct Hfa { + pub a: f32, + pub b: f32, +} +impl Copy for Hfa {} + +// ppc64: define void @test_hfa(i64 %0) +// ppc64_vsx: define void @test_hfa(i64 %0) +// ppc64le: define void @test_hfa([2 x float] %0) +#[unsafe(no_mangle)] +pub extern "C" fn test_hfa(a: Hfa) { + hint::black_box(a); +} + +// Fields can be vectors too. +#[repr(C)] +pub struct Hfa2V2F64 { + pub a: f64x2, + pub b: f64x2, +} + +// ppc64: define void @test_hfa_2_f64x2([2 x i128] %0) +// ppc64_vsx: define void @test_hfa_2_f64x2([2 x i128] %0) +// ppc64le: define void @test_hfa_2_f64x2([2 x <2 x double>] %0) +#[unsafe(no_mangle)] +pub extern "C" fn test_hfa_2_f64x2(a: Hfa2V2F64) { + hint::black_box(a); +} + +#[repr(C)] +pub struct Hfa2V2U64 { + pub a: u64x2, + pub b: u64x2, +} + +// ppc64: define void @test_hfa_2_u64x2([2 x i128] %0) +// ppc64_vsx: define void @test_hfa_2_u64x2([2 x i128] %0) +// ppc64le: define void @test_hfa_2_u64x2([2 x <16 x i8>] %0) +#[unsafe(no_mangle)] +pub extern "C" fn test_hfa_2_u64x2(a: Hfa2V2U64) { + hint::black_box(a); +} + +#[repr(C)] +pub struct Hfa2V2F32 { + pub a: f32x2, + pub b: f32x2, +} + +// On PowerPC only 128-bit units are eligible for HVA. +// +// ppc64: define void @test_hfa_2_f32x2([2 x i64] %0) +// ppc64_vsx: define void @test_hfa_2_f32x2([2 x i64] %0) +// ppc64le: define void @test_hfa_2_f32x2([2 x i64] %0) +#[unsafe(no_mangle)] +pub extern "C" fn test_hfa_2_f32x2(a: Hfa2V2F32) { + hint::black_box(a); +} + +#[repr(C)] +pub struct Hfa4V2F64 { + pub a: f64x2, + pub b: f64x2, + pub c: f64x2, + pub d: f64x2, +} + +// ppc64: define void @test_hfa_4_f64x2([4 x i128] %0) +// ppc64_vsx: define void @test_hfa_4_f64x2([4 x i128] %0) +// ppc64le: define void @test_hfa_4_f64x2([4 x <2 x double>] %0) +#[unsafe(no_mangle)] +pub extern "C" fn test_hfa_4_f64x2(a: Hfa4V2F64) { + hint::black_box(a); +} diff --git a/tests/codegen-llvm/preserve-vec-element-types.rs b/tests/codegen-llvm/preserve-vec-element-types.rs index b3908b1c24cc2..00f9ef6fab2a6 100644 --- a/tests/codegen-llvm/preserve-vec-element-types.rs +++ b/tests/codegen-llvm/preserve-vec-element-types.rs @@ -52,21 +52,22 @@ mod tests { // CHECK: define [2 x <1 x ptr>] @pair_ptrx1_t([2 x <1 x ptr>] {{.*}} %0) #[unsafe(no_mangle)] extern "C" fn pair_ptrx1_t(x: Pair>) -> Pair> { x } - // When it fits in a 128-bit register, it's passed directly. + // When the fields are not 64 or 128 bits in size, they do not qualify as a homogeneous + // aggregate, and passed as type-erased sequences of integers. - // CHECK: define [4 x <4 x i8>] @quad_int8x4_t([4 x <4 x i8>] {{.*}} %0) + // CHECK: define [2 x i64] @quad_int8x4_t([2 x i64] {{.*}} %0) #[unsafe(no_mangle)] extern "C" fn quad_int8x4_t(x: Quad>) -> Quad> { x } - // CHECK: define [4 x <2 x i16>] @quad_int16x2_t([4 x <2 x i16>] {{.*}} %0) + // CHECK: define [2 x i64] @quad_int16x2_t([2 x i64] {{.*}} %0) #[unsafe(no_mangle)] extern "C" fn quad_int16x2_t(x: Quad>) -> Quad> { x } - // CHECK: define [4 x <1 x i32>] @quad_int32x1_t([4 x <1 x i32>] {{.*}} %0) + // CHECK: define [2 x i64] @quad_int32x1_t([2 x i64] {{.*}} %0) #[unsafe(no_mangle)] extern "C" fn quad_int32x1_t(x: Quad>) -> Quad> { x } - // CHECK: define [4 x <2 x half>] @quad_float16x2_t([4 x <2 x half>] {{.*}} %0) + // CHECK: define [2 x i64] @quad_float16x2_t([2 x i64] {{.*}} %0) #[unsafe(no_mangle)] extern "C" fn quad_float16x2_t(x: Quad>) -> Quad> { x } - // CHECK: define [4 x <1 x float>] @quad_float32x1_t([4 x <1 x float>] {{.*}} %0) + // CHECK: define [2 x i64] @quad_float32x1_t([2 x i64] {{.*}} %0) #[unsafe(no_mangle)] extern "C" fn quad_float32x1_t(x: Quad>) -> Quad> { x } // When it doesn't quite fit, padding is added which does erase the type. @@ -74,23 +75,23 @@ mod tests { // CHECK: define [2 x i64] @triple_int8x4_t #[unsafe(no_mangle)] extern "C" fn triple_int8x4_t(x: Triple>) -> Triple> { x } - // Other configurations are not passed by-value but indirectly. + // Other configurations passed directly when they qualify as a homogeneous aggregate. - // CHECK: define void @pair_int128x1_t + // CHECK: define [2 x <1 x i128>] @pair_int128x1_t([2 x <1 x i128>] #[unsafe(no_mangle)] extern "C" fn pair_int128x1_t(x: Pair>) -> Pair> { x } - // CHECK: define void @pair_float128x1_t + // CHECK: define [2 x <1 x fp128>] @pair_float128x1_t([2 x <1 x fp128>] #[unsafe(no_mangle)] extern "C" fn pair_float128x1_t(x: Pair>) -> Pair> { x } - // CHECK: define void @pair_int8x16_t + // CHECK: define [2 x <16 x i8>] @pair_int8x16_t([2 x <16 x i8>] #[unsafe(no_mangle)] extern "C" fn pair_int8x16_t(x: Pair>) -> Pair> { x } - // CHECK: define void @pair_int16x8_t + // CHECK: define [2 x <8 x i16>] @pair_int16x8_t([2 x <8 x i16>] #[unsafe(no_mangle)] extern "C" fn pair_int16x8_t(x: Pair>) -> Pair> { x } - // CHECK: define void @triple_int16x8_t + // CHECK: define [3 x <8 x i16>] @triple_int16x8_t([3 x <8 x i16>] #[unsafe(no_mangle)] extern "C" fn triple_int16x8_t(x: Triple>) -> Triple> { x } - // CHECK: define void @quad_int16x8_t + // CHECK: define [4 x <8 x i16>] @quad_int16x8_t([4 x <8 x i16>] #[unsafe(no_mangle)] extern "C" fn quad_int16x8_t(x: Quad>) -> Quad> { x } }