[PAC] Transmutes (5/8)

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1843,6 +1843,17 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     fn fptosi_sat(&mut self, val: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
         self.fptoint_sat(true, val, dest_ty)
     }
+
+    fn ptrauth_resign(
+        &mut self,
+        _value: Self::Value,
+        _old_key: u32,
+        _old_discriminator: u64,
+        _new_key: u32,
+        _new_discriminator: u64,
+    ) -> Self::Value {
+        bug!("Resigning of pointers not implemented");
+    }
 }
 
 impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {

compiler/rustc_codegen_gcc/src/common.rs

@@ -1,9 +1,10 @@
 use gccjit::{GlobalKind, LValue, RValue, ToRValue, Type};
 use rustc_abi::Primitive::Pointer;
-use rustc_abi::{self as abi, HasDataLayout};
+use rustc_abi::{self as abi, HasDataLayout, Size};
 use rustc_codegen_ssa::traits::{
     BaseTypeCodegenMethods, ConstCodegenMethods, MiscCodegenMethods, StaticCodegenMethods,
 };
+use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::mir::Mutability;
 use rustc_middle::mir::interpret::{GlobalAlloc, PointerArithmetic, Scalar};
 use rustc_middle::ty::layout::LayoutOf;
@@ -115,7 +116,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             Mutability::Mut => {
                 self.static_addr_of_mut(const_alloc_to_gcc(self, alloc), alloc.inner().align, None)
             }
-            _ => self.static_addr_of(alloc, None),
+            _ => self.static_addr_of(alloc, None, None),
         };
         if !self.sess().fewer_names() {
             // FIXME(antoyo): set value name.
@@ -323,7 +324,8 @@ impl<'gcc, 'tcx> ConstCodegenMethods for CodegenCx<'gcc, 'tcx> {
         cv: Scalar,
         layout: abi::Scalar,
         ty: Type<'gcc>,
-        _schema: Option<&PointerAuthSchema>,
+        _ptrauth_schema: Option<PointerAuthSchema>,
+        _ptrauth_discriminators: Option<&FxHashMap<Size, u64>>,
     ) -> RValue<'gcc> {
         let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() };
         match cv {

compiler/rustc_codegen_gcc/src/consts.rs

@@ -5,6 +5,7 @@ use rustc_abi::{self as abi, Align, HasDataLayout, Primitive, Size, WrappingRang
 use rustc_codegen_ssa::traits::{
     BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods,
 };
+use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::attrs::Linkage;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LOCAL_CRATE;
@@ -56,7 +57,12 @@ fn set_global_alignment<'gcc, 'tcx>(
 }
 
 impl<'gcc, 'tcx> StaticCodegenMethods for CodegenCx<'gcc, 'tcx> {
-    fn static_addr_of(&self, alloc: ConstAllocation<'_>, kind: Option<&str>) -> RValue<'gcc> {
+    fn static_addr_of(
+        &self,
+        alloc: ConstAllocation<'_>,
+        kind: Option<&str>,
+        _ptrauth_discriminators: Option<&FxHashMap<Size, u64>>,
+    ) -> RValue<'gcc> {
         let cv = const_alloc_to_gcc(self, alloc);
         let align = alloc.inner().align;
 

compiler/rustc_codegen_gcc/src/context.rs

@@ -405,7 +405,7 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     fn get_fn_addr(
         &self,
         instance: Instance<'tcx>,
-        _pointer_auth_schema: Option<&PointerAuthSchema>,
+        _ptrauth_schema: Option<PointerAuthSchema>,
     ) -> RValue<'gcc> {
         let func_name = self.tcx.symbol_name(instance).name;
 

compiler/rustc_codegen_gcc/src/int.rs

@@ -375,6 +375,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             fixed_count: 3,
             conv: CanonAbi::C,
             can_unwind: false,
+            ptrauth_discriminator: 0,
         };
         fn_abi.adjust_for_foreign_abi(self.cx, ExternAbi::C { unwind: false });
 

compiler/rustc_codegen_llvm/src/asm.rs

@@ -164,7 +164,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     ConstScalar::Ptr(ptr, _) => {
                         let (prov, _) = ptr.prov_and_relative_offset();
                         let global_alloc = self.tcx.global_alloc(prov.alloc_id());
-                        let value = self.cx.alloc_to_backend(global_alloc, false, None).unwrap();
+                        let value =
+                            self.cx.alloc_to_backend(global_alloc, false, None, None).unwrap();
                         inputs.push(value);
                         op_idx.insert(idx, constraints.len());
                         constraints.push("s".to_string());
@@ -453,8 +454,9 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
                                 ConstScalar::Ptr(ptr, _) => {
                                     let (prov, offset) = ptr.prov_and_relative_offset();
                                     let global_alloc = self.tcx.global_alloc(prov.alloc_id());
-                                    let llval =
-                                        self.alloc_to_backend(global_alloc, true, None).unwrap();
+                                    let llval = self
+                                        .alloc_to_backend(global_alloc, true, None, None)
+                                        .unwrap();
 
                                     self.add_compiler_used_global(llval);
                                     let symbol = llvm::build_string(|s| unsafe {

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1542,6 +1542,30 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
         attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
     }
+
+    fn ptrauth_resign(
+        &mut self,
+        value: &'ll Value,
+        old_key: u32,
+        old_discriminator: u64,
+        new_key: u32,
+        new_discriminator: u64,
+    ) -> &'ll Value {
+        let ptr_as_int = self.ptrtoint(value, self.type_i64());
+        let resigned_int = self.call_intrinsic(
+            "llvm.ptrauth.resign",
+            &[],
+            &[
+                ptr_as_int,
+                self.const_i32(old_key as i32),
+                self.const_i64(old_discriminator as i64),
+                self.const_i32(new_key as i32),
+                self.const_i64(new_discriminator as i64),
+            ],
+        );
+
+        self.inttoptr(resigned_int, self.val_ty(value))
+    }
 }
 
 impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
@@ -2059,8 +2083,12 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         // bundles.
         // Once this is resolved, we should analyze each call and skip direct calls. See the
         // discussion in the rust-lang issue: <https://github.com/rust-lang/rust/issues/152532>
-        let key: u32 = 0;
-        let discriminator: u64 = 0;
+
+        let key: u32 = self.sess().pointer_authentication_fn_ptr_key().unwrap() as u32;
+        // If sess().pointer_authentication_fn_ptr_type_discrimination() is true, this contains
+        // the function pointer type discriminator; otherwise, it is 0.
+        let discriminator = fn_abi?.ptrauth_discriminator;
+
         Some(llvm::OperandBundleBox::new(
             "ptrauth",
             &[self.const_u32(key), self.const_u64(discriminator)],

compiler/rustc_codegen_llvm/src/common.rs

@@ -4,10 +4,11 @@ use std::borrow::Borrow;
 
 use libc::{c_char, c_uint};
 use rustc_abi::Primitive::Pointer;
-use rustc_abi::{self as abi, ExternAbi, HasDataLayout as _};
+use rustc_abi::{self as abi, ExternAbi, HasDataLayout as _, Size};
 use rustc_ast::Mutability;
 use rustc_codegen_ssa::common::TypeKind;
 use rustc_codegen_ssa::traits::*;
+use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hash::{StableHash, StableHasher};
 use rustc_hashes::Hash128;
 use rustc_hir::def::DefKind;
@@ -30,11 +31,9 @@ pub(crate) fn maybe_sign_fn_ptr<'ll, 'tcx>(
     cx: &CodegenCx<'ll, '_>,
     instance: Instance<'tcx>,
     llfn: &'ll llvm::Value,
-    schema: &PointerAuthSchema,
+    ptrauth_schema: PointerAuthSchema,
 ) -> &'ll llvm::Value {
-    if cx.tcx.sess.pointer_authentication_functions().is_none() {
-        return llfn;
-    }
+    assert!(cx.tcx.sess.pointer_authentication_functions().is_some());
 
     // Only free functions or methods
     let def_id = instance.def_id();
@@ -54,7 +53,7 @@ pub(crate) fn maybe_sign_fn_ptr<'ll, 'tcx>(
         return llfn;
     }
 
-    let addr_diversity = match schema.is_address_discriminated {
+    let addr_diversity = match ptrauth_schema.is_address_discriminated {
         PointerAuthAddressDiscriminator::HardwareAddress(true) => Some(llfn),
         PointerAuthAddressDiscriminator::HardwareAddress(false) => None,
         PointerAuthAddressDiscriminator::Synthetic(val) => {
@@ -63,7 +62,12 @@ pub(crate) fn maybe_sign_fn_ptr<'ll, 'tcx>(
             Some(unsafe { llvm::LLVMConstIntToPtr(llval, llty) })
         }
     };
-    const_ptr_auth(llfn, schema.key as u32, schema.constant_discriminator as u64, addr_diversity)
+    const_ptr_auth(
+        llfn,
+        ptrauth_schema.key as u32,
+        ptrauth_schema.constant_discriminator as u64,
+        addr_diversity,
+    )
 }
 
 /*
@@ -179,11 +183,12 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         &self,
         global_alloc: GlobalAlloc<'tcx>,
         need_symbol_name: bool,
-        schema: Option<&PointerAuthSchema>,
+        ptrauth_schema: Option<PointerAuthSchema>,
+        ptrauth_discriminators: Option<&FxHashMap<Size, u64>>,
     ) -> Result<&'ll Value, u64> {
         let alloc = match global_alloc {
             GlobalAlloc::Function { instance, .. } => {
-                return Ok(self.get_fn_addr(instance, schema));
+                return Ok(self.get_fn_addr(instance, ptrauth_schema));
             }
             GlobalAlloc::Static(def_id) => {
                 assert!(self.tcx.is_static(def_id));
@@ -226,7 +231,13 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
             }
         };
 
-        let init = const_alloc_to_llvm(self, alloc.inner(), IsStatic::No, IsInitOrFini::No);
+        let init = const_alloc_to_llvm(
+            self,
+            alloc.inner(),
+            IsStatic::No,
+            IsInitOrFini::No,
+            ptrauth_discriminators,
+        );
         let alloc = alloc.inner();
 
         if need_symbol_name {
@@ -405,7 +416,8 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
         cv: Scalar,
         layout: abi::Scalar,
         llty: &'ll Type,
-        schema: Option<&PointerAuthSchema>,
+        ptrauth_schema: Option<PointerAuthSchema>,
+        ptrauth_discriminators: Option<&FxHashMap<Size, u64>>,
     ) -> &'ll Value {
         let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() };
         match cv {
@@ -422,7 +434,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
                 let (prov, offset) = ptr.prov_and_relative_offset();
                 let global_alloc = self.tcx.global_alloc(prov.alloc_id());
                 let base_addr_space = global_alloc.address_space(self);
-                let base_addr = match self.alloc_to_backend(global_alloc, false, schema) {
+                let base_addr = match self.alloc_to_backend(
+                    global_alloc,
+                    false,
+                    ptrauth_schema,
+                    ptrauth_discriminators,
+                ) {
                     Ok(base_addr) => base_addr,
                     Err(base_addr) => {
                         let val = base_addr.wrapping_add(offset.bytes());

compiler/rustc_codegen_llvm/src/consts.rs

@@ -3,6 +3,7 @@ use std::ops::Range;
 use rustc_abi::{Align, ExternAbi, HasDataLayout, Primitive, Scalar, Size, WrappingRange};
 use rustc_codegen_ssa::common;
 use rustc_codegen_ssa::traits::*;
+use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::LangItem;
 use rustc_hir::attrs::Linkage;
 use rustc_hir::def::DefKind;
@@ -13,6 +14,7 @@ use rustc_middle::mir::interpret::{
     read_target_uint,
 };
 use rustc_middle::mono::MonoItem;
+use rustc_middle::ptrauth::ptrauth_collect_fn_ptr_discriminators;
 use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
 use rustc_middle::ty::{self, Instance};
 use rustc_middle::{bug, span_bug};
@@ -37,11 +39,13 @@ pub(crate) enum IsInitOrFini {
     Yes,
     No,
 }
+
 pub(crate) fn const_alloc_to_llvm<'ll>(
     cx: &CodegenCx<'ll, '_>,
     alloc: &Allocation,
     is_static: IsStatic,
     is_init_fini: IsInitOrFini,
+    ptrauth_discriminators: Option<&FxHashMap<Size, u64>>,
 ) -> &'ll Value {
     // We expect that callers of const_alloc_to_llvm will instead directly codegen a pointer or
     // integer for any &ZST where the ZST is a constant (i.e. not a static). We should never be
@@ -121,14 +125,24 @@ pub(crate) fn const_alloc_to_llvm<'ll>(
             as u64;
 
         let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx);
-        let schema = if cx.sess().pointer_authentication() {
+        let mut schema = if cx.sess().pointer_authentication() {
             match is_init_fini {
                 IsInitOrFini::Yes => cx.sess().pointer_authentication_init_fini(),
                 IsInitOrFini::No => cx.sess().pointer_authentication_functions(),
             }
         } else {
             None
         };
+        let discr =
+            ptrauth_discriminators.as_ref().and_then(|m| m.get(&Size::from_bytes(offset as u64)));
+
+        // Init/fini entries must not participate in function pointer type discrimination, they use
+        // a dedicated constant value (ptrauth_string_discriminator("init_fini") which is: 0xd9d4).
+        if let (Some(schema), Some(discr)) = (schema.as_mut(), discr)
+            && is_init_fini == IsInitOrFini::No
+        {
+            schema.constant_discriminator = *discr as u16;
+        }
         llvals.push(cx.scalar_to_backend_with_pac(
             InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx),
             Scalar::Initialized {
@@ -137,6 +151,7 @@ pub(crate) fn const_alloc_to_llvm<'ll>(
             },
             cx.type_ptr_ext(address_space),
             schema,
+            ptrauth_discriminators,
         ));
         next_offset = offset + pointer_size_bytes;
     }
@@ -160,6 +175,15 @@ fn codegen_static_initializer<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     def_id: DefId,
 ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> {
+    let ptrauth_discriminators = if cx.sess().pointer_authentication_fn_ptr_type_discrimination() {
+        let instance = Instance::mono(cx.tcx, def_id);
+        let ty = instance.ty(cx.tcx, cx.typing_env());
+
+        Some(ptrauth_collect_fn_ptr_discriminators(cx.tcx, cx.typing_env(), ty))
+    } else {
+        None
+    };
+
     let alloc = cx.tcx.eval_static_initializer(def_id)?;
     let attrs = cx.tcx.codegen_fn_attrs(def_id);
     // FIXME(jchlanda) Decide if this could be better served by `ctor` crate. See the discussion
@@ -175,7 +199,16 @@ fn codegen_static_initializer<'ll, 'tcx>(
             }
         })
         .unwrap_or(IsInitOrFini::No);
-    Ok((const_alloc_to_llvm(cx, alloc.inner(), IsStatic::Yes, is_in_init_fini), alloc))
+    Ok((
+        const_alloc_to_llvm(
+            cx,
+            alloc.inner(),
+            IsStatic::Yes,
+            is_in_init_fini,
+            ptrauth_discriminators.as_ref(),
+        ),
+        alloc,
+    ))
 }
 
 fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) {
@@ -834,10 +867,21 @@ impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> {
     ///
     /// The pointer will always be in the default address space. If global variables default to a
     /// different address space, an addrspacecast is inserted.
-    fn static_addr_of(&self, alloc: ConstAllocation<'_>, kind: Option<&str>) -> &'ll Value {
+    fn static_addr_of(
+        &self,
+        alloc: ConstAllocation<'_>,
+        kind: Option<&str>,
+        ptrauth_discriminators: Option<&FxHashMap<Size, u64>>,
+    ) -> &'ll Value {
         // FIXME: should we cache `const_alloc_to_llvm` to avoid repeating this for the
         // same `ConstAllocation`?
-        let cv = const_alloc_to_llvm(self, alloc.inner(), IsStatic::No, IsInitOrFini::No);
+        let cv = const_alloc_to_llvm(
+            self,
+            alloc.inner(),
+            IsStatic::No,
+            IsInitOrFini::No,
+            ptrauth_discriminators,
+        );
 
         let gv = self.static_addr_of_impl(cv, alloc.inner().align, kind);
         // static_addr_of_impl returns the bare global variable, which might not be in the default

compiler/rustc_codegen_llvm/src/context.rs

@@ -917,7 +917,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn get_fn_addr(
         &self,
         instance: Instance<'tcx>,
-        pointer_auth_schema: Option<&PointerAuthSchema>,
+        ptrauth_schema: Option<PointerAuthSchema>,
     ) -> &'ll Value {
         // When pointer authentication metadata is provided, `get_fn_addr` will
         // attempt to sign the pointer using LLVM's `ConstPtrAuth` constant
@@ -932,7 +932,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         // <https://github.com/rust-lang/rust/issues/152532>, and comment in
         // builder's `ptrauth_operand_bundle`.
         let llfn = get_fn(self, instance);
-        match pointer_auth_schema {
+        match ptrauth_schema {
             Some(schema) => common::maybe_sign_fn_ptr(self, instance, llfn, schema),
             None => llfn,
         }