diff --git a/library/core/src/mem/maybe_uninit.rs b/library/core/src/mem/maybe_uninit.rs index 9889f32c502ad..9bc8b0d128d2b 100644 --- a/library/core/src/mem/maybe_uninit.rs +++ b/library/core/src/mem/maybe_uninit.rs @@ -1,7 +1,7 @@ use crate::any::type_name; use crate::clone::TrivialClone; use crate::marker::Destruct; -use crate::mem::ManuallyDrop; +use crate::mem::{ManuallyDrop, transmute_neo}; use crate::{fmt, intrinsics, ptr, slice}; /// A wrapper type to construct uninitialized instances of `T`. @@ -724,9 +724,9 @@ impl MaybeUninit { // This also means that `self` must be a `value` variant. unsafe { intrinsics::assert_inhabited::(); - // We do this via a raw ptr read instead of `ManuallyDrop::into_inner` so that there's + // We do this via a transmute instead of `ManuallyDrop::into_inner` so that there's // no trace of `ManuallyDrop` in Miri's error messages here. - (&raw const self.value).cast::().read() + transmute_neo(self) } } diff --git a/library/core/src/mem/mod.rs b/library/core/src/mem/mod.rs index 828572df6968b..a00a6c4783ba6 100644 --- a/library/core/src/mem/mod.rs +++ b/library/core/src/mem/mod.rs @@ -1196,6 +1196,9 @@ pub const unsafe fn transmute_prefix(src: Src) -> Dst { /// New version of `transmute`, exposed under this name so it can be iterated upon /// without risking breakage to uses of "real" transmute. /// +/// Uses a `const`-`assert` to check the sizes instead of typeck hacks, +/// but is semantially identical to `transmute` otherwise. +/// /// It will not be stabilized under this name. /// /// # Examples @@ -1214,6 +1217,8 @@ pub const unsafe fn transmute_prefix(src: Src) -> Dst { /// unsafe { transmute_neo::(123) }; /// ``` #[unstable(feature = "transmute_neo", issue = "155079")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +#[inline] pub const unsafe fn transmute_neo(src: Src) -> Dst { const { assert!(Src::SIZE == Dst::SIZE) }; diff --git a/tests/codegen-llvm/read_write_unaligned.rs b/tests/codegen-llvm/read_write_unaligned.rs new file mode 100644 index 0000000000000..beba4400dc54a --- /dev/null +++ b/tests/codegen-llvm/read_write_unaligned.rs @@ -0,0 +1,71 @@ +//@ compile-flags: -Copt-level=3 +//@ only-64bit + +#![crate_type = "lib"] + +use std::num::NonZero; +use std::ptr::NonNull; + +// CHECK-LABEL: nonnull ptr @read_unaligned_ptr(ptr{{.+}}%ptr) +#[no_mangle] +unsafe fn read_unaligned_ptr(ptr: *const NonNull) -> NonNull { + // CHECK: start: + // CHECK-NEXT: [[TEMP:%.+]] = load ptr, ptr %ptr, align 1 + // CHECK-SAME: !nonnull + // CHECK-SAME: !noundef + // CHECK-NEXT: ret ptr [[TEMP]] + ptr.read_unaligned() +} + +// CHECK-LABEL: range(i16 1, 0) i16 @read_unaligned_i16(ptr{{.+}}%ptr) +#[no_mangle] +unsafe fn read_unaligned_i16(ptr: *const NonZero) -> NonZero { + // CHECK: start: + // CHECK-NEXT: [[TEMP:%.+]] = load i16, ptr %ptr, align 1 + // CHECK-NOT: !noundef + // CHECK-NOT: !range + // CHECK-NEXT: ret i16 [[TEMP]] + ptr.read_unaligned() +} + +// CHECK-LABEL: void @typed_copy_unaligned_i32(ptr{{.+}}%src, ptr{{.+}}%dst) +#[no_mangle] +unsafe fn typed_copy_unaligned_i32(src: *const NonZero, dst: *mut NonZero) { + // CHECK: start: + // CHECK-NEXT: [[TEMP:%.+]] = load i32, ptr %src, align 1 + // CHECK-NOT: !noundef + // CHECK-NOT: !range + // CHECK-NEXT: store i32 [[TEMP]], ptr %dst, align 1 + // CHECK-NEXT: ret void + dst.write_unaligned(src.read_unaligned()) +} + +// CHECK-LABEL: void @write_unaligned_i64(ptr{{.+}}%ptr, i64{{.+}}%val) +#[no_mangle] +unsafe fn write_unaligned_i64(ptr: *mut NonZero, val: NonZero) { + // CHECK: start: + // CHECK-NEXT: store i64 %val, ptr %ptr, align 1 + // CHECK-NEXT: ret void + ptr.write_unaligned(val) +} + +#[repr(align(128))] +struct HugeBuffer([u64; 1 << 10]); + +// CHECK-LABEL: void @read_unaligned_huge(ptr{{.+}}%_0, ptr{{.+}}%ptr) +#[no_mangle] +unsafe fn read_unaligned_huge(ptr: *const HugeBuffer) -> HugeBuffer { + // CHECK: start: + // CHECK-NEXT: call void @llvm.memcpy{{.+}} align 128 dereferenceable(8192) %_0, {{.+}} align 1 dereferenceable(8192) %ptr, i64 8192, + // CHECK-NEXT: ret void + ptr.read_unaligned() +} + +// CHECK-LABEL: void @write_unaligned_huge(ptr{{.+}}%ptr, ptr{{.+}}%val) +#[no_mangle] +unsafe fn write_unaligned_huge(ptr: *mut HugeBuffer, val: HugeBuffer) { + // CHECK: start: + // CHECK-NEXT: call void @llvm.memcpy{{.+}} align 1 dereferenceable(8192) %ptr, {{.+}} align 128 dereferenceable(8192) %val, i64 8192, + // CHECK-NEXT: ret void + ptr.write_unaligned(val) +} diff --git a/tests/mir-opt/pre-codegen/unaligned.rs b/tests/mir-opt/pre-codegen/unaligned.rs new file mode 100644 index 0000000000000..70fbc373ca499 --- /dev/null +++ b/tests/mir-opt/pre-codegen/unaligned.rs @@ -0,0 +1,37 @@ +//@ compile-flags: -O -Zmir-opt-level=2 +//@ ignore-std-debug-assertions (there's one in `ptr::read` via `MaybeUninit::assume_init`) + +#![crate_type = "lib"] + +// EMIT_MIR unaligned.unaligned_copy_manual.runtime-optimized.after.mir +pub unsafe fn unaligned_copy_manual(src: *const u128, dst: *mut u128) { + #[repr(packed)] + struct Packed(T); + + // CHECK-LABEL: fn unaligned_copy_manual(_1: *const u128, _2: *mut u128) -> () + // CHECK: [[SRCU:_.+]] = copy _1 as *const unaligned_copy_manual::Packed (PtrToPtr); + // CHECK: [[DSTU:_.+]] = copy _2 as *mut unaligned_copy_manual::Packed (PtrToPtr); + // CHECK: [[TEMP:_.+]] = copy ((*[[SRCU]]).0: u128); + // ((*[[DSTU]]).0: u128) = move [[TEMP]]; + let src = src.cast::>(); + let dst = dst.cast::>(); + unsafe { (*dst).0 = (*src).0 }; +} + +// EMIT_MIR unaligned.unaligned_copy_generic.runtime-optimized.after.mir +pub unsafe fn unaligned_copy_generic(src: *const T, dst: *mut T) { + // CHECK-LABEL: fn unaligned_copy_generic( + // CHECK: debug src => _1; + // CHECK: debug dst => _2; + // CHECK: debug val => [[VAL:_.+]]; + // CHECK: copy _1 as *const u8 (PtrToPtr); + // CHECK: &raw mut + // CHECK: copy_nonoverlapping{{.+}}count = const ::SIZE + // CHECK: &raw const + // CHECK: copy _2 as *mut u8 (PtrToPtr); + // CHECK: copy_nonoverlapping{{.+}}count = const ::SIZE + unsafe { + let val = std::ptr::read_unaligned(src); + std::ptr::write_unaligned(dst, val); + } +} diff --git a/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir new file mode 100644 index 0000000000000..ad474ba1edb73 --- /dev/null +++ b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir @@ -0,0 +1,99 @@ +// MIR for `unaligned_copy_generic` after runtime-optimized + +fn unaligned_copy_generic(_1: *const T, _2: *mut T) -> () { + debug src => _1; + debug dst => _2; + let mut _0: (); + let _9: T; + let mut _10: T; + scope 1 { + debug val => _9; + scope 14 (inlined #[track_caller] write_unaligned::) { + let mut _11: *const T; + let mut _12: *const u8; + let mut _13: *mut u8; + scope 15 (inlined std::mem::size_of::) { + } + scope 16 (inlined std::ptr::copy_nonoverlapping::) { + scope 17 (inlined core::ub_checks::check_language_ub) { + scope 18 (inlined core::ub_checks::check_language_ub::runtime) { + } + } + scope 19 (inlined std::mem::size_of::) { + } + scope 20 (inlined std::mem::align_of::) { + } + } + } + } + scope 2 (inlined #[track_caller] read_unaligned::) { + let mut _3: std::mem::MaybeUninit; + let mut _4: *const u8; + let mut _6: *mut u8; + let mut _7: std::mem::MaybeUninit; + scope 3 { + scope 5 (inlined MaybeUninit::::as_mut_ptr) { + let mut _5: *mut std::mem::MaybeUninit; + } + scope 6 (inlined std::mem::size_of::) { + } + scope 7 (inlined std::ptr::copy_nonoverlapping::) { + scope 8 (inlined core::ub_checks::check_language_ub) { + scope 9 (inlined core::ub_checks::check_language_ub::runtime) { + } + } + scope 10 (inlined std::mem::size_of::) { + } + scope 11 (inlined std::mem::align_of::) { + } + } + scope 12 (inlined #[track_caller] MaybeUninit::::assume_init) { + let _8: (); + scope 13 (inlined transmute_neo::, T>) { + } + } + } + scope 4 (inlined MaybeUninit::::uninit) { + } + } + + bb0: { + StorageLive(_9); + StorageLive(_5); + StorageLive(_3); + _3 = MaybeUninit:: { uninit: const () }; + StorageLive(_4); + _4 = copy _1 as *const u8 (PtrToPtr); + StorageLive(_6); + _5 = &raw mut _3; + _6 = copy _5 as *mut u8 (PtrToPtr); + copy_nonoverlapping(dst = copy _6, src = copy _4, count = const ::SIZE); + StorageDead(_6); + StorageDead(_4); + StorageLive(_7); + _7 = move _3; + _8 = assert_inhabited::() -> [return: bb1, unwind unreachable]; + } + + bb1: { + _9 = copy _7 as T (Transmute); + StorageDead(_7); + StorageDead(_3); + StorageDead(_5); + StorageLive(_10); + _10 = move _9; + StorageLive(_11); + StorageLive(_12); + _11 = &raw const _10; + _12 = copy _11 as *const u8 (PtrToPtr); + StorageLive(_13); + _13 = copy _2 as *mut u8 (PtrToPtr); + copy_nonoverlapping(dst = copy _13, src = copy _12, count = const ::SIZE); + StorageDead(_13); + StorageDead(_12); + StorageDead(_11); + StorageDead(_10); + StorageDead(_9); + return; + } +} diff --git a/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir new file mode 100644 index 0000000000000..450df935a0fb3 --- /dev/null +++ b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir @@ -0,0 +1,34 @@ +// MIR for `unaligned_copy_manual` after runtime-optimized + +fn unaligned_copy_manual(_1: *const u128, _2: *mut u128) -> () { + debug src => _1; + debug dst => _2; + let mut _0: (); + let _3: *const unaligned_copy_manual::Packed; + let mut _5: u128; + scope 1 { + debug src => _3; + let _4: *mut unaligned_copy_manual::Packed; + scope 2 { + debug dst => _4; + } + scope 4 (inlined std::ptr::mut_ptr::::cast::>) { + } + } + scope 3 (inlined std::ptr::const_ptr::::cast::>) { + } + + bb0: { + StorageLive(_3); + _3 = copy _1 as *const unaligned_copy_manual::Packed (PtrToPtr); + StorageLive(_4); + _4 = copy _2 as *mut unaligned_copy_manual::Packed (PtrToPtr); + StorageLive(_5); + _5 = copy ((*_3).0: u128); + ((*_4).0: u128) = move _5; + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + return; + } +}