diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index c0593bb467796..1dd460c409737 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -324,9 +324,33 @@ impl CodegenBackend for LlvmCodegenBackend { target_config(sess) } + /// Intrinsics whose fallback body will not be used by the LLVM backend. fn replaced_intrinsics(&self) -> Vec { - let mut will_not_use_fallback = - vec![sym::unchecked_funnel_shl, sym::unchecked_funnel_shr, sym::carrying_mul_add]; + #[rustfmt::skip] + let mut will_not_use_fallback = vec![ + // These are mapped to LLVM intrinsics instead. + sym::unchecked_funnel_shl, + sym::unchecked_funnel_shr, + sym::carrying_mul_add, + + // Fallback via libm, but the LLVM intrinsic is used instead. + sym::sinf16, sym::sinf32, sym::sinf64, + sym::cosf16, sym::cosf32, sym::cosf64, + sym::powf16, sym::powf32, sym::powf64, + sym::expf16, sym::expf32, sym::expf64, + sym::exp2f16, sym::exp2f32, sym::exp2f64, + sym::logf16, sym::logf32, sym::logf64, + sym::log10f16, sym::log10f32, sym::log10f64, + sym::log2f16, sym::log2f32, sym::log2f64, + + // Fallback via f32 or f64, but the LLVM intrinsic is used instead. + sym::floorf16, sym::ceilf16, sym::truncf16, + sym::round_ties_even_f16, sym::roundf16, + sym::sqrtf16, sym::powif16, + sym::fmaf16, + + sym::copysignf16, sym::copysignf32, sym::copysignf64, sym::copysignf128, + ]; if llvm_util::get_version() >= (22, 0, 0) { will_not_use_fallback.push(sym::carryless_mul); diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index c5da5055e16d9..db825a1c970db 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -1026,9 +1026,12 @@ pub unsafe fn unaligned_volatile_store(dst: *mut T, val: T); /// /// The stabilized version of this intrinsic is /// [`f16::sqrt`](../../std/primitive.f16.html#method.sqrt) +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub fn sqrtf16(x: f16) -> f16; +pub fn sqrtf16(x: f16) -> f16 { + sqrtf32(x as f32) as f16 +} /// Returns the square root of an `f32` /// /// The stabilized version of this intrinsic is @@ -1055,9 +1058,12 @@ pub fn sqrtf128(x: f128) -> f128; /// /// The stabilized version of this intrinsic is /// [`f16::powi`](../../std/primitive.f16.html#method.powi) +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub fn powif16(a: f16, x: i32) -> f16; +pub fn powif16(a: f16, x: i32) -> f16 { + powif32(a as f32, x) as f16 +} /// Raises an `f32` to an integer power. /// /// The stabilized version of this intrinsic is @@ -1437,9 +1443,14 @@ pub fn log2f128(x: f128) -> f128 { /// The stabilized version of this intrinsic is /// [`f16::mul_add`](../../std/primitive.f16.html#method.mul_add) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn fmaf16(a: f16, b: f16, c: f16) -> f16; +pub const fn fmaf16(a: f16, b: f16, c: f16) -> f16 { + // NOTE: f32 does not have sufficient precision, so use f64 instead. + // see also https://github.com/llvm/llvm-project/issues/128450#issuecomment-2727540179. + fmaf64(a as f64, b as f64, c as f64) as f16 +} /// Returns `a * b + c` without rounding the intermediate result for `f32` values. /// /// The stabilized version of this intrinsic is @@ -1535,9 +1546,12 @@ pub const fn fmuladdf128(a: f128, b: f128, c: f128) -> f128 { /// The stabilized version of this intrinsic is /// [`f16::floor`](../../std/primitive.f16.html#method.floor) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn floorf16(x: f16) -> f16; +pub const fn floorf16(x: f16) -> f16 { + floorf32(x as f32) as f16 +} /// Returns the largest integer less than or equal to an `f32`. /// /// The stabilized version of this intrinsic is @@ -1568,9 +1582,12 @@ pub const fn floorf128(x: f128) -> f128; /// The stabilized version of this intrinsic is /// [`f16::ceil`](../../std/primitive.f16.html#method.ceil) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn ceilf16(x: f16) -> f16; +pub const fn ceilf16(x: f16) -> f16 { + ceilf32(x as f32) as f16 +} /// Returns the smallest integer greater than or equal to an `f32`. /// /// The stabilized version of this intrinsic is @@ -1601,9 +1618,12 @@ pub const fn ceilf128(x: f128) -> f128; /// The stabilized version of this intrinsic is /// [`f16::trunc`](../../std/primitive.f16.html#method.trunc) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn truncf16(x: f16) -> f16; +pub const fn truncf16(x: f16) -> f16 { + truncf32(x as f32) as f16 +} /// Returns the integer part of an `f32`. /// /// The stabilized version of this intrinsic is @@ -1635,9 +1655,12 @@ pub const fn truncf128(x: f128) -> f128; /// The stabilized version of this intrinsic is /// [`f16::round_ties_even`](../../std/primitive.f16.html#method.round_ties_even) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn round_ties_even_f16(x: f16) -> f16; +pub const fn round_ties_even_f16(x: f16) -> f16 { + round_ties_even_f32(x as f32) as f16 +} /// Returns the nearest integer to an `f32`. Rounds half-way cases to the number with an even /// least significant digit. @@ -1674,9 +1697,12 @@ pub const fn round_ties_even_f128(x: f128) -> f128; /// The stabilized version of this intrinsic is /// [`f16::round`](../../std/primitive.f16.html#method.round) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn roundf16(x: f16) -> f16; +pub const fn roundf16(x: f16) -> f16 { + roundf32(x as f32) as f16 +} /// Returns the nearest integer to an `f32`. Rounds half-way cases away from zero. /// /// The stabilized version of this intrinsic is @@ -3610,34 +3636,46 @@ pub const fn fabs(x: T) -> T; /// /// The stabilized version of this intrinsic is /// [`f16::copysign`](../../std/primitive.f16.html#method.copysign) +#[inline] #[rustc_nounwind] #[rustc_intrinsic] -pub const fn copysignf16(x: f16, y: f16) -> f16; +pub const fn copysignf16(x: f16, y: f16) -> f16 { + f16::from_bits((x.to_bits() & !f16::SIGN_MASK) | (y.to_bits() & f16::SIGN_MASK)) +} /// Copies the sign from `y` to `x` for `f32` values. /// /// The stabilized version of this intrinsic is /// [`f32::copysign`](../../std/primitive.f32.html#method.copysign) +#[inline] #[rustc_nounwind] #[rustc_intrinsic_const_stable_indirect] #[rustc_intrinsic] -pub const fn copysignf32(x: f32, y: f32) -> f32; +pub const fn copysignf32(x: f32, y: f32) -> f32 { + f32::from_bits((x.to_bits() & !f32::SIGN_MASK) | (y.to_bits() & f32::SIGN_MASK)) +} /// Copies the sign from `y` to `x` for `f64` values. /// /// The stabilized version of this intrinsic is /// [`f64::copysign`](../../std/primitive.f64.html#method.copysign) +#[inline] #[rustc_nounwind] #[rustc_intrinsic_const_stable_indirect] #[rustc_intrinsic] -pub const fn copysignf64(x: f64, y: f64) -> f64; +pub const fn copysignf64(x: f64, y: f64) -> f64 { + f64::from_bits((x.to_bits() & !f64::SIGN_MASK) | (y.to_bits() & f64::SIGN_MASK)) +} /// Copies the sign from `y` to `x` for `f128` values. /// /// The stabilized version of this intrinsic is /// [`f128::copysign`](../../std/primitive.f128.html#method.copysign) +#[inline] #[rustc_nounwind] #[rustc_intrinsic] -pub const fn copysignf128(x: f128, y: f128) -> f128; +pub const fn copysignf128(x: f128, y: f128) -> f128 { + f128::from_bits((x.to_bits() & !f128::SIGN_MASK) | (y.to_bits() & f128::SIGN_MASK)) +} /// Generates the LLVM body for the automatic differentiation of `f` using Enzyme, /// with `df` as the derivative function and `args` as its arguments.