diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 270ea4e88a999..244d543adcb71 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -417,7 +417,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( | sym::simd_shr | sym::simd_and | sym::simd_or - | sym::simd_xor => { + | sym::simd_xor + | sym::simd_add_algebraic + | sym::simd_sub_algebraic + | sym::simd_mul_algebraic + | sym::simd_div_algebraic + | sym::simd_rem_algebraic => { intrinsic_args!(fx, args => (x, y); intrinsic); // FIXME use vector instructions when possible @@ -439,32 +444,49 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane), (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane), - (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane), - (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane), - (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane), - (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane), - (ty::Float(FloatTy::F16), sym::simd_rem) => fx.lib_call( - "fmodf", - vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], - vec![AbiParam::new(types::F32)], - // FIXME(bytecodealliance/wasmtime#8312): Already converted - // by the FIXME above. - // fx.bcx.ins().fpromote(types::F32, lhs), - // fx.bcx.ins().fpromote(types::F32, rhs), - &[x_lane, y_lane], - )[0], - (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call( - "fmodf", - vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], - vec![AbiParam::new(types::F32)], - &[x_lane, y_lane], - )[0], - (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call( - "fmod", - vec![AbiParam::new(types::F64), AbiParam::new(types::F64)], - vec![AbiParam::new(types::F64)], - &[x_lane, y_lane], - )[0], + (ty::Float(_), sym::simd_add | sym::simd_add_algebraic) => { + fx.bcx.ins().fadd(x_lane, y_lane) + } + (ty::Float(_), sym::simd_sub | sym::simd_sub_algebraic) => { + fx.bcx.ins().fsub(x_lane, y_lane) + } + (ty::Float(_), sym::simd_mul | sym::simd_mul_algebraic) => { + fx.bcx.ins().fmul(x_lane, y_lane) + } + (ty::Float(_), sym::simd_div | sym::simd_div_algebraic) => { + fx.bcx.ins().fdiv(x_lane, y_lane) + } + (ty::Float(FloatTy::F16), sym::simd_rem | sym::simd_rem_algebraic) => { + let ret = fx.lib_call( + "fmodf", + vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], + vec![AbiParam::new(types::F32)], + // FIXME(bytecodealliance/wasmtime#8312): Already converted + // by the FIXME above. + // fx.bcx.ins().fpromote(types::F32, lhs), + // fx.bcx.ins().fpromote(types::F32, rhs), + &[x_lane, y_lane], + ); + ret[0] + } + (ty::Float(FloatTy::F32), sym::simd_rem | sym::simd_rem_algebraic) => { + let ret = fx.lib_call( + "fmodf", + vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], + vec![AbiParam::new(types::F32)], + &[x_lane, y_lane], + ); + ret[0] + } + (ty::Float(FloatTy::F64), sym::simd_rem | sym::simd_rem_algebraic) => { + let ret = fx.lib_call( + "fmod", + vec![AbiParam::new(types::F64), AbiParam::new(types::F64)], + vec![AbiParam::new(types::F64)], + &[x_lane, y_lane], + ); + ret[0] + } (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane), (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane), @@ -666,6 +688,26 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }); } + sym::simd_reduce_add_algebraic => { + intrinsic_args!(fx, args => (v); intrinsic); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + if !v.layout().ty.simd_size_and_type(fx.tcx).1.is_floating_point() { + fx.tcx.dcx().span_err(span, "algebraic reduction requires a float vector"); + return; + } + + simd_reduce(fx, v, None, ret, &|fx, _lane_ty, a, b| { + codegen_f16_f128::maybe_with_f16_to_f32_pair(fx, a, b, |fx, a, b| { + fx.bcx.ins().fadd(a, b) + }) + }); + } + sym::simd_reduce_mul_ordered => { intrinsic_args!(fx, args => (v, acc); intrinsic); let acc = acc.load_scalar(fx); @@ -707,6 +749,26 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }); } + sym::simd_reduce_mul_algebraic => { + intrinsic_args!(fx, args => (v); intrinsic); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + if !v.layout().ty.simd_size_and_type(fx.tcx).1.is_floating_point() { + fx.tcx.dcx().span_err(span, "algebraic reduction requires a float vector"); + return; + } + + simd_reduce(fx, v, None, ret, &|fx, _lane_ty, a, b| { + codegen_f16_f128::maybe_with_f16_to_f32_pair(fx, a, b, |fx, a, b| { + fx.bcx.ins().fmul(a, b) + }) + }); + } + sym::simd_reduce_all => { intrinsic_args!(fx, args => (v); intrinsic); diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 2f300b02babb4..bbd17e00b6a58 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -839,7 +839,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { } #[cfg(feature = "master")] - match self.cx.float_type(a_type) { + match self.cx.float_type(a_type_unqualified) { Some(abi::Float::F16) => { let fmodf = self.context.get_builtin_function("fmodf"); let f32_type = self.type_f32(); @@ -2329,6 +2329,14 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { unimplemented!(); } + pub fn vector_reduce_fadd_algebraic( + &mut self, + _acc: RValue<'gcc>, + _src: RValue<'gcc>, + ) -> RValue<'gcc> { + unimplemented!(); + } + #[cfg(feature = "master")] pub fn vector_reduce_fadd(&mut self, acc: RValue<'gcc>, src: RValue<'gcc>) -> RValue<'gcc> { let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type"); @@ -2359,6 +2367,14 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { unimplemented!(); } + pub fn vector_reduce_fmul_algebraic( + &mut self, + _acc: RValue<'gcc>, + _src: RValue<'gcc>, + ) -> RValue<'gcc> { + unimplemented!(); + } + #[cfg(feature = "master")] pub fn vector_reduce_fmul(&mut self, acc: RValue<'gcc>, src: RValue<'gcc>) -> RValue<'gcc> { let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type"); diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs index eaa5c415f9fda..236cb40042d68 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs @@ -1278,6 +1278,11 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( simd_xor: Uint, Int => xor; simd_minimum_number_nsz: Float => vector_minimum_number_nsz; simd_maximum_number_nsz: Float => vector_maximum_number_nsz; + simd_add_algebraic: Float => fadd; + simd_sub_algebraic: Float => fsub; + simd_mul_algebraic: Float => fmul; + simd_div_algebraic: Float => fdiv; + simd_rem_algebraic: Float => frem; } macro_rules! arith_unary { @@ -1396,7 +1401,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( } macro_rules! arith_red { - ($name:ident : $vec_op:expr, $float_reduce:ident, $ordered:expr, $op:ident, + ($name:ident : $vec_op:expr, $float_reduce:ident, ordered: $ordered:expr, allow_int: $int:expr, $op:ident, $identity:expr) => { if name == sym::$name { require!( @@ -1404,7 +1409,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return match *in_elem.kind() { - ty::Int(_) | ty::Uint(_) => { + ty::Int(_) | ty::Uint(_) if $int => { let r = bx.vector_reduce_op(args[0].immediate(), $vec_op); if $ordered { // if overflow occurs, the result is the @@ -1439,28 +1444,48 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( arith_red!( simd_reduce_add_unordered: BinaryOp::Plus, vector_reduce_fadd_reassoc, - false, + ordered: false, + allow_int: true, add, 0.0 // FIXME: Use this argument. ); arith_red!( simd_reduce_mul_unordered: BinaryOp::Mult, vector_reduce_fmul_reassoc, - false, + ordered: false, + allow_int: true, + mul, + 1.0 + ); + arith_red!( + simd_reduce_add_algebraic: BinaryOp::Plus, + vector_reduce_fadd_algebraic, + ordered: false, + allow_int: false, + add, + 0.0 + ); + arith_red!( + simd_reduce_mul_algebraic: BinaryOp::Mult, + vector_reduce_fmul_algebraic, + ordered: false, + allow_int: false, mul, 1.0 ); arith_red!( simd_reduce_add_ordered: BinaryOp::Plus, vector_reduce_fadd, - true, + ordered: true, + allow_int: true, add, 0.0 ); arith_red!( simd_reduce_mul_ordered: BinaryOp::Mult, vector_reduce_fmul, - true, + ordered: true, + allow_int: true, mul, 1.0 ); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 9e3492d0ca696..f8a97d5dfbdf5 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1798,6 +1798,30 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { instr } } + pub(crate) fn vector_reduce_fadd_algebraic( + &mut self, + acc: &'ll Value, + src: &'ll Value, + ) -> &'ll Value { + unsafe { + let instr = + self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src]); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + pub(crate) fn vector_reduce_fmul_algebraic( + &mut self, + acc: &'ll Value, + src: &'ll Value, + ) -> &'ll Value { + unsafe { + let instr = + self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src]); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value { self.call_intrinsic("llvm.vector.reduce.add", &[self.val_ty(src)], &[src]) } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 2b529aa2c5360..361a0a939a701 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -2894,7 +2894,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } macro_rules! arith_red { - ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident, + ($name:ident : $integer_reduce:ident, $float_reduce:ident, ordered: $ordered:expr, allow_int: $int:expr, $op:ident, $identity:expr) => { if name == sym::$name { require!( @@ -2902,7 +2902,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return match in_elem.kind() { - ty::Int(_) | ty::Uint(_) => { + ty::Int(_) | ty::Uint(_) if $int => { let r = bx.$integer_reduce(args[0].immediate()); if $ordered { // if overflow occurs, the result is the @@ -2949,19 +2949,37 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; } - arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0); - arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); + arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, ordered: true, allow_int: true, add, -0.0); + arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, ordered: true, allow_int: true, mul, 1.0); arith_red!( simd_reduce_add_unordered: vector_reduce_add, vector_reduce_fadd_reassoc, - false, + ordered: false, + allow_int: true, add, -0.0 ); arith_red!( simd_reduce_mul_unordered: vector_reduce_mul, vector_reduce_fmul_reassoc, - false, + ordered: false, + allow_int: true, + mul, + 1.0 + ); + arith_red!( + simd_reduce_add_algebraic: vector_reduce_add, + vector_reduce_fadd_algebraic, + ordered: false, + allow_int: false, + add, + 0.0 + ); + arith_red!( + simd_reduce_mul_algebraic: vector_reduce_mul, + vector_reduce_fmul_algebraic, + ordered: false, + allow_int: false, mul, 1.0 ); @@ -3291,6 +3309,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( simd_xor: Uint, Int => xor; simd_maximum_number_nsz: Float => maximum_number_nsz; simd_minimum_number_nsz: Float => minimum_number_nsz; + simd_add_algebraic: Float => fadd_algebraic; + simd_sub_algebraic: Float => fsub_algebraic; + simd_mul_algebraic: Float => fmul_algebraic; + simd_div_algebraic: Float => fdiv_algebraic; + simd_rem_algebraic: Float => frem_algebraic; } macro_rules! arith_unary { diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs index a31d1ec4e920d..2409877f735ec 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs @@ -186,6 +186,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { | sym::simd_minimum_number_nsz | sym::simd_saturating_add | sym::simd_saturating_sub + | sym::simd_add_algebraic + | sym::simd_sub_algebraic + | sym::simd_mul_algebraic + | sym::simd_div_algebraic + | sym::simd_rem_algebraic | sym::simd_arith_offset => { use mir::BinOp; @@ -199,6 +204,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { enum Op { MirOp(BinOp), SaturatingOp(BinOp), + AlgebraicOp(BinOp), FMinMax(MinMax), WrappingOffset, } @@ -223,6 +229,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { sym::simd_minimum_number_nsz => Op::FMinMax(MinMax::MinimumNumberNsz), sym::simd_saturating_add => Op::SaturatingOp(BinOp::Add), sym::simd_saturating_sub => Op::SaturatingOp(BinOp::Sub), + sym::simd_add_algebraic => Op::AlgebraicOp(BinOp::Add), + sym::simd_sub_algebraic => Op::AlgebraicOp(BinOp::Sub), + sym::simd_mul_algebraic => Op::AlgebraicOp(BinOp::Mul), + sym::simd_div_algebraic => Op::AlgebraicOp(BinOp::Div), + sym::simd_rem_algebraic => Op::AlgebraicOp(BinOp::Rem), sym::simd_arith_offset => Op::WrappingOffset, _ => unreachable!(), }; @@ -232,9 +243,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let right = self.read_immediate(&self.project_index(&right, i)?)?; let dest = self.project_index(&dest, i)?; let val = match which { - Op::MirOp(mir_op) => { + Op::MirOp(mir_op) | Op::AlgebraicOp(mir_op) => { // this does NaN adjustments. - let val = self.binary_op(mir_op, &left, &right).map_err_kind(|kind| { + let mut val = self.binary_op(mir_op, &left, &right).map_err_kind(|kind| { match kind { InterpErrorKind::UndefinedBehavior(UndefinedBehaviorInfo::ShiftOverflow { shift_amount, .. }) => { // this resets the interpreter backtrace, but it's not worth avoiding that. @@ -247,6 +258,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { kind => kind } })?; + if matches!(which, Op::AlgebraicOp(_)) { + assert!(val.layout.ty.is_floating_point()); + val = M::apply_float_nondet(self, val)?; + } if matches!( mir_op, BinOp::Eq @@ -370,6 +385,36 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } self.write_immediate(*res, &dest)?; } + sym::simd_reduce_add_algebraic | sym::simd_reduce_mul_algebraic => { + use mir::BinOp; + + if !dest.layout.ty.is_floating_point() { + span_bug!(self.cur_span(), "algebraic reduction requires a float vector"); + } + + let (op, op_len) = self.project_to_simd(&args[0])?; + + // Compute starting value and reduction MIR op + let from_int = |i: i32| { + self.int_to_int_or_float( + &ImmTy::from_int(i, self.layout_of(self.tcx.types.i32)?), + dest.layout, + ) + }; + let (init, mir_op) = match intrinsic_name { + sym::simd_reduce_add_algebraic => (from_int(0)?, BinOp::Add), + sym::simd_reduce_mul_algebraic => (from_int(1)?, BinOp::Mul), + _ => unreachable!(), + }; + + let mut res = init; + for i in M::unordered_float_reduction_iter(self, op_len) { + let op = self.read_immediate(&self.project_index(&op, i)?)?; + res = self.binary_op(mir_op, &res, &op)?; + res = M::apply_float_nondet(self, res)?; + } + self.write_immediate(*res, &dest)?; + } sym::simd_select => { let (mask, mask_len) = self.project_to_simd(&args[0])?; let (yes, yes_len) = self.project_to_simd(&args[1])?; diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index 9760c8fc2004f..1713198c0598d 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -307,6 +307,14 @@ pub trait Machine<'tcx>: Sized { interp_ok(val) } + /// Generate a non-deterministic index iterator. + fn unordered_float_reduction_iter( + _ecx: &mut InterpCx<'tcx, Self>, + len: u64, + ) -> impl Iterator + use<'tcx, Self> { + (0..len).into_iter() + } + /// Determines the result of `min`/`max` on floats when the arguments are equal. fn equal_float_min_max(_ecx: &InterpCx<'tcx, Self>, a: F, _b: F) -> F { // By default, we pick the left argument. diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 0aa3e97568729..0ff99ef526160 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -657,6 +657,11 @@ pub(crate) fn check_intrinsic_type( | sym::simd_maximum_number_nsz | sym::simd_saturating_add | sym::simd_saturating_sub + | sym::simd_add_algebraic + | sym::simd_sub_algebraic + | sym::simd_mul_algebraic + | sym::simd_div_algebraic + | sym::simd_rem_algebraic | sym::simd_carryless_mul => (1, 0, vec![param(0), param(0)], param(0)), sym::simd_arith_offset => (2, 0, vec![param(0), param(1)], param(0)), sym::simd_neg @@ -708,6 +713,8 @@ pub(crate) fn check_intrinsic_type( } sym::simd_reduce_add_unordered | sym::simd_reduce_mul_unordered + | sym::simd_reduce_add_algebraic + | sym::simd_reduce_mul_algebraic | sym::simd_reduce_and | sym::simd_reduce_or | sym::simd_reduce_xor diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index eec3dad3802af..8d0fa7ec48705 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1960,6 +1960,7 @@ symbols! { signed, simd, simd_add, + simd_add_algebraic, simd_and, simd_arith_offset, simd_as, @@ -1974,6 +1975,7 @@ symbols! { simd_ctpop, simd_cttz, simd_div, + simd_div_algebraic, simd_eq, simd_expose_provenance, simd_extract, @@ -2004,9 +2006,11 @@ symbols! { simd_maximum_number_nsz, simd_minimum_number_nsz, simd_mul, + simd_mul_algebraic, simd_ne, simd_neg, simd_or, + simd_reduce_add_algebraic, simd_reduce_add_ordered, simd_reduce_add_unordered, simd_reduce_all, @@ -2014,12 +2018,14 @@ symbols! { simd_reduce_any, simd_reduce_max, simd_reduce_min, + simd_reduce_mul_algebraic, simd_reduce_mul_ordered, simd_reduce_mul_unordered, simd_reduce_or, simd_reduce_xor, simd_relaxed_fma, simd_rem, + simd_rem_algebraic, simd_round, simd_round_ties_even, simd_saturating_add, @@ -2033,6 +2039,7 @@ symbols! { simd_shuffle_const_generic, simd_splat, simd_sub, + simd_sub_algebraic, simd_trunc, simd_with_exposed_provenance, simd_xor, diff --git a/library/core/src/intrinsics/simd/mod.rs b/library/core/src/intrinsics/simd/mod.rs index e9488a6bdb263..0eeb3330ddc7e 100644 --- a/library/core/src/intrinsics/simd/mod.rs +++ b/library/core/src/intrinsics/simd/mod.rs @@ -494,7 +494,7 @@ pub const unsafe fn simd_saturating_sub(lhs: T, rhs: T) -> T; pub const unsafe fn simd_reduce_add_ordered(x: T, y: U) -> U; /// Adds elements within a vector in arbitrary order. May also be re-associated with -/// unordered additions on the inputs/outputs. +/// unordered/algebraic/fast additions on the inputs/outputs. /// /// `T` must be a vector of integers or floats. /// @@ -503,6 +503,15 @@ pub const unsafe fn simd_reduce_add_ordered(x: T, y: U) -> U; #[rustc_nounwind] pub unsafe fn simd_reduce_add_unordered(x: T) -> U; +/// Adds elements within a vector, allowing optimizations based on algebraic rules. +/// +/// `T` must be a vector of floats. +/// +/// `U` must be the element type of `T`. +#[rustc_intrinsic] +#[rustc_nounwind] +pub const unsafe fn simd_reduce_add_algebraic(x: T) -> U; + /// Multiplies elements within a vector from left to right. /// /// `T` must be a vector of integers or floats. @@ -515,7 +524,7 @@ pub unsafe fn simd_reduce_add_unordered(x: T) -> U; pub const unsafe fn simd_reduce_mul_ordered(x: T, y: U) -> U; /// Multiplies elements within a vector in arbitrary order. May also be re-associated with -/// unordered additions on the inputs/outputs. +/// unordered/algebraic/fast multiplications on the inputs/outputs. /// /// `T` must be a vector of integers or floats. /// @@ -524,6 +533,15 @@ pub const unsafe fn simd_reduce_mul_ordered(x: T, y: U) -> U; #[rustc_nounwind] pub unsafe fn simd_reduce_mul_unordered(x: T) -> U; +/// Multiplies elements within a vector, allowing optimizations based on algebraic rules. +/// +/// `T` must be a vector of floats. +/// +/// `U` must be the element type of `T`. +#[rustc_intrinsic] +#[rustc_nounwind] +pub const unsafe fn simd_reduce_mul_algebraic(x: T) -> U; + /// Checks if all mask values are true. /// /// `T` must be a vector of integers. @@ -845,3 +863,38 @@ pub unsafe fn simd_flog2(a: T) -> T; #[rustc_intrinsic] #[rustc_nounwind] pub unsafe fn simd_flog(a: T) -> T; + +/// Float addition that allows optimizations based on algebraic rules. +/// +/// `T` must be a vector of floats. +#[rustc_nounwind] +#[rustc_intrinsic] +pub const unsafe fn simd_add_algebraic(a: T, b: T) -> T; + +/// Float subtraction that allows optimizations based on algebraic rules. +/// +/// `T` must be a vector of floats. +#[rustc_nounwind] +#[rustc_intrinsic] +pub const unsafe fn simd_sub_algebraic(a: T, b: T) -> T; + +/// Float multiplication that allows optimizations based on algebraic rules. +/// +/// `T` must be a vector of floats. +#[rustc_nounwind] +#[rustc_intrinsic] +pub const unsafe fn simd_mul_algebraic(a: T, b: T) -> T; + +/// Float division that allows optimizations based on algebraic rules. +/// +/// `T` must be a vector of floats. +#[rustc_nounwind] +#[rustc_intrinsic] +pub const unsafe fn simd_div_algebraic(a: T, b: T) -> T; + +/// Float remainder that allows optimizations based on algebraic rules. +/// +/// `T` must be a vector of floats. +#[rustc_nounwind] +#[rustc_intrinsic] +pub const unsafe fn simd_rem_algebraic(a: T, b: T) -> T; diff --git a/src/tools/miri/src/machine.rs b/src/tools/miri/src/machine.rs index b1f816c92e6f9..17ab239f9693b 100644 --- a/src/tools/miri/src/machine.rs +++ b/src/tools/miri/src/machine.rs @@ -9,6 +9,7 @@ use std::rc::Rc; use std::{fmt, process}; use rand::rngs::StdRng; +use rand::seq::SliceRandom; use rand::{RngExt, SeedableRng}; use rustc_abi::{Align, ExternAbi, Size}; use rustc_apfloat::{Float, FloatConvert}; @@ -1441,6 +1442,18 @@ impl<'tcx> Machine<'tcx> for MiriMachine<'tcx> { crate::math::apply_random_float_error_to_imm(ecx, val, 4) } + #[inline(always)] + fn unordered_float_reduction_iter( + ecx: &mut InterpCx<'tcx, Self>, + len: u64, + ) -> impl Iterator + use<'tcx> { + let mut indices = (0..len).collect::>(); + if ecx.machine.float_nondet { + indices.shuffle(&mut ecx.machine.rng.borrow_mut()); + } + indices.into_iter() + } + #[inline(always)] fn equal_float_min_max(ecx: &MiriInterpCx<'tcx>, a: F, b: F) -> F { ecx.equal_float_min_max(a, b) diff --git a/src/tools/miri/tests/pass/float.rs b/src/tools/miri/tests/pass/float.rs index 019b775a450b1..083c8b957c81d 100644 --- a/src/tools/miri/tests/pass/float.rs +++ b/src/tools/miri/tests/pass/float.rs @@ -20,42 +20,8 @@ use std::hint::black_box; #[path = "../utils/mod.rs"] mod utils; -use utils::check_nondet; +use utils::{assert_approx_eq, check_nondet, ERR_TOLERANCE}; -/// Error tolerance in ULP. -/// Miri adds 4 ULP of errors itself, and we allow for 4 additional ULP of host float error. -const ERR_TOLERANCE: i32 = 8; - -/// Compare the two floats, allowing for $ulp many ULPs of error. -/// -/// ULP means "Units in the Last Place" or "Units of Least Precision". -/// The ULP of a float `a` is the smallest possible change at `a`, so the ULP difference represents how -/// many discrete floating-point steps are needed to reach the actual value from the expected value. -/// -/// Essentially ULP can be seen as a distance metric of floating-point numbers, but with -/// the same amount of "spacing" between all consecutive representable values. So even though 2 very large floating point numbers -/// have a large value difference, their ULP can still be 1, so they are still "approximatly equal", -/// but the EPSILON check would have failed. -macro_rules! assert_approx_eq { - ($a:expr, $b:expr, $ulp:expr $( , )?) => {{ - let (actual, expected) = ($a, $b); - let allowed_ulp_diff = $ulp; - let _force_same_type = actual == expected; - // Approximate the ULP by taking half the distance between the number one place "up" - // and the number one place "down". - let ulp = (expected.next_up() - expected.next_down()) / 2.0; - let ulp_diff = ((actual - expected) / ulp).abs().round() as i32; - if ulp_diff > allowed_ulp_diff { - panic!("{actual:?} is not approximately equal to {expected:?}\ndifference in ULP: {ulp_diff} > {allowed_ulp_diff}"); - }; - }}; - - ($a:expr, $b: expr $( , )?) => { - // Accept up to 12ULP (4ULP for miri artificial error and the rest for host floats). - // We saw failures on an i686-linux host with a limit of 8! - assert_approx_eq!($a, $b, ERR_TOLERANCE); - }; -} /// We turn the quiet NaN f*::NAN into a signaling one by flipping the first (most significant) /// two bits of the mantissa. For this we have to shift by `MANTISSA_DIGITS-3` because: diff --git a/src/tools/miri/tests/pass/intrinsics/portable-simd.rs b/src/tools/miri/tests/pass/intrinsics/portable-simd.rs index c16bc21636f78..f6e5872aca6e9 100644 --- a/src/tools/miri/tests/pass/intrinsics/portable-simd.rs +++ b/src/tools/miri/tests/pass/intrinsics/portable-simd.rs @@ -21,6 +21,10 @@ use std::ptr; use std::simd::StdFloat; use std::simd::prelude::*; +#[path = "../../utils/mod.rs"] +mod utils; +use utils::{assert_approx_eq, check_nondet, ERR_TOLERANCE}; + // small hack to make type inference better macro_rules! assert_eq { ($a:expr, $b:expr $(,$t:tt)* $(,)?) => {{ @@ -44,13 +48,13 @@ impl Clone for PackedSimd { impl PartialEq for PackedSimd { fn eq(&self, other: &Self) -> bool { - self.into_array() == other.into_array() + self.to_array() == other.to_array() } } impl Debug for PackedSimd { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - Debug::fmt(&self.into_array(), f) + Debug::fmt(&self.to_array(), f) } } @@ -64,7 +68,7 @@ impl PackedSimd { fn from_array(a: [T; N]) -> Self { Self(a) } - fn into_array(self) -> [T; N] { + fn to_array(self) -> [T; N] { // as we have `repr(packed)`, there shouldn't be any padding bytes unsafe { std::mem::transmute_copy(&self) } } @@ -1135,6 +1139,73 @@ fn test_simd_ops_non_pow2() { assert_eq!(unsafe { *(&raw const z).cast::<[u32; 3]>() }, [3u32; 3]); } +fn test_algebraic() { + use std::intrinsics::simd::{ + simd_add_algebraic, + simd_sub_algebraic, + simd_mul_algebraic, + simd_div_algebraic, + simd_rem_algebraic, + simd_reduce_add_algebraic, + simd_reduce_mul_algebraic, + }; + + if !cfg!(miri) { + return; // this test does not make sense natively + } + + macro_rules! assert_approx_eq_simd { + ($a:expr, $b:expr $(,)?) => { + let a = $a.to_array(); + let b = $b.to_array(); + assert_eq!(a.len(), b.len()); + for i in 0..a.len() { + assert_approx_eq!(a[i], b[i]); + } + } + } + + macro_rules! check_nondet_elementwise { + (@check $e:expr, $expected:expr) => { + check_nondet(|| { + let e = $e; + assert_approx_eq_simd!(e, $expected); + e + }); + }; + ($vector:ty, $a:expr, $b:expr) => { + check_nondet_elementwise!(@check unsafe { simd_add_algebraic($a, $b) }, unsafe { simd_add($a, $b) }); + check_nondet_elementwise!(@check unsafe { simd_sub_algebraic($a, $b) }, unsafe { simd_sub($a, $b) }); + check_nondet_elementwise!(@check unsafe { simd_mul_algebraic($a, $b) }, unsafe { simd_mul($a, $b) }); + check_nondet_elementwise!(@check unsafe { simd_div_algebraic($a, $b) }, unsafe { simd_div($a, $b) }); + check_nondet_elementwise!(@check unsafe { simd_rem_algebraic($a, $b) }, unsafe { simd_rem($a, $b) }); + } + } + + macro_rules! check_nondet_reduce { + (@check $e:expr, $expected:expr, $len:expr) => { + check_nondet(|| { + let e = $e; + assert_approx_eq!(e, $expected, ($len as i32) * ERR_TOLERANCE); + e + }); + }; + ($vector:ty, $scalar:ty, $a:expr) => { + check_nondet_reduce!(@check unsafe { simd_reduce_add_algebraic::<_, $scalar>($a) }, unsafe { simd_reduce_add_ordered::<_, $scalar>($a, -0.0) }, $a.to_array().len()); + check_nondet_reduce!(@check unsafe { simd_reduce_mul_algebraic::<_, $scalar>($a) }, unsafe { simd_reduce_mul_ordered::<_, $scalar>($a, 1.0) }, $a.to_array().len()); + } + } + + check_nondet_elementwise!(f16, f16x4::from_array([1.0, 2.0, 3.0, 4.0]), f16x4::from_array([5.0, 6.0, 7.0, 8.0])); + check_nondet_elementwise!(f32, f32x4::from_array([1.0, 2.0, 3.0, 4.0]), f32x4::from_array([5.0, 6.0, 7.0, 8.0])); + check_nondet_elementwise!(f64, f64x4::from_array([1.0, 2.0, 3.0, 4.0]), f64x4::from_array([5.0, 6.0, 7.0, 8.0])); + check_nondet_elementwise!(f128, f128x4::from_array([1.0, 2.0, 3.0, 4.0]), f128x4::from_array([5.0, 6.0, 7.0, 8.0])); + check_nondet_reduce!(f16, f16, f16x4::from_array([1.0, 2.0, 3.0, 4.0])); + check_nondet_reduce!(f32, f32, f32x4::from_array([1.0, 2.0, 3.0, 4.0])); + check_nondet_reduce!(f64, f64, f64x4::from_array([1.0, 2.0, 3.0, 4.0])); + check_nondet_reduce!(f128, f128, f128x4::from_array([1.0, 2.0, 3.0, 4.0])); +} + fn main() { test_simd_mask(); #[cfg(any(miri, target_has_reliable_f16_math))] @@ -1153,4 +1224,5 @@ fn main() { test_simd_intrinsics(); test_simd_float_intrinsics(); test_simd_masked_loadstore(); + test_algebraic(); } diff --git a/src/tools/miri/tests/utils/float.rs b/src/tools/miri/tests/utils/float.rs new file mode 100644 index 0000000000000..4d4d481e60338 --- /dev/null +++ b/src/tools/miri/tests/utils/float.rs @@ -0,0 +1,38 @@ +#![allow(unused_macros)] + +/// Error tolerance in ULP. +/// Miri adds 4 ULP of errors itself, and we allow for 4 additional ULP of host float error. +pub const ERR_TOLERANCE: i32 = 8; + +/// Compare the two floats, allowing for $ulp many ULPs of error. +/// +/// ULP means "Units in the Last Place" or "Units of Least Precision". +/// The ULP of a float `a` is the smallest possible change at `a`, so the ULP difference represents how +/// many discrete floating-point steps are needed to reach the actual value from the expected value. +/// +/// Essentially ULP can be seen as a distance metric of floating-point numbers, but with +/// the same amount of "spacing" between all consecutive representable values. So even though 2 very large floating point numbers +/// have a large value difference, their ULP can still be 1, so they are still "approximatly equal", +/// but the EPSILON check would have failed. +macro_rules! assert_approx_eq { + ($a:expr, $b:expr, $ulp:expr $( , )?) => {{ + let (actual, expected) = ($a, $b); + let allowed_ulp_diff = $ulp; + let _force_same_type = actual == expected; + // Approximate the ULP by taking half the distance between the number one place "up" + // and the number one place "down". + let ulp = (expected.next_up() - expected.next_down()) / 2.0; + let ulp_diff = ((actual - expected) / ulp).abs().round() as i32; + if ulp_diff > allowed_ulp_diff { + panic!("{actual:?} is not approximately equal to {expected:?}\ndifference in ULP: {ulp_diff} > {allowed_ulp_diff}"); + }; + }}; + + ($a:expr, $b: expr $( , )?) => { + // Accept up to 12ULP (4ULP for miri artificial error and the rest for host floats). + // We saw failures on an i686-linux host with a limit of 8! + $crate::utils::assert_approx_eq!($a, $b, $crate::utils::ERR_TOLERANCE); + }; +} + +pub(crate) use assert_approx_eq; diff --git a/src/tools/miri/tests/utils/mod.rs b/src/tools/miri/tests/utils/mod.rs index 3fe655350776c..bfc5dd350ae79 100644 --- a/src/tools/miri/tests/utils/mod.rs +++ b/src/tools/miri/tests/utils/mod.rs @@ -9,10 +9,12 @@ mod macros; mod fs; mod io; mod miri_extern; +mod float; pub use self::fs::*; pub use self::io::*; pub use self::miri_extern::*; +pub use self::float::*; /// Return a `CString` version of the given string. pub fn into_c_string(str: impl Into) -> CString { diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-algebraic-pass.rs b/tests/ui/simd/intrinsic/generic-arithmetic-algebraic-pass.rs new file mode 100644 index 0000000000000..3c9e589441d37 --- /dev/null +++ b/tests/ui/simd/intrinsic/generic-arithmetic-algebraic-pass.rs @@ -0,0 +1,58 @@ +//@ run-pass +//@ compile-flags: --cfg minisimd_const + +#![allow(non_camel_case_types)] +#![feature(repr_simd, core_intrinsics, const_trait_impl, const_cmp, const_index)] + +#[path = "../../../auxiliary/minisimd.rs"] +mod minisimd; +use minisimd::*; + +macro_rules! all_eq { + ($a: expr, $b: expr $(,)?) => {{ + let a = $a; + let b = $b; + assert!(a == b); + }}; +} + +use std::intrinsics::simd::*; + +const fn arithmetic() { + let x1 = f32x4::from_array([1.0, 2.0, 3.0, 4.0]); + let x2 = f32x4::from_array([2.0, 3.0, 4.0, 5.0]); + + unsafe { + all_eq!(simd_add_algebraic(x1, x2), f32x4::from_array([3.0, 5.0, 7.0, 9.0])); + all_eq!(simd_add_algebraic(x2, x1), f32x4::from_array([3.0, 5.0, 7.0, 9.0])); + + all_eq!(simd_mul_algebraic(x1, x2), f32x4::from_array([2.0, 6.0, 12.0, 20.0])); + all_eq!(simd_mul_algebraic(x2, x1), f32x4::from_array([2.0, 6.0, 12.0, 20.0])); + + all_eq!(simd_sub_algebraic(x2, x1), f32x4::from_array([1.0, 1.0, 1.0, 1.0])); + all_eq!(simd_sub_algebraic(x1, x2), f32x4::from_array([-1.0, -1.0, -1.0, -1.0])); + + all_eq!( + simd_div_algebraic(x1, x2), + f32x4::from_array([1.0 / 2.0, 2.0 / 3.0, 3.0 / 4.0, 4.0 / 5.0]) + ); + all_eq!( + simd_div_algebraic(x2, x1), + f32x4::from_array([2.0 / 1.0, 3.0 / 2.0, 4.0 / 3.0, 5.0 / 4.0]) + ); + + all_eq!(simd_rem_algebraic(x1, x1), f32x4::from_array([0.0, 0.0, 0.0, 0.0])); + all_eq!(simd_rem_algebraic(x1, x2), x1); + all_eq!(simd_rem_algebraic(x2, x1), f32x4::from_array([0.0, 1.0, 1.0, 1.0])); + + assert_eq!(simd_reduce_add_algebraic(x1), 10.0_f32); + assert_eq!(simd_reduce_add_algebraic(x2), 14.0_f32); + assert_eq!(simd_reduce_mul_algebraic(x1), 24.0_f32); + assert_eq!(simd_reduce_mul_algebraic(x2), 120.0_f32); + } +} + +fn main() { + const { arithmetic() }; + arithmetic(); +} diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs b/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs index 62782db37ed4c..b967be7f85527 100644 --- a/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs +++ b/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs @@ -223,6 +223,13 @@ const fn arithmetic() { all_eq!(simd_cttz(x1), i32x4::from_array([0, 1, 0, 2])); all_eq!(simd_cttz(y1), U32::<4>::from_array([0, 1, 0, 2])); + + assert_eq!(simd_reduce_add_ordered(x1, 0i32), 10i32); + assert_eq!(simd_reduce_add_ordered(y1, 0u32), 10u32); + assert_eq!(simd_reduce_add_ordered(z1, 0.0_f32), 10.0_f32); + assert_eq!(simd_reduce_mul_ordered(x1, 1i32), 24i32); + assert_eq!(simd_reduce_mul_ordered(y1, 1u32), 24u32); + assert_eq!(simd_reduce_mul_ordered(z1, 1.0_f32), 24.0_f32); } }