Skip to content

Add SIMD algebraic intrinsics #164952

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
calebzulawski wants to merge 14 commits into
base: main
Choose a base branch
from
Open

Add SIMD algebraic intrinsics #164952

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
40d741c
Add algebraic simd intrinsics
calebzulawski Oct 26, 2026
b2b4efe
Split ordered and unordered intrinsics, implement non-deterministic r…
calebzulawski Oct 26, 2026
3ff298e
Implement SIMD algebraic intrinsics in cg_clif and cg_gcc by emitting…
calebzulawski Oct 27, 2026
0bf20d3
Update library/core/src/intrinsics/simd/mod.rs
calebzulawski Oct 27, 2026
537e8cd
Disable unordered reduces when nondet is disabled in miri
calebzulawski Oct 27, 2026
b8ffa42
Undo const unordered operations
calebzulawski Oct 28, 2026
909538c
Make match arms a little neater
calebzulawski Oct 28, 2026
d0d112a
Don't use host floats
calebzulawski Oct 30, 2026
d5da8b1
Test f16/f128, and check that results match the expected value
calebzulawski Oct 31, 2026
95f5629
Fix cg_gcc frem
calebzulawski Oct 31, 2026
1cc44ad
Rename float reduction iterator
calebzulawski Nov 8, 2026
e60d499
Algebraic reduce add doesn't need a signed zero
calebzulawski Nov 8, 2026
d617bff
Limit algebraic reductions to floats
calebzulawski Nov 8, 2026
ac54a52
Make macros easier to read
calebzulawski Nov 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 89 additions & 27 deletions compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
| sym::simd_shr
| sym::simd_and
| sym::simd_or
| sym::simd_xor => {
| sym::simd_xor
| sym::simd_add_algebraic
| sym::simd_sub_algebraic
| sym::simd_mul_algebraic
| sym::simd_div_algebraic
| sym::simd_rem_algebraic => {
intrinsic_args!(fx, args => (x, y); intrinsic);

// FIXME use vector instructions when possible
Expand All @@ -439,32 +444,49 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
(ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
(ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),

(ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
(ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
(ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
(ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
(ty::Float(FloatTy::F16), sym::simd_rem) => fx.lib_call(
"fmodf",
vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
vec![AbiParam::new(types::F32)],
// FIXME(bytecodealliance/wasmtime#8312): Already converted
// by the FIXME above.
// fx.bcx.ins().fpromote(types::F32, lhs),
// fx.bcx.ins().fpromote(types::F32, rhs),
&[x_lane, y_lane],
)[0],
(ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
"fmodf",
vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
vec![AbiParam::new(types::F32)],
&[x_lane, y_lane],
)[0],
(ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
"fmod",
vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
vec![AbiParam::new(types::F64)],
&[x_lane, y_lane],
)[0],
(ty::Float(_), sym::simd_add | sym::simd_add_algebraic) => {
fx.bcx.ins().fadd(x_lane, y_lane)
}
(ty::Float(_), sym::simd_sub | sym::simd_sub_algebraic) => {
fx.bcx.ins().fsub(x_lane, y_lane)
}
(ty::Float(_), sym::simd_mul | sym::simd_mul_algebraic) => {
fx.bcx.ins().fmul(x_lane, y_lane)
}
(ty::Float(_), sym::simd_div | sym::simd_div_algebraic) => {
fx.bcx.ins().fdiv(x_lane, y_lane)
}
(ty::Float(FloatTy::F16), sym::simd_rem | sym::simd_rem_algebraic) => {
let ret = fx.lib_call(
"fmodf",
vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
vec![AbiParam::new(types::F32)],
// FIXME(bytecodealliance/wasmtime#8312): Already converted
// by the FIXME above.
// fx.bcx.ins().fpromote(types::F32, lhs),
// fx.bcx.ins().fpromote(types::F32, rhs),
&[x_lane, y_lane],
);
ret[0]
}
(ty::Float(FloatTy::F32), sym::simd_rem | sym::simd_rem_algebraic) => {
let ret = fx.lib_call(
"fmodf",
vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
vec![AbiParam::new(types::F32)],
&[x_lane, y_lane],
);
ret[0]
}
(ty::Float(FloatTy::F64), sym::simd_rem | sym::simd_rem_algebraic) => {
let ret = fx.lib_call(
"fmod",
vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
vec![AbiParam::new(types::F64)],
&[x_lane, y_lane],
);
ret[0]
}

(ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
(ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
Expand Down Expand Up @@ -666,6 +688,26 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
});
}

sym::simd_reduce_add_algebraic => {
intrinsic_args!(fx, args => (v); intrinsic);

if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}

if !v.layout().ty.simd_size_and_type(fx.tcx).1.is_floating_point() {
fx.tcx.dcx().span_err(span, "algebraic reduction requires a float vector");
return;
}

simd_reduce(fx, v, None, ret, &|fx, _lane_ty, a, b| {
codegen_f16_f128::maybe_with_f16_to_f32_pair(fx, a, b, |fx, a, b| {
fx.bcx.ins().fadd(a, b)
})
});
}

sym::simd_reduce_mul_ordered => {
intrinsic_args!(fx, args => (v, acc); intrinsic);
let acc = acc.load_scalar(fx);
Expand Down Expand Up @@ -707,6 +749,26 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
});
}

sym::simd_reduce_mul_algebraic => {
intrinsic_args!(fx, args => (v); intrinsic);

if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}

if !v.layout().ty.simd_size_and_type(fx.tcx).1.is_floating_point() {
fx.tcx.dcx().span_err(span, "algebraic reduction requires a float vector");
return;
}

simd_reduce(fx, v, None, ret, &|fx, _lane_ty, a, b| {
codegen_f16_f128::maybe_with_f16_to_f32_pair(fx, a, b, |fx, a, b| {
fx.bcx.ins().fmul(a, b)
})
});
}

sym::simd_reduce_all => {
intrinsic_args!(fx, args => (v); intrinsic);

Expand Down
18 changes: 17 additions & 1 deletion compiler/rustc_codegen_gcc/src/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -839,7 +839,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
}

#[cfg(feature = "master")]
match self.cx.float_type(a_type) {
match self.cx.float_type(a_type_unqualified) {
Some(abi::Float::F16) => {
let fmodf = self.context.get_builtin_function("fmodf");
let f32_type = self.type_f32();
Expand Down Expand Up @@ -2329,6 +2329,14 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
unimplemented!();
}

pub fn vector_reduce_fadd_algebraic(
&mut self,
_acc: RValue<'gcc>,
_src: RValue<'gcc>,
) -> RValue<'gcc> {
unimplemented!();
}

#[cfg(feature = "master")]
pub fn vector_reduce_fadd(&mut self, acc: RValue<'gcc>, src: RValue<'gcc>) -> RValue<'gcc> {
let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
Expand Down Expand Up @@ -2359,6 +2367,14 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
unimplemented!();
}

pub fn vector_reduce_fmul_algebraic(
&mut self,
_acc: RValue<'gcc>,
_src: RValue<'gcc>,
) -> RValue<'gcc> {
unimplemented!();
}

#[cfg(feature = "master")]
pub fn vector_reduce_fmul(&mut self, acc: RValue<'gcc>, src: RValue<'gcc>) -> RValue<'gcc> {
let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
Expand Down
37 changes: 31 additions & 6 deletions compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1278,6 +1278,11 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
simd_xor: Uint, Int => xor;
simd_minimum_number_nsz: Float => vector_minimum_number_nsz;
simd_maximum_number_nsz: Float => vector_maximum_number_nsz;
simd_add_algebraic: Float => fadd;
simd_sub_algebraic: Float => fsub;
simd_mul_algebraic: Float => fmul;
simd_div_algebraic: Float => fdiv;
simd_rem_algebraic: Float => frem;
}

macro_rules! arith_unary {
Expand Down Expand Up @@ -1396,15 +1401,15 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
}

macro_rules! arith_red {
($name:ident : $vec_op:expr, $float_reduce:ident, $ordered:expr, $op:ident,
($name:ident : $vec_op:expr, $float_reduce:ident, ordered: $ordered:expr, allow_int: $int:expr, $op:ident,
$identity:expr) => {
if name == sym::$name {
require!(
ret_ty == in_elem,
InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
);
return match *in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
ty::Int(_) | ty::Uint(_) if $int => {
let r = bx.vector_reduce_op(args[0].immediate(), $vec_op);
if $ordered {
// if overflow occurs, the result is the
Expand Down Expand Up @@ -1439,28 +1444,48 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
arith_red!(
simd_reduce_add_unordered: BinaryOp::Plus,
vector_reduce_fadd_reassoc,
false,
ordered: false,
allow_int: true,
add,
0.0 // FIXME: Use this argument.
);
arith_red!(
simd_reduce_mul_unordered: BinaryOp::Mult,
vector_reduce_fmul_reassoc,
false,
ordered: false,
allow_int: true,
mul,
1.0
);
arith_red!(
simd_reduce_add_algebraic: BinaryOp::Plus,
vector_reduce_fadd_algebraic,
ordered: false,
allow_int: false,
add,
0.0
);
arith_red!(
simd_reduce_mul_algebraic: BinaryOp::Mult,
vector_reduce_fmul_algebraic,
ordered: false,
allow_int: false,
mul,
1.0
);
arith_red!(
simd_reduce_add_ordered: BinaryOp::Plus,
vector_reduce_fadd,
true,
ordered: true,
allow_int: true,
add,
0.0
);
arith_red!(
simd_reduce_mul_ordered: BinaryOp::Mult,
vector_reduce_fmul,
true,
ordered: true,
allow_int: true,
mul,
1.0
);
Expand Down
24 changes: 24 additions & 0 deletions compiler/rustc_codegen_llvm/src/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1798,6 +1798,30 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
instr
}
}
pub(crate) fn vector_reduce_fadd_algebraic(
&mut self,
acc: &'ll Value,
src: &'ll Value,
) -> &'ll Value {
unsafe {
let instr =
self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src]);
llvm::LLVMRustSetAlgebraicMath(instr);
instr
}
}
pub(crate) fn vector_reduce_fmul_algebraic(
&mut self,
acc: &'ll Value,
src: &'ll Value,
) -> &'ll Value {
unsafe {
let instr =
self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src]);
llvm::LLVMRustSetAlgebraicMath(instr);
instr
}
}
pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
self.call_intrinsic("llvm.vector.reduce.add", &[self.val_ty(src)], &[src])
}
Expand Down
35 changes: 29 additions & 6 deletions compiler/rustc_codegen_llvm/src/intrinsic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2894,15 +2894,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
}

macro_rules! arith_red {
($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
($name:ident : $integer_reduce:ident, $float_reduce:ident, ordered: $ordered:expr, allow_int: $int:expr, $op:ident,
$identity:expr) => {
if name == sym::$name {
require!(
ret_ty == in_elem,
InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
);
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
ty::Int(_) | ty::Uint(_) if $int => {
let r = bx.$integer_reduce(args[0].immediate());
if $ordered {
// if overflow occurs, the result is the
Expand Down Expand Up @@ -2949,19 +2949,37 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
};
}

arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, ordered: true, allow_int: true, add, -0.0);
arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, ordered: true, allow_int: true, mul, 1.0);
arith_red!(
simd_reduce_add_unordered: vector_reduce_add,
vector_reduce_fadd_reassoc,
false,
ordered: false,
allow_int: true,
add,
-0.0
);
arith_red!(
simd_reduce_mul_unordered: vector_reduce_mul,
vector_reduce_fmul_reassoc,
false,
ordered: false,
allow_int: true,
mul,
1.0
);
arith_red!(
simd_reduce_add_algebraic: vector_reduce_add,
vector_reduce_fadd_algebraic,
ordered: false,
allow_int: false,
add,
0.0
);
arith_red!(
simd_reduce_mul_algebraic: vector_reduce_mul,
vector_reduce_fmul_algebraic,
ordered: false,
allow_int: false,
mul,
1.0
);
Expand Down Expand Up @@ -3291,6 +3309,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
simd_xor: Uint, Int => xor;
simd_maximum_number_nsz: Float => maximum_number_nsz;
simd_minimum_number_nsz: Float => minimum_number_nsz;
simd_add_algebraic: Float => fadd_algebraic;
simd_sub_algebraic: Float => fsub_algebraic;
simd_mul_algebraic: Float => fmul_algebraic;
simd_div_algebraic: Float => fdiv_algebraic;
simd_rem_algebraic: Float => frem_algebraic;
Comment thread
bjorn3 marked this conversation as resolved.
Resolved

}
macro_rules! arith_unary {
Expand Down
Loading
Loading