https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/238175
Adds WaveMultiPrefixSum for Shader Model 6.5 and newer. Supports numeric scalars, vectors, long vectors, and matrices in Clang. Adds DXIL lowering with the Sum operation value 0 and the correct signedness. Adds SPIR-V subgroup lowering with either the EXT or NV partition extension. Prefers EXT when both are enabled. The existing SPIR-V limit on long-vector function parameters remains. The backend test uses memory for the long-vector case. The build and all 33 focused tests pass. SPIR-V binaries pass spirv-val with NV, EXT, and both extensions. Stacked on #238164. Fixes #99213 >From 5b45596f070ab6126c50a77bd183b4a186414d21 Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Mon, 2 Nov 2026 17:00:27 -0800 Subject: [PATCH] first attempt Co-authored-by: Copilot <[email protected]> Copilot-Session: 40997c2f-5aaa-41c6-8d1f-3e2e9ada315f --- clang/include/clang/Basic/Builtins.td | 6 + clang/include/clang/Basic/HLSLIntrinsics.td | 10 ++ clang/lib/CodeGen/CGHLSLBuiltins.cpp | 9 ++ clang/lib/CodeGen/CGHLSLRuntime.h | 6 + clang/lib/Sema/SemaHLSL.cpp | 12 +- .../builtins/WaveMultiPrefixSum.hlsl | 113 ++++++++++++++ .../BuiltIns/WaveMultiPrefixSum-errors.hlsl | 56 +++++++ llvm/include/llvm/IR/IntrinsicsDirectX.td | 14 ++ llvm/include/llvm/IR/IntrinsicsSPIRV.td | 7 + llvm/lib/Target/DirectX/DXIL.td | 15 ++ llvm/lib/Target/DirectX/DXILShaderFlags.cpp | 2 + .../DirectX/DirectXTargetTransformInfo.cpp | 2 + .../Target/SPIRV/SPIRVInstructionSelector.cpp | 8 + .../CodeGen/DirectX/ShaderFlags/wave-ops.ll | 14 ++ .../CodeGen/DirectX/WaveMultiPrefixSum.ll | 81 ++++++++++ .../DirectX/WaveMultiPrefixSum_errors.ll | 14 ++ .../hlsl-intrinsics/WaveMultiPrefixSum.ll | 147 ++++++++++++++++++ 17 files changed, 511 insertions(+), 5 deletions(-) create mode 100644 clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl create mode 100644 llvm/test/CodeGen/DirectX/WaveMultiPrefixSum.ll create mode 100644 llvm/test/CodeGen/DirectX/WaveMultiPrefixSum_errors.ll create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixSum.ll diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index 0d0bc61907fbb..ad5f1e10f34b1 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -5813,6 +5813,12 @@ def HLSLWaveMultiPrefixProduct : LangBuiltin<"HLSL_LANG"> { let Prototype = "void(...)"; } +def HLSLWaveMultiPrefixSum : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_wave_multi_prefix_sum"]; + let Attributes = [NoThrow, Const]; + let Prototype = "void(...)"; +} + def HLSLWavePrefixProduct : LangBuiltin<"HLSL_LANG"> { let Spellings = ["__builtin_hlsl_wave_prefix_product"]; let Attributes = [NoThrow, Const]; diff --git a/clang/include/clang/Basic/HLSLIntrinsics.td b/clang/include/clang/Basic/HLSLIntrinsics.td index 7009127c444b6..e87cabb5c865b 100644 --- a/clang/include/clang/Basic/HLSLIntrinsics.td +++ b/clang/include/clang/Basic/HLSLIntrinsics.td @@ -1922,6 +1922,16 @@ def hlsl_wave_multi_prefix_product : let Availability = SM6_5; } +def hlsl_wave_multi_prefix_sum : + HLSLTwoArgBuiltin<"WaveMultiPrefixSum", + "__builtin_hlsl_wave_multi_prefix_sum"> { + let Args = [Varying, VectorType<UIntTy, 4>]; + let VaryingTypes = AllNumericTypes; + let VaryingLongVector = 1; + let IsConvergent = 1; + let Availability = SM6_5; +} + // Returns the count of bits set to 1 on prior lanes. def hlsl_wave_prefix_count_bits : HLSLBuiltin<"WavePrefixCountBits", diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp index d301a16f5b2e1..38893d841d922 100644 --- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp +++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp @@ -1723,6 +1723,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, } case Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits: case Builtin::BI__builtin_hlsl_wave_multi_prefix_product: + case Builtin::BI__builtin_hlsl_wave_multi_prefix_sum: case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: { Value *ValueOp = EmitScalarExpr(E->getArg(0)); @@ -1741,6 +1742,14 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, return EmitIntrinsicCall(IID, {ValueOp->getType()}, Args, "hlsl.wave.multi.prefix.product"); } + if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_sum) { + Intrinsic::ID IID = + E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() + ? CGM.getHLSLRuntime().getWaveMultiPrefixUSumIntrinsic() + : CGM.getHLSLRuntime().getWaveMultiPrefixSumIntrinsic(); + return EmitIntrinsicCall(IID, {ValueOp->getType()}, Args, + "hlsl.wave.multi.prefix.sum"); + } if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and) return EmitIntrinsicCall( CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(), diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h index 50a7729be6875..35237123713c9 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/clang/lib/CodeGen/CGHLSLRuntime.h @@ -161,6 +161,12 @@ class CGHLSLRuntime { GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixUProduct, wave_multi_prefix_uproduct, subgroup_multi_prefix_product) + GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixSum, + wave_multi_prefix_sum, + subgroup_multi_prefix_sum) + GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixUSum, + wave_multi_prefix_usum, + subgroup_multi_prefix_sum) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min) diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp index d6a280e1fb6cb..7b8d911d8309d 100644 --- a/clang/lib/Sema/SemaHLSL.cpp +++ b/clang/lib/Sema/SemaHLSL.cpp @@ -5408,7 +5408,8 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { } case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: - case Builtin::BI__builtin_hlsl_wave_multi_prefix_product: { + case Builtin::BI__builtin_hlsl_wave_multi_prefix_product: + case Builtin::BI__builtin_hlsl_wave_multi_prefix_sum: { if (SemaRef.checkArgCount(TheCall, 2)) return true; @@ -5418,14 +5419,15 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { QualType ValueTy = TheCall->getArg(0)->getType(); QualType ElementTy = getScalarComponentType(ValueTy); - bool IsProduct = - BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_product; + bool IsArithmetic = + BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_product || + BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_sum; if ((!ElementTy->isIntegerType() && - !(IsProduct && ElementTy->hasFloatingRepresentation())) || + !(IsArithmetic && ElementTy->hasFloatingRepresentation())) || ElementTy->isBooleanType()) { SemaRef.Diag(TheCall->getArg(0)->getBeginLoc(), diag::err_builtin_invalid_arg_type) - << 1 << (ValueTy->isMatrixType() ? 3 : 5) << 1 << IsProduct + << 1 << (ValueTy->isMatrixType() ? 3 : 5) << 1 << IsArithmetic << ValueTy; return true; } diff --git a/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl new file mode 100644 index 0000000000000..943a90884521d --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl @@ -0,0 +1,113 @@ +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \ +// RUN: -disable-llvm-passes -o - | FileCheck %s \ +// RUN: -DINTRINSIC=dx.wave.multi.prefix.sum \ +// RUN: -DUINTRINSIC=dx.wave.multi.prefix.usum +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple spirv-pc-vulkan-library %s -emit-llvm \ +// RUN: -disable-llvm-passes -o - | FileCheck %s \ +// RUN: -DINTRINSIC=spv.subgroup.multi.prefix.sum \ +// RUN: -DUINTRINSIC=spv.subgroup.multi.prefix.sum + +// CHECK-LABEL: test_float +float test_float(float value, uint4 mask) { + // CHECK: %[[TOKEN:.*]] = call token @llvm.experimental.convergence.entry() + // CHECK: %[[MASK:.*]] = load <4 x i32> + // CHECK: %[[M0:.*]] = extractelement <4 x i32> %[[MASK]], i64 0 + // CHECK: %[[M1:.*]] = extractelement <4 x i32> %[[MASK]], i64 1 + // CHECK: %[[M2:.*]] = extractelement <4 x i32> %[[MASK]], i64 2 + // CHECK: %[[M3:.*]] = extractelement <4 x i32> %[[MASK]], i64 3 + // CHECK: %[[RET:.*]] = call {{.*}}float @llvm.[[INTRINSIC]].f32(float %{{.*}}, i32 %[[M0]], i32 %[[M1]], i32 %[[M2]], i32 %[[M3]]) [ "convergencectrl"(token %[[TOKEN]]) ] + // CHECK: ret float %[[RET]] + return WaveMultiPrefixSum(value, mask); +} + +// CHECK: declare float @llvm.[[INTRINSIC]].f32(float, i32, i32, i32, i32) #[[ATTR:[0-9]+]] + +// CHECK-LABEL: test_half +float16_t test_half(float16_t value) { + // CHECK: call {{.*}}half @llvm.[[INTRINSIC]].f16(half %{{.*}}, i32 1, i32 2, i32 4, i32 8) + return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_double +double test_double(double value) { + // CHECK: call {{.*}}double @llvm.[[INTRINSIC]].f64(double %{{.*}}, i32 1, i32 2, i32 4, i32 8) + return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int +int test_int(int value) { + // CHECK: call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + return __builtin_hlsl_wave_multi_prefix_sum(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int16 +int16_t test_int16(int16_t value) { + // CHECK: call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int64 +int64_t test_int64(int64_t value) { + // CHECK: call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint +uint test_uint(uint value) { + // CHECK: call i32 @llvm.[[UINTRINSIC]].i32(i32 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint16 +uint16_t test_uint16(uint16_t value) { + // CHECK: call i16 @llvm.[[UINTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint64 +uint64_t test_uint64(uint64_t value) { + // CHECK: call i64 @llvm.[[UINTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint2 +uint2 test_uint2(uint2 value, uint4 mask) { + // CHECK: call <2 x i32> @llvm.[[UINTRINSIC]].v2i32(<2 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) + return WaveMultiPrefixSum(value, mask); +} + +// CHECK-LABEL: test_float4 +float4 test_float4(float4 value, uint4 mask) { + // CHECK: call {{.*}}<4 x float> @llvm.[[INTRINSIC]].v4f32(<4 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) + return WaveMultiPrefixSum(value, mask); +} + +// CHECK-LABEL: test_uint_matrix +uint2x2 test_uint_matrix(uint2x2 value, uint4 mask) { + // CHECK: call <4 x i32> @llvm.[[UINTRINSIC]].v4i32(<4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) + return WaveMultiPrefixSum(value, mask); +} + +// CHECK-LABEL: test_float_matrix +float2x2 test_float_matrix(float2x2 value, uint4 mask) { + // CHECK: call {{.*}}<4 x float> @llvm.[[INTRINSIC]].v4f32(<4 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) + return WaveMultiPrefixSum(value, mask); +} + +// CHECK-LABEL: test_uint5 +vector<uint, 5> test_uint5(vector<uint, 5> value) { + // CHECK: call <5 x i32> @llvm.[[UINTRINSIC]].v5i32(<5 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_float5 +vector<float, 5> test_float5(vector<float, 5> value) { + // CHECK: call {{.*}}<5 x float> @llvm.[[INTRINSIC]].v5f32(<5 x float> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8)); +} + +// CHECK: attributes #[[ATTR]] = { convergent {{.*}}memory(none) } diff --git a/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl new file mode 100644 index 0000000000000..f6ffeee2e1eaf --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl @@ -0,0 +1,56 @@ +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple dxil-pc-shadermodel6.5-library -verify %s +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple spirv-pc-vulkan-library -verify %s +// RUN: %clang_cc1 -finclude-default-header -DTEST_AVAILABILITY \ +// RUN: -triple dxil-pc-shadermodel6.4-library -verify=availability %s + +#ifdef TEST_AVAILABILITY +[shader("compute")] +[numthreads(1, 1, 1)] +void main() { + (void)WaveMultiPrefixSum(1.0f, uint4(1, 2, 4, 8)); + // availability-error@-1 {{only available on Shader Model 6.5 or newer}} + // availability-note@*:* {{introduced in Shader Model 6.5}} +} +#else +void test_arg_count(float value, uint4 mask) { + __builtin_hlsl_wave_multi_prefix_sum(value); + // expected-error@-1 {{too few arguments to function call, expected 2}} + __builtin_hlsl_wave_multi_prefix_sum(value, mask, mask); + // expected-error@-1 {{too many arguments to function call, expected 2}} +} + +struct S { float value; }; + +void test_value_types(bool b, bool2 bv, bool2x2 bm, S s, uint4 mask) { + __builtin_hlsl_wave_multi_prefix_sum(b, mask); + // expected-error@-1 {{invalid operand of type 'bool'}} + __builtin_hlsl_wave_multi_prefix_sum(bv, mask); + // expected-error@-1 {{invalid operand of type 'bool2'}} + __builtin_hlsl_wave_multi_prefix_sum(bm, mask); + // expected-error@-1 {{matrix, integer or floating-point}} + // expected-error@-2 {{cannot pass object of non-trivial type 'bool2x2'}} + __builtin_hlsl_wave_multi_prefix_sum(s, mask); + // expected-error@-1 {{invalid operand of type 'S'}} +} + +void test_mask_types(float value, uint scalar, uint3 short_mask, + int4 signed_mask, float4 float_mask, + uint64_t4 wide_mask, S struct_mask) { + __builtin_hlsl_wave_multi_prefix_sum(value, scalar); + // expected-error@-1 {{passing 'uint' (aka 'unsigned int')}} + __builtin_hlsl_wave_multi_prefix_sum(value, short_mask); + // expected-error@-1 {{passing 'uint3' (aka 'vector<uint, 3>')}} + __builtin_hlsl_wave_multi_prefix_sum(value, signed_mask); + // expected-error@-1 {{passing 'int4' (aka 'vector<int, 4>')}} + __builtin_hlsl_wave_multi_prefix_sum(value, float_mask); + // expected-error@-1 {{passing 'float4' (aka 'vector<float, 4>')}} + __builtin_hlsl_wave_multi_prefix_sum(value, wide_mask); + // expected-error@-1 {{passing 'uint64_t4'}} + __builtin_hlsl_wave_multi_prefix_sum(value, struct_mask); + // expected-error@-1 {{passing 'S' to parameter of incompatible type}} +} +#endif diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index 1aaabaec97309..4a7b49831c9d4 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -508,6 +508,20 @@ let TargetPrefix = "dx" in { [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>; + def int_dx_wave_multi_prefix_sum + : DefaultAttrsIntrinsic<[llvm_any_ty], + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, + IntrTriviallyScalarizable]>; + + def int_dx_wave_multi_prefix_usum + : DefaultAttrsIntrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, + IntrTriviallyScalarizable]>; + def int_dx_wave_prefix_usum : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td index 926320653ab52..28e4d580bbcdf 100644 --- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -404,6 +404,13 @@ let TargetPrefix = "spv" in { [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>; + def int_spv_subgroup_multi_prefix_sum + : DefaultAttrsIntrinsic<[llvm_any_ty], + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, + IntrTriviallyScalarizable]>; + def int_spv_wave_prefix_product : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 361ddac0a1b3e..accd5be5801f1 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -324,6 +324,7 @@ defvar WaveBitOpKind_And = 0; defvar WaveBitOpKind_Or = 1; defvar WaveBitOpKind_Xor = 2; +defvar WaveMultiPrefixOpKind_Sum = 0; defvar WaveMultiPrefixOpKind_And = 1; defvar WaveMultiPrefixOpKind_Xor = 3; defvar WaveMultiPrefixOpKind_Product = 4; @@ -1593,6 +1594,20 @@ def WaveMultiPrefixOp : DXILOp<166, waveMultiPrefixOp> { IntrinArgIndex<3>, IntrinArgIndex<4>, IntrinArgI8<WaveMultiPrefixOpKind_Product>, IntrinArgI8<SignedOpKind_Unsigned> + ]>, + IntrinSelect<int_dx_wave_multi_prefix_sum, + [ + IntrinArgIndex<0>, IntrinArgIndex<1>, IntrinArgIndex<2>, + IntrinArgIndex<3>, IntrinArgIndex<4>, + IntrinArgI8<WaveMultiPrefixOpKind_Sum>, + IntrinArgI8<SignedOpKind_Signed> + ]>, + IntrinSelect<int_dx_wave_multi_prefix_usum, + [ + IntrinArgIndex<0>, IntrinArgIndex<1>, IntrinArgIndex<2>, + IntrinArgIndex<3>, IntrinArgIndex<4>, + IntrinArgI8<WaveMultiPrefixOpKind_Sum>, + IntrinArgI8<SignedOpKind_Unsigned> ]> ]; let arguments = [ diff --git a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp index ccfe6e9c947c0..cbbc17de32e96 100644 --- a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp +++ b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp @@ -116,6 +116,8 @@ static bool checkWaveOps(Intrinsic::ID IID) { case Intrinsic::dx_wave_multi_prefix_bit_count: case Intrinsic::dx_wave_multi_prefix_product: case Intrinsic::dx_wave_multi_prefix_uproduct: + case Intrinsic::dx_wave_multi_prefix_sum: + case Intrinsic::dx_wave_multi_prefix_usum: // Quad Op Variants case Intrinsic::dx_quad_read_across_x: case Intrinsic::dx_quad_read_across_y: diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp index 320156ee3d153..8bf0de1c60859 100644 --- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp +++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp @@ -24,6 +24,8 @@ bool DirectXTTIImpl::isTargetIntrinsicWithScalarOpAtArg( case Intrinsic::dx_wave_multi_prefix_bit_xor: case Intrinsic::dx_wave_multi_prefix_product: case Intrinsic::dx_wave_multi_prefix_uproduct: + case Intrinsic::dx_wave_multi_prefix_sum: + case Intrinsic::dx_wave_multi_prefix_usum: return ScalarOpdIdx >= 1 && ScalarOpdIdx <= 4; default: return false; diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index b72d5517e5ce9..c61ed5c02d8a8 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -5846,6 +5846,14 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, : SPIRV::OpGroupNonUniformIMul, "WaveMultiPrefixProduct"); } + case Intrinsic::spv_subgroup_multi_prefix_sum: { + bool IsFloatTy = + GR.isScalarOrVectorOfType(I.getOperand(2).getReg(), SPIRV::OpTypeFloat); + return selectWaveMultiPrefixOp(ResVReg, ResType, I, + IsFloatTy ? SPIRV::OpGroupNonUniformFAdd + : SPIRV::OpGroupNonUniformIAdd, + "WaveMultiPrefixSum"); + } case Intrinsic::spv_wave_prefix_product: return selectWaveExclusiveScanProduct(ResVReg, ResType, I); case Intrinsic::spv_quad_read_across_x: { diff --git a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll index 5ee508ab5c49c..f045f0f85b95a 100644 --- a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll +++ b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll @@ -105,6 +105,20 @@ define i32 @wave_multi_prefix_uproduct(i32 %x) { ret i32 %ret } +define float @wave_multi_prefix_sum(float %x) { + ; CHECK: Function wave_multi_prefix_sum : [[WAVE_FLAG]] + %ret = call float @llvm.dx.wave.multi.prefix.sum.f32( + float %x, i32 1, i32 2, i32 4, i32 8) + ret float %ret +} + +define i32 @wave_multi_prefix_usum(i32 %x) { + ; CHECK: Function wave_multi_prefix_usum : [[WAVE_FLAG]] + %ret = call i32 @llvm.dx.wave.multi.prefix.usum.i32( + i32 %x, i32 1, i32 2, i32 4, i32 8) + ret i32 %ret +} + define noundef i1 @wave_all_equal(i1 %x) { entry: ; CHECK: Function wave_all_equal : [[WAVE_FLAG]] diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixSum.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixSum.ll new file mode 100644 index 0000000000000..9ae6c051ddba1 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixSum.ll @@ -0,0 +1,81 @@ +; RUN: opt -S -scalarizer -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.5-library < %s | FileCheck %s + +; CHECK-LABEL: define float @test_float( +; CHECK: call float @dx.op.waveMultiPrefixOp.f32(i32 166, float %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 0, i8 0) +define float @test_float(float %value, i32 %m0, i32 %m1, + i32 %m2, i32 %m3) { + %ret = call float @llvm.dx.wave.multi.prefix.sum.f32( + float %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret float %ret +} + +; CHECK-LABEL: define half @test_half( +; CHECK: call half @dx.op.waveMultiPrefixOp.f16(i32 166, half %value, i32 1, i32 2, i32 4, i32 8, i8 0, i8 0) +define half @test_half(half %value) { + %ret = call half @llvm.dx.wave.multi.prefix.sum.f16( + half %value, i32 1, i32 2, i32 4, i32 8) + ret half %ret +} + +; CHECK-LABEL: define double @test_double( +; CHECK: call double @dx.op.waveMultiPrefixOp.f64(i32 166, double %value, i32 1, i32 2, i32 4, i32 8, i8 0, i8 0) +define double @test_double(double %value) { + %ret = call double @llvm.dx.wave.multi.prefix.sum.f64( + double %value, i32 1, i32 2, i32 4, i32 8) + ret double %ret +} + +; CHECK-LABEL: define i16 @test_i16( +; CHECK: call i16 @dx.op.waveMultiPrefixOp.i16(i32 166, i16 %value, i32 1, i32 2, i32 4, i32 8, i8 0, i8 0) +define i16 @test_i16(i16 %value) { + %ret = call i16 @llvm.dx.wave.multi.prefix.sum.i16( + i16 %value, i32 1, i32 2, i32 4, i32 8) + ret i16 %ret +} + +; CHECK-LABEL: define i32 @test_i32( +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value, i32 1, i32 2, i32 4, i32 8, i8 0, i8 0) +define i32 @test_i32(i32 %value) { + %ret = call i32 @llvm.dx.wave.multi.prefix.sum.i32( + i32 %value, i32 1, i32 2, i32 4, i32 8) + ret i32 %ret +} + +; CHECK-LABEL: define i64 @test_i64( +; CHECK: call i64 @dx.op.waveMultiPrefixOp.i64(i32 166, i64 %value, i32 1, i32 2, i32 4, i32 8, i8 0, i8 0) +define i64 @test_i64(i64 %value) { + %ret = call i64 @llvm.dx.wave.multi.prefix.sum.i64( + i64 %value, i32 1, i32 2, i32 4, i32 8) + ret i64 %ret +} + +; CHECK-LABEL: define i32 @test_unsigned( +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value, i32 1, i32 2, i32 4, i32 8, i8 0, i8 1) +define i32 @test_unsigned(i32 %value) { + %ret = call i32 @llvm.dx.wave.multi.prefix.usum.i32( + i32 %value, i32 1, i32 2, i32 4, i32 8) + ret i32 %ret +} + +; CHECK-LABEL: define <2 x i32> @test_unsigned_vector( +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i0, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 0, i8 1) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i1, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 0, i8 1) +define <2 x i32> @test_unsigned_vector(<2 x i32> %value, i32 %m0, i32 %m1, + i32 %m2, i32 %m3) { + %ret = call <2 x i32> @llvm.dx.wave.multi.prefix.usum.v2i32( + <2 x i32> %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret <2 x i32> %ret +} + +; CHECK-LABEL: define <5 x float> @test_long_vector( +; CHECK: call float @dx.op.waveMultiPrefixOp.f32(i32 166, float %value.i0, i32 1, i32 2, i32 4, i32 8, i8 0, i8 0) +; CHECK: call float @dx.op.waveMultiPrefixOp.f32(i32 166, float %value.i1, i32 1, i32 2, i32 4, i32 8, i8 0, i8 0) +; CHECK: call float @dx.op.waveMultiPrefixOp.f32(i32 166, float %value.i2, i32 1, i32 2, i32 4, i32 8, i8 0, i8 0) +; CHECK: call float @dx.op.waveMultiPrefixOp.f32(i32 166, float %value.i3, i32 1, i32 2, i32 4, i32 8, i8 0, i8 0) +; CHECK: call float @dx.op.waveMultiPrefixOp.f32(i32 166, float %value.i4, i32 1, i32 2, i32 4, i32 8, i8 0, i8 0) +define <5 x float> @test_long_vector(<5 x float> %value) { + %ret = call <5 x float> @llvm.dx.wave.multi.prefix.sum.v5f32( + <5 x float> %value, i32 1, i32 2, i32 4, i32 8) + ret <5 x float> %ret +} diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixSum_errors.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixSum_errors.ll new file mode 100644 index 0000000000000..face2902a9b90 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixSum_errors.ll @@ -0,0 +1,14 @@ +; RUN: not opt -S -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.4-library %s 2>&1 | \ +; RUN: FileCheck %s --check-prefix=VERSION +; RUN: not opt -S -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.5-library %s 2>&1 | \ +; RUN: FileCheck %s --check-prefix=TYPE + +; VERSION: Cannot create WaveMultiPrefixOp operation: No valid overloads for DXIL version 1.4 +; TYPE: Cannot create WaveMultiPrefixOp operation: Invalid overload type +define i1 @test_bool(i1 %value) { + %ret = call i1 @llvm.dx.wave.multi.prefix.sum.i1( + i1 %value, i32 1, i32 2, i32 4, i32 8) + ret i1 %ret +} diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixSum.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixSum.ll new file mode 100644 index 0000000000000..5b63c53e8c690 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixSum.ll @@ -0,0 +1,147 @@ +; DEFINE: %{nv} = SPV_NV_shader_subgroup_partitioned +; DEFINE: %{ext} = SPV_EXT_shader_subgroup_partitioned +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{nv} %s -o - | FileCheck %s -DEXTENSION=NV \ +; RUN: --implicit-check-not=%{ext} +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{ext} %s -o - | FileCheck %s -DEXTENSION=EXT \ +; RUN: --implicit-check-not=%{nv} +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{nv},+%{ext} %s -o - | FileCheck %s -DEXTENSION=EXT \ +; RUN: --implicit-check-not=%{nv} +; RUN: not llc -O0 -mtriple=spirv-vulkan-unknown %s -o - 2>&1 | \ +; RUN: FileCheck %s --check-prefix=NO-EXT +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{nv} %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{ext} %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{nv},+%{ext} %s -o - \ +; RUN: -filetype=obj | spirv-val %} + +; NO-EXT: WaveMultiPrefixSum requires SPV_EXT_shader_subgroup_partitioned or SPV_NV_shader_subgroup_partitioned + +; CHECK-DAG: OpCapability GroupNonUniformPartitionedNV +; CHECK-DAG: OpExtension "SPV_[[EXTENSION]]_shader_subgroup_partitioned" +; CHECK-DAG: %[[#I16:]] = OpTypeInt 16 0 +; CHECK-DAG: %[[#I32:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#I64:]] = OpTypeInt 64 0 +; CHECK-DAG: %[[#F16:]] = OpTypeFloat 16 +; CHECK-DAG: %[[#F32:]] = OpTypeFloat 32 +; CHECK-DAG: %[[#F64:]] = OpTypeFloat 64 +; CHECK-DAG: %[[#V2I32:]] = OpTypeVector %[[#I32]] 2 +; CHECK-DAG: %[[#V4I32:]] = OpTypeVector %[[#I32]] 4 +; CHECK-DAG: %[[#V4F32:]] = OpTypeVector %[[#F32]] 4 +; CHECK-DAG: %[[#SCOPE:]] = OpConstant %[[#I32]] 3 +; CHECK-DAG: %[[#C1:]] = OpConstant %[[#I32]] 1 +; CHECK-DAG: %[[#C2:]] = OpConstant %[[#I32]] 2 +; CHECK-DAG: %[[#C4:]] = OpConstant %[[#I32]] 4 +; CHECK-DAG: %[[#C8:]] = OpConstant %[[#I32]] 8 + +@wide_float = internal addrspace(10) global [5 x float] zeroinitializer + +; CHECK-LABEL: Begin function test_float +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#F32]] +; CHECK: %[[#M0:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M1:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M2:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M3:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#M0]] %[[#M1]] %[[#M2]] %[[#M3]] +; CHECK: %[[#RET:]] = OpGroupNonUniformFAdd %[[#F32]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define float @test_float(float %value, i32 %m0, i32 %m1, + i32 %m2, i32 %m3) { + %ret = call float @llvm.spv.subgroup.multi.prefix.sum.f32( + float %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret float %ret +} + +; CHECK-LABEL: Begin function test_half +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#F16]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformFAdd %[[#F16]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define half @test_half(half %value) { + %ret = call half @llvm.spv.subgroup.multi.prefix.sum.f16( + half %value, i32 1, i32 2, i32 4, i32 8) + ret half %ret +} + +; CHECK-LABEL: Begin function test_double +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#F64]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformFAdd %[[#F64]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define double @test_double(double %value) { + %ret = call double @llvm.spv.subgroup.multi.prefix.sum.f64( + double %value, i32 1, i32 2, i32 4, i32 8) + ret double %ret +} + +; CHECK-LABEL: Begin function test_i16 +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I16]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformIAdd %[[#I16]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define i16 @test_i16(i16 %value) { + %ret = call i16 @llvm.spv.subgroup.multi.prefix.sum.i16( + i16 %value, i32 1, i32 2, i32 4, i32 8) + ret i16 %ret +} + +; CHECK-LABEL: Begin function test_i32 +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformIAdd %[[#I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define i32 @test_i32(i32 %value) { + %ret = call i32 @llvm.spv.subgroup.multi.prefix.sum.i32( + i32 %value, i32 1, i32 2, i32 4, i32 8) + ret i32 %ret +} + +; CHECK-LABEL: Begin function test_i64 +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I64]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformIAdd %[[#I64]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define i64 @test_i64(i64 %value) { + %ret = call i64 @llvm.spv.subgroup.multi.prefix.sum.i64( + i64 %value, i32 1, i32 2, i32 4, i32 8) + ret i64 %ret +} + +; CHECK-LABEL: Begin function test_int_vector +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#V2I32]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformIAdd %[[#V2I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define <2 x i32> @test_int_vector(<2 x i32> %value) { + %ret = call <2 x i32> @llvm.spv.subgroup.multi.prefix.sum.v2i32( + <2 x i32> %value, i32 1, i32 2, i32 4, i32 8) + ret <2 x i32> %ret +} + +; CHECK-LABEL: Begin function test_float_vector +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#V4F32]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformFAdd %[[#V4F32]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define <4 x float> @test_float_vector(<4 x float> %value) { + %ret = call <4 x float> @llvm.spv.subgroup.multi.prefix.sum.v4f32( + <4 x float> %value, i32 1, i32 2, i32 4, i32 8) + ret <4 x float> %ret +} + +; CHECK-LABEL: Begin function test_long_vector +; CHECK: %[[#MASK0:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: OpGroupNonUniformFAdd %[[#V4F32]] %[[#SCOPE]] PartitionedExclusiveScanNV %{{[0-9]+}} %[[#MASK0]] +; CHECK: %[[#MASK1:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: OpGroupNonUniformFAdd %[[#F32]] %[[#SCOPE]] PartitionedExclusiveScanNV %{{[0-9]+}} %[[#MASK1]] +define void @test_long_vector() { + %value = load <5 x float>, ptr addrspace(10) @wide_float + %ret = call <5 x float> @llvm.spv.subgroup.multi.prefix.sum.v5f32( + <5 x float> %value, i32 1, i32 2, i32 4, i32 8) + store <5 x float> %ret, ptr addrspace(10) @wide_float + ret void +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
