https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/238083
Add WaveMultiPrefixBitAnd for Shader Model 6.5. Add Clang type checks and code generation for integer scalar, vector, and matrix values. Add DXIL lowering and SPIR-V partitioned subgroup scans. The SPIR-V intrinsic uses the subgroup name and requires SPV_NV_shader_subgroup_partitioned. This is the first PR in a planned stack of five WaveMultiPrefix builtins. An existing SPIR-V issue affects five-element vector function parameters. Memory-based long vectors work. This PR does not change that separate issue. Fixes #99208. >From 56183dcedb3d892a3983cfa9e42d3f1da5bfc0ef Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Mon, 2 Nov 2026 12:01:32 -0800 Subject: [PATCH] first attempt Co-authored-by: Copilot <[email protected]> Copilot-Session: 40997c2f-5aaa-41c6-8d1f-3e2e9ada315f --- clang/include/clang/Basic/Builtins.td | 6 ++ clang/include/clang/Basic/HLSLIntrinsics.td | 10 ++ clang/lib/CodeGen/CGHLSLBuiltins.cpp | 10 ++ clang/lib/CodeGen/CGHLSLRuntime.h | 3 + clang/lib/Sema/SemaHLSL.cpp | 30 ++++++ .../builtins/WaveMultiPrefixBitAnd.hlsl | 97 +++++++++++++++++++ .../WaveMultiPrefixBitAnd-errors.hlsl | 64 ++++++++++++ llvm/include/llvm/IR/IntrinsicsDirectX.td | 8 ++ llvm/include/llvm/IR/IntrinsicsSPIRV.td | 7 ++ llvm/lib/Target/DirectX/DXIL.td | 22 +++++ llvm/lib/Target/DirectX/DXILShaderFlags.cpp | 1 + .../DirectX/DirectXTargetTransformInfo.cpp | 2 + .../Target/SPIRV/SPIRVInstructionSelector.cpp | 39 ++++++++ llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 1 + .../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 4 +- .../CodeGen/DirectX/ShaderFlags/wave-ops.ll | 7 ++ .../CodeGen/DirectX/WaveMultiPrefixBitAnd.ll | 48 +++++++++ .../DirectX/WaveMultiPrefixBitAnd_errors.ll | 14 +++ .../hlsl-intrinsics/WaveMultiPrefixBitAnd.ll | 85 ++++++++++++++++ 19 files changed, 457 insertions(+), 1 deletion(-) create mode 100644 clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitAnd.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitAnd-errors.hlsl create mode 100644 llvm/test/CodeGen/DirectX/WaveMultiPrefixBitAnd.ll create mode 100644 llvm/test/CodeGen/DirectX/WaveMultiPrefixBitAnd_errors.ll create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitAnd.ll diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index d04fd899eb63a6..5ae3edcc6601c3 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -5789,6 +5789,12 @@ def HLSLWavePrefixSum : LangBuiltin<"HLSL_LANG"> { let Prototype = "void(...)"; } +def HLSLWaveMultiPrefixBitAnd : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_wave_multi_prefix_bit_and"]; + let Attributes = [NoThrow, Const]; + let Prototype = "void(...)"; +} + def HLSLWavePrefixProduct : LangBuiltin<"HLSL_LANG"> { let Spellings = ["__builtin_hlsl_wave_prefix_product"]; let Attributes = [NoThrow, Const]; diff --git a/clang/include/clang/Basic/HLSLIntrinsics.td b/clang/include/clang/Basic/HLSLIntrinsics.td index fd59e3477b8c79..902a31d8278b37 100644 --- a/clang/include/clang/Basic/HLSLIntrinsics.td +++ b/clang/include/clang/Basic/HLSLIntrinsics.td @@ -1883,6 +1883,16 @@ def hlsl_wave_is_first_lane : let Availability = SM6_0; } +def hlsl_wave_multi_prefix_bit_and : + HLSLTwoArgBuiltin<"WaveMultiPrefixBitAnd", + "__builtin_hlsl_wave_multi_prefix_bit_and"> { + let Args = [Varying, VectorType<UIntTy, 4>]; + let VaryingTypes = AllIntTypes; + let VaryingLongVector = 1; + let IsConvergent = 1; + let Availability = SM6_5; +} + // Returns the count of bits set to 1 on prior lanes. def hlsl_wave_prefix_count_bits : HLSLBuiltin<"WavePrefixCountBits", diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp index fc422918499ed2..d4f5f47d9b24e9 100644 --- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp +++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp @@ -1721,6 +1721,16 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, return EmitIntrinsicCall(IID, {OpExpr->getType()}, ArrayRef{OpExpr}, "hlsl.wave.prefix.sum"); } + case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: { + Value *ValueOp = EmitScalarExpr(E->getArg(0)); + Value *Mask = EmitScalarExpr(E->getArg(1)); + SmallVector<Value *, 5> Args{ValueOp}; + for (unsigned I = 0; I != 4; ++I) + Args.push_back(Builder.CreateExtractElement(Mask, I)); + return EmitIntrinsicCall( + CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(), + {ValueOp->getType()}, Args, "hlsl.wave.multi.prefix.bit.and"); + } case Builtin::BI__builtin_hlsl_wave_prefix_product: { Value *OpExpr = EmitScalarExpr(E->getArg(0)); Intrinsic::ID IID = getWavePrefixProductIntrinsic( diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h index 1957b946826d37..89b52f287d9ed7 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/clang/lib/CodeGen/CGHLSLRuntime.h @@ -149,6 +149,9 @@ class CGHLSLRuntime { GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitOr, wave_reduce_or) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitXor, wave_reduce_xor) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitAnd, wave_reduce_and) + GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixBitAnd, + wave_multi_prefix_bit_and, + subgroup_multi_prefix_bit_and) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min) diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp index 746c45662030a1..6147a0446a838a 100644 --- a/clang/lib/Sema/SemaHLSL.cpp +++ b/clang/lib/Sema/SemaHLSL.cpp @@ -5377,6 +5377,36 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { TheCall->setType(ArgTyExpr); break; } + case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: { + if (SemaRef.checkArgCount(TheCall, 2)) + return true; + + if (CheckAnyScalarOrVectorOrMatrix(&SemaRef, TheCall, 0) || + CheckNotBoolScalarOrVector(&SemaRef, TheCall, 0)) + return true; + + QualType ValueTy = TheCall->getArg(0)->getType(); + QualType ElementTy = getScalarComponentType(ValueTy); + if (!ElementTy->isIntegerType() || ElementTy->isBooleanType()) { + SemaRef.Diag(TheCall->getArg(0)->getBeginLoc(), + diag::err_builtin_invalid_arg_type) + << 1 << (ValueTy->isMatrixType() ? 3 : 5) << 1 << 0 << ValueTy; + return true; + } + + QualType MaskTy = TheCall->getArg(1)->getType(); + QualType UInt4Ty = + getASTContext().getExtVectorType(getASTContext().UnsignedIntTy, 4); + if (!getASTContext().hasSameUnqualifiedType(MaskTy, UInt4Ty)) { + SemaRef.Diag(TheCall->getArg(1)->getBeginLoc(), + diag::err_typecheck_convert_incompatible) + << MaskTy << UInt4Ty << 1 << 0 << 0; + return true; + } + + TheCall->setType(ValueTy); + break; + } case Builtin::BI__builtin_hlsl_quad_read_across_x: case Builtin::BI__builtin_hlsl_quad_read_across_y: case Builtin::BI__builtin_hlsl_quad_read_across_diagonal: { diff --git a/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitAnd.hlsl b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitAnd.hlsl new file mode 100644 index 00000000000000..d266d9752b8471 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitAnd.hlsl @@ -0,0 +1,97 @@ +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \ +// RUN: -disable-llvm-passes -o - | \ +// RUN: FileCheck %s -DINTRINSIC=dx.wave.multi.prefix.bit.and +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple spirv-pc-vulkan-library %s -emit-llvm \ +// RUN: -disable-llvm-passes -o - | \ +// RUN: FileCheck %s -DINTRINSIC=spv.subgroup.multi.prefix.bit.and + +// CHECK-LABEL: test_uint +uint test_uint(uint value, uint4 mask) { + // CHECK: %[[TOKEN:.*]] = call token @llvm.experimental.convergence.entry() + // CHECK: %[[MASK:.*]] = load <4 x i32> + // CHECK: %[[M0:.*]] = extractelement <4 x i32> %[[MASK]], i64 0 + // CHECK: %[[M1:.*]] = extractelement <4 x i32> %[[MASK]], i64 1 + // CHECK: %[[M2:.*]] = extractelement <4 x i32> %[[MASK]], i64 2 + // CHECK: %[[M3:.*]] = extractelement <4 x i32> %[[MASK]], i64 3 + // CHECK: %[[RET:.*]] = call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 %[[M0]], i32 %[[M1]], i32 %[[M2]], i32 %[[M3]]) [ "convergencectrl"(token %[[TOKEN]]) ] + // CHECK: ret i32 %[[RET]] + return WaveMultiPrefixBitAnd(value, mask); +} + +// CHECK: declare i32 @llvm.[[INTRINSIC]].i32(i32, i32, i32, i32, i32) #[[ATTR:[0-9]+]] + +// CHECK-LABEL: test_int +int test_int(int value) { + // CHECK: %[[RET:.*]] = call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret i32 %[[RET]] + return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int16 +int16_t test_int16(int16_t value) { + // CHECK: %[[RET:.*]] = call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret i16 %[[RET]] + return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint16 +uint16_t test_uint16(uint16_t value) { + // CHECK: %[[RET:.*]] = call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret i16 %[[RET]] + return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int64 +int64_t test_int64(int64_t value) { + // CHECK: %[[RET:.*]] = call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret i64 %[[RET]] + return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint64 +uint64_t test_uint64(uint64_t value) { + // CHECK: %[[RET:.*]] = call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret i64 %[[RET]] + return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint2 +uint2 test_uint2(uint2 value) { + // CHECK: %[[RET:.*]] = call <2 x i32> @llvm.[[INTRINSIC]].v2i32(<2 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret <2 x i32> %[[RET]] + return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int3 +int3 test_int3(int3 value) { + // CHECK: %[[RET:.*]] = call <3 x i32> @llvm.[[INTRINSIC]].v3i32(<3 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret <3 x i32> %[[RET]] + return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint64_t4 +uint64_t4 test_uint64_t4(uint64_t4 value) { + // CHECK: %[[RET:.*]] = call <4 x i64> @llvm.[[INTRINSIC]].v4i64(<4 x i64> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret <4 x i64> %[[RET]] + return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint5 +vector<uint, 5> test_uint5(vector<uint, 5> value) { + // CHECK: %[[RET:.*]] = call <5 x i32> @llvm.[[INTRINSIC]].v5i32(<5 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret <5 x i32> %[[RET]] + return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int2x3 +int2x3 test_int2x3(int2x3 value) { + // CHECK: %[[RET:.*]] = call <6 x i32> @llvm.[[INTRINSIC]].v6i32(<6 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret <6 x i32> %[[RET]] + return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8)); +} + +// CHECK: attributes #[[ATTR]] = { {{.*}}convergent {{.*}} } diff --git a/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitAnd-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitAnd-errors.hlsl new file mode 100644 index 00000000000000..ae4142001c037e --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitAnd-errors.hlsl @@ -0,0 +1,64 @@ +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple dxil-pc-shadermodel6.5-library -verify %s +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple spirv-pc-vulkan-library -verify %s +// RUN: %clang_cc1 -finclude-default-header -DTEST_AVAILABILITY \ +// RUN: -triple dxil-pc-shadermodel6.4-library -verify=availability %s + +#ifdef TEST_AVAILABILITY +[shader("compute")] +[numthreads(1, 1, 1)] +void main() { + (void)WaveMultiPrefixBitAnd(1u, uint4(1, 2, 4, 8)); + // availability-error@-1 {{'WaveMultiPrefixBitAnd' is only available on Shader Model 6.5 or newer}} + // availability-note@hlsl/hlsl_alias_intrinsics_gen.inc:* {{'WaveMultiPrefixBitAnd' has been marked as being introduced in Shader Model 6.5 here}} +} +#else +void test_arg_count(uint value, uint4 mask) { + __builtin_hlsl_wave_multi_prefix_bit_and(value); + // expected-error@-1 {{too few arguments to function call, expected 2, have 1}} + __builtin_hlsl_wave_multi_prefix_bit_and(value, mask, mask); + // expected-error@-1 {{too many arguments to function call, expected 2, have 3}} +} + +struct S { uint value; }; + +void test_value_types(float f, float2 fv, bool b, bool2 bv, S s, + float2x2 fm, bool2x2 bm, uint4 mask) { + __builtin_hlsl_wave_multi_prefix_bit_and(f, mask); + // expected-error@-1 {{1st argument must be a scalar or vector of integer}} + __builtin_hlsl_wave_multi_prefix_bit_and(fv, mask); + // expected-error@-1 {{1st argument must be a scalar or vector of integer}} + __builtin_hlsl_wave_multi_prefix_bit_and(b, mask); + // expected-error@-1 {{invalid operand of type 'bool'}} + __builtin_hlsl_wave_multi_prefix_bit_and(bv, mask); + // expected-error@-1 {{invalid operand of type 'bool2'}} + __builtin_hlsl_wave_multi_prefix_bit_and(s, mask); + // expected-error@-1 {{invalid operand of type 'S'}} + __builtin_hlsl_wave_multi_prefix_bit_and(fm, mask); + // expected-error@-1 {{1st argument must be a matrix integer type}} + // expected-error@-2 {{cannot pass object of non-trivial type 'float2x2'}} + __builtin_hlsl_wave_multi_prefix_bit_and(bm, mask); + // expected-error@-1 {{1st argument must be a matrix integer type}} + // expected-error@-2 {{cannot pass object of non-trivial type 'bool2x2'}} +} + +void test_mask_types(uint value, uint scalar, uint3 short_mask, + int4 signed_mask, float4 float_mask, + uint64_t4 wide_mask, S struct_mask) { + __builtin_hlsl_wave_multi_prefix_bit_and(value, scalar); + // expected-error@-1 {{passing 'uint' (aka 'unsigned int') to parameter of incompatible type 'vector<unsigned int, 4>'}} + __builtin_hlsl_wave_multi_prefix_bit_and(value, short_mask); + // expected-error@-1 {{passing 'uint3' (aka 'vector<uint, 3>') to parameter of incompatible type 'vector<unsigned int, 4>'}} + __builtin_hlsl_wave_multi_prefix_bit_and(value, signed_mask); + // expected-error@-1 {{passing 'int4' (aka 'vector<int, 4>') to parameter of incompatible type 'vector<unsigned int, 4>'}} + __builtin_hlsl_wave_multi_prefix_bit_and(value, float_mask); + // expected-error@-1 {{passing 'float4' (aka 'vector<float, 4>') to parameter of incompatible type 'vector<unsigned int, 4>'}} + __builtin_hlsl_wave_multi_prefix_bit_and(value, wide_mask); + // expected-error@-1 {{passing 'uint64_t4'}} + __builtin_hlsl_wave_multi_prefix_bit_and(value, struct_mask); + // expected-error@-1 {{passing 'S' to parameter of incompatible type 'vector<unsigned int, 4>'}} +} +#endif diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index b1cb80cd96f143..4f23541e7eeed0 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -473,6 +473,14 @@ let TargetPrefix = "dx" in { [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>; + // Scalar mask words preserve the lane mask during vector scalarization. + def int_dx_wave_multi_prefix_bit_and + : DefaultAttrsIntrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, + IntrTriviallyScalarizable]>; + def int_dx_wave_prefix_usum : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td index f9319918694d7d..2b1127445f7990 100644 --- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -383,6 +383,13 @@ let TargetPrefix = "spv" in { [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>; + def int_spv_subgroup_multi_prefix_bit_and + : DefaultAttrsIntrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, + IntrTriviallyScalarizable]>; + def int_spv_wave_prefix_product : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index a3a373d79f73cb..c2165c9664a3d6 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -324,6 +324,8 @@ defvar WaveBitOpKind_And = 0; defvar WaveBitOpKind_Or = 1; defvar WaveBitOpKind_Xor = 2; +defvar WaveMultiPrefixOpKind_And = 1; + defvar SignedOpKind_Signed = 0; defvar SignedOpKind_Unsigned = 1; @@ -1559,6 +1561,26 @@ def Dot4AddU8Packed : DXILOp<164, dot4AddPacked> { let attributes = [Attributes<DXIL1_4, [DXILReadNone]>]; } +def WaveMultiPrefixOp : DXILOp<166, waveMultiPrefixOp> { + let Doc = "returns the prefix result for lanes selected by a mask"; + let intrinsics = [ + IntrinSelect<int_dx_wave_multi_prefix_bit_and, + [ + IntrinArgIndex<0>, IntrinArgIndex<1>, IntrinArgIndex<2>, + IntrinArgIndex<3>, IntrinArgIndex<4>, + IntrinArgI8<WaveMultiPrefixOpKind_And>, + IntrinArgI8<SignedOpKind_Signed> + ]> + ]; + let arguments = [ + OverloadTy, Int32Ty, Int32Ty, Int32Ty, Int32Ty, Int8Ty, Int8Ty + ]; + let result = OverloadTy; + let overloads = [Overloads<DXIL1_5, [Int16Ty, Int32Ty, Int64Ty]>]; + let stages = [Stages<DXIL1_5, [all_stages]>]; + let attributes = [Attributes<DXIL1_5, []>]; +} + def AnnotateHandle : DXILOp<216, annotateHandle> { let Doc = "annotate handle with resource properties"; let arguments = [HandleTy, ResPropsTy]; diff --git a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp index 9c709cf5a1ca1c..1c27d16e05979b 100644 --- a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp +++ b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp @@ -111,6 +111,7 @@ static bool checkWaveOps(Intrinsic::ID IID) { case Intrinsic::dx_wave_prefix_usum: case Intrinsic::dx_wave_prefix_product: case Intrinsic::dx_wave_prefix_uproduct: + case Intrinsic::dx_wave_multi_prefix_bit_and: // Quad Op Variants case Intrinsic::dx_quad_read_across_x: case Intrinsic::dx_quad_read_across_y: diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp index 6c65fb7b9e0fc7..6fa1d275334423 100644 --- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp +++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp @@ -20,6 +20,8 @@ bool DirectXTTIImpl::isTargetIntrinsicWithScalarOpAtArg( switch (ID) { case Intrinsic::dx_wave_readlane: return ScalarOpdIdx == 1; + case Intrinsic::dx_wave_multi_prefix_bit_and: + return ScalarOpdIdx >= 1 && ScalarOpdIdx <= 4; default: return false; } diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 71040b3b85faf1..ea7723cea88fef 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -322,6 +322,9 @@ class SPIRVInstructionSelector : public InstructionSelector { bool selectWaveExclusiveScanProduct(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const; + bool selectWaveMultiPrefixBitAnd(Register ResVReg, SPIRVTypeInst ResType, + MachineInstr &I) const; + bool selectQuadSwap(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, unsigned Direction) const; @@ -4011,6 +4014,40 @@ bool SPIRVInstructionSelector::selectWaveExclusiveScan( return true; } +bool SPIRVInstructionSelector::selectWaveMultiPrefixBitAnd( + Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { + if (!STI.canUseExtension( + SPIRV::Extension::SPV_NV_shader_subgroup_partitioned)) + return diagnoseUnsupported( + I, "WaveMultiPrefixBitAnd requires the SPIR-V extension " + "SPV_NV_shader_subgroup_partitioned"); + + assert(I.getNumOperands() == 7); + SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); + SPIRVTypeInst MaskTy = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); + Register Mask = MRI->createVirtualRegister(GR.getRegClass(MaskTy)); + GR.assignSPIRVTypeToVReg(MaskTy, Mask, MRI->getMF()); + auto MaskMIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), + TII.get(SPIRV::OpCompositeConstruct)) + .addDef(Mask) + .addUse(GR.getSPIRVTypeID(MaskTy)); + for (unsigned Op = 3; Op != 7; ++Op) + MaskMIB.addUse(I.getOperand(Op).getReg()); + MaskMIB.constrainAllUses(TII, TRI, RBI); + + BuildMI(*I.getParent(), I, I.getDebugLoc(), + TII.get(SPIRV::OpGroupNonUniformBitwiseAnd)) + .addDef(ResVReg) + .addUse(GR.getSPIRVTypeID(ResType)) + .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII, + !STI.isShader())) + .addImm(SPIRV::GroupOperation::PartitionedExclusiveScanNV) + .addUse(I.getOperand(2).getReg()) + .addUse(Mask) + .constrainAllUses(TII, TRI, RBI); + return true; +} + bool SPIRVInstructionSelector::selectQuadSwap(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, @@ -5789,6 +5826,8 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, SPIRV::OpGroupNonUniformBroadcastFirst); case Intrinsic::spv_wave_prefix_sum: return selectWaveExclusiveScanSum(ResVReg, ResType, I); + case Intrinsic::spv_subgroup_multi_prefix_bit_and: + return selectWaveMultiPrefixBitAnd(ResVReg, ResType, I); case Intrinsic::spv_wave_prefix_product: return selectWaveExclusiveScanProduct(ResVReg, ResType, I); case Intrinsic::spv_quad_read_across_x: { diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp index f34a4888a66990..28e514461523af 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -1992,6 +1992,7 @@ void addInstrRequirements(const MachineInstr &MI, case SPIRV::GroupOperation::PartitionedReduceNV: case SPIRV::GroupOperation::PartitionedInclusiveScanNV: case SPIRV::GroupOperation::PartitionedExclusiveScanNV: + Reqs.addExtension(SPIRV::Extension::SPV_NV_shader_subgroup_partitioned); Reqs.addCapability(SPIRV::Capability::GroupNonUniformPartitionedNV); break; } diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td index ff7335ed095822..3ffa840dd9c804 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -561,7 +561,9 @@ defm SubgroupImageMediaBlockIOINTEL : CapabilityOperand<5579, 0, 0, [SPV_INTEL_m defm SubgroupAvcMotionEstimationINTEL : CapabilityOperand<5696, 0, 0, [], []>; defm SubgroupAvcMotionEstimationIntraINTEL : CapabilityOperand<5697, 0, 0, [], []>; defm SubgroupAvcMotionEstimationChromaINTEL : CapabilityOperand<5698, 0, 0, [], []>; -defm GroupNonUniformPartitionedNV : CapabilityOperand<5297, 0, 0, [], []>; +defm GroupNonUniformPartitionedNV : + CapabilityOperand<5297, 0, 0, [SPV_NV_shader_subgroup_partitioned], + [GroupNonUniform]>; defm VulkanMemoryModelKHR : CapabilityOperand<5345, 0, 0, [], []>; defm VulkanMemoryModelDeviceScopeKHR : CapabilityOperand<5346, 0, 0, [], []>; defm ImageFootprintNV : CapabilityOperand<5282, 0, 0, [], []>; diff --git a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll index 37a991e1c1fa8e..55dd18c81f420c 100644 --- a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll +++ b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll @@ -70,6 +70,13 @@ entry: ret i32 %ret } +define i32 @wave_multi_prefix_bit_and(i32 %x) { + ; CHECK: Function wave_multi_prefix_bit_and : [[WAVE_FLAG]] + %ret = call i32 @llvm.dx.wave.multi.prefix.bit.and.i32( + i32 %x, i32 1, i32 2, i32 4, i32 8) + ret i32 %ret +} + define noundef i1 @wave_all_equal(i1 %x) { entry: ; CHECK: Function wave_all_equal : [[WAVE_FLAG]] diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitAnd.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitAnd.ll new file mode 100644 index 00000000000000..0cace3953812af --- /dev/null +++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitAnd.ll @@ -0,0 +1,48 @@ +; RUN: opt -S -scalarizer -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.5-library < %s | FileCheck %s + +; CHECK-LABEL: define i16 @test_i16( +; CHECK: call i16 @dx.op.waveMultiPrefixOp.i16(i32 166, i16 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 1, i8 0) +define i16 @test_i16(i16 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) { + %ret = call i16 @llvm.dx.wave.multi.prefix.bit.and.i16( + i16 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret i16 %ret +} + +; CHECK-LABEL: define i32 @test_i32( +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 1, i8 0) +define i32 @test_i32(i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) { + %ret = call i32 @llvm.dx.wave.multi.prefix.bit.and.i32( + i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret i32 %ret +} + +; CHECK-LABEL: define i64 @test_i64( +; CHECK: call i64 @dx.op.waveMultiPrefixOp.i64(i32 166, i64 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 1, i8 0) +define i64 @test_i64(i64 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) { + %ret = call i64 @llvm.dx.wave.multi.prefix.bit.and.i64( + i64 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret i64 %ret +} + +; CHECK-LABEL: define <2 x i32> @test_vector( +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i0, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 1, i8 0) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i1, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 1, i8 0) +define <2 x i32> @test_vector(<2 x i32> %value, i32 %m0, i32 %m1, + i32 %m2, i32 %m3) { + %ret = call <2 x i32> @llvm.dx.wave.multi.prefix.bit.and.v2i32( + <2 x i32> %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret <2 x i32> %ret +} + +; CHECK-LABEL: define <5 x i32> @test_long_vector( +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i0, i32 1, i32 2, i32 4, i32 8, i8 1, i8 0) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i1, i32 1, i32 2, i32 4, i32 8, i8 1, i8 0) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i2, i32 1, i32 2, i32 4, i32 8, i8 1, i8 0) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i3, i32 1, i32 2, i32 4, i32 8, i8 1, i8 0) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i4, i32 1, i32 2, i32 4, i32 8, i8 1, i8 0) +define <5 x i32> @test_long_vector(<5 x i32> %value) { + %ret = call <5 x i32> @llvm.dx.wave.multi.prefix.bit.and.v5i32( + <5 x i32> %value, i32 1, i32 2, i32 4, i32 8) + ret <5 x i32> %ret +} diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitAnd_errors.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitAnd_errors.ll new file mode 100644 index 00000000000000..0f87b37d1c7e13 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitAnd_errors.ll @@ -0,0 +1,14 @@ +; RUN: not opt -S -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.4-library %s 2>&1 | \ +; RUN: FileCheck %s --check-prefix=VERSION +; RUN: not opt -S -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.5-library %s 2>&1 | \ +; RUN: FileCheck %s --check-prefix=TYPE + +; VERSION: Cannot create WaveMultiPrefixOp operation: No valid overloads for DXIL version 1.4 +; TYPE: Cannot create WaveMultiPrefixOp operation: Invalid overload type +define i8 @test_i8(i8 %value) { + %ret = call i8 @llvm.dx.wave.multi.prefix.bit.and.i8( + i8 %value, i32 1, i32 2, i32 4, i32 8) + ret i8 %ret +} diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitAnd.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitAnd.ll new file mode 100644 index 00000000000000..a8e7bd23220e72 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitAnd.ll @@ -0,0 +1,85 @@ +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+SPV_NV_shader_subgroup_partitioned %s -o - | FileCheck %s +; RUN: not llc -O0 -mtriple=spirv-vulkan-unknown %s -o - 2>&1 | \ +; RUN: FileCheck %s --check-prefix=NO-EXT +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+SPV_NV_shader_subgroup_partitioned %s -o - \ +; RUN: -filetype=obj | spirv-val %} + +; NO-EXT: WaveMultiPrefixBitAnd requires the SPIR-V extension SPV_NV_shader_subgroup_partitioned + +; CHECK-DAG: OpCapability GroupNonUniformPartitionedNV +; CHECK-DAG: OpExtension "SPV_NV_shader_subgroup_partitioned" +; CHECK-DAG: %[[#I16:]] = OpTypeInt 16 0 +; CHECK-DAG: %[[#I32:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#I64:]] = OpTypeInt 64 0 +; CHECK-DAG: %[[#V2I32:]] = OpTypeVector %[[#I32]] 2 +; CHECK-DAG: %[[#V4I32:]] = OpTypeVector %[[#I32]] 4 +; CHECK-DAG: %[[#SCOPE:]] = OpConstant %[[#I32]] 3 +; CHECK-DAG: %[[#C1:]] = OpConstant %[[#I32]] 1 +; CHECK-DAG: %[[#C2:]] = OpConstant %[[#I32]] 2 +; CHECK-DAG: %[[#C4:]] = OpConstant %[[#I32]] 4 +; CHECK-DAG: %[[#C8:]] = OpConstant %[[#I32]] 8 + +@wide_i32 = internal addrspace(10) global [5 x i32] zeroinitializer + +; CHECK-LABEL: Begin function test_i32 +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M0:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M1:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M2:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M3:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#M0]] %[[#M1]] %[[#M2]] %[[#M3]] +; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseAnd %[[#I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define i32 @test_i32(i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) { + %ret = call i32 @llvm.spv.subgroup.multi.prefix.bit.and.i32( + i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret i32 %ret +} + +; CHECK-LABEL: Begin function test_i16 +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I16]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseAnd %[[#I16]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define i16 @test_i16(i16 %value) { + %ret = call i16 @llvm.spv.subgroup.multi.prefix.bit.and.i16( + i16 %value, i32 1, i32 2, i32 4, i32 8) + ret i16 %ret +} + +; CHECK-LABEL: Begin function test_i64 +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I64]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseAnd %[[#I64]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define i64 @test_i64(i64 %value) { + %ret = call i64 @llvm.spv.subgroup.multi.prefix.bit.and.i64( + i64 %value, i32 1, i32 2, i32 4, i32 8) + ret i64 %ret +} + +; CHECK-LABEL: Begin function test_vector +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#V2I32]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseAnd %[[#V2I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define <2 x i32> @test_vector(<2 x i32> %value) { + %ret = call <2 x i32> @llvm.spv.subgroup.multi.prefix.bit.and.v2i32( + <2 x i32> %value, i32 1, i32 2, i32 4, i32 8) + ret <2 x i32> %ret +} + +; CHECK-LABEL: Begin function test_long_vector +; CHECK: %[[#MASK0:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: OpGroupNonUniformBitwiseAnd %[[#V4I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %{{[0-9]+}} %[[#MASK0]] +; CHECK: %[[#MASK1:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: OpGroupNonUniformBitwiseAnd %[[#I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %{{[0-9]+}} %[[#MASK1]] +define void @test_long_vector() { + %value = load <5 x i32>, ptr addrspace(10) @wide_i32 + %ret = call <5 x i32> @llvm.spv.subgroup.multi.prefix.bit.and.v5i32( + <5 x i32> %value, i32 1, i32 2, i32 4, i32 8) + store <5 x i32> %ret, ptr addrspace(10) @wide_i32 + ret void +} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
