llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-directx Author: Joshua Batista (bob80905) <details> <summary>Changes</summary> This is the third PR in the WaveMultiPrefix stack. It builds on #<!-- -->238097. This PR adds Clang and DXIL support for `WaveMultiPrefixCountBits(bool value, uint4 mask)` in Shader Model 6.5 or newer. It returns a `uint` and lowers to DXIL opcode 167, `WaveMultiPrefixBitCount`. This PR does not add SPIR-V support. SPIR-V calls report that the builtin is not supported on the target. Fixes #<!-- -->99211. --- Full diff: https://github.com/llvm/llvm-project/pull/238132.diff 12 Files Affected: - (modified) clang/include/clang/Basic/Builtins.td (+6) - (modified) clang/include/clang/Basic/HLSLIntrinsics.td (+9) - (modified) clang/lib/CodeGen/CGHLSLBuiltins.cpp (+4) - (modified) clang/lib/Sema/SemaHLSL.cpp (+32-10) - (added) clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl (+45) - (added) clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl (+63) - (modified) llvm/include/llvm/IR/IntrinsicsDirectX.td (+6) - (modified) llvm/lib/Target/DirectX/DXIL.td (+9) - (modified) llvm/lib/Target/DirectX/DXILShaderFlags.cpp (+1) - (modified) llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll (+7) - (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll (+20) - (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll (+9) ``````````diff diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index ab4b7f4b1187a..e2679df195475 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -5801,6 +5801,12 @@ def HLSLWaveMultiPrefixBitXor : LangBuiltin<"HLSL_LANG"> { let Prototype = "void(...)"; } +def HLSLWaveMultiPrefixCountBits : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_wave_multi_prefix_count_bits"]; + let Attributes = [NoThrow, Const, CustomTypeChecking]; + let Prototype = "unsigned int(bool, _ExtVector<4, unsigned int>)"; +} + def HLSLWavePrefixProduct : LangBuiltin<"HLSL_LANG"> { let Spellings = ["__builtin_hlsl_wave_prefix_product"]; let Attributes = [NoThrow, Const]; diff --git a/clang/include/clang/Basic/HLSLIntrinsics.td b/clang/include/clang/Basic/HLSLIntrinsics.td index 300d46a7fd324..89728b4ba885c 100644 --- a/clang/include/clang/Basic/HLSLIntrinsics.td +++ b/clang/include/clang/Basic/HLSLIntrinsics.td @@ -1903,6 +1903,15 @@ def hlsl_wave_multi_prefix_bit_xor : let Availability = SM6_5; } +def hlsl_wave_multi_prefix_count_bits : + HLSLBuiltin<"WaveMultiPrefixCountBits", + "__builtin_hlsl_wave_multi_prefix_count_bits"> { + let Args = [BoolTy, VectorType<UIntTy, 4>]; + let ReturnType = UIntTy; + let IsConvergent = 1; + let Availability = SM6_5; +} + // Returns the count of bits set to 1 on prior lanes. def hlsl_wave_prefix_count_bits : HLSLBuiltin<"WavePrefixCountBits", diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp index 3afc8a0668f27..229a1cc4f830a 100644 --- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp +++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp @@ -1721,6 +1721,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, return EmitIntrinsicCall(IID, {OpExpr->getType()}, ArrayRef{OpExpr}, "hlsl.wave.prefix.sum"); } + case Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits: case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: { Value *ValueOp = EmitScalarExpr(E->getArg(0)); @@ -1728,6 +1729,9 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, SmallVector<Value *, 5> Args{ValueOp}; for (unsigned I = 0; I != 4; ++I) Args.push_back(Builder.CreateExtractElement(Mask, I)); + if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits) + return EmitIntrinsicCall(Intrinsic::dx_wave_multi_prefix_bit_count, Args, + "hlsl.wave.multi.prefix.bit.count"); if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and) return EmitIntrinsicCall( CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(), diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp index d0da34ef79f89..e0a92ff1d472c 100644 --- a/clang/lib/Sema/SemaHLSL.cpp +++ b/clang/lib/Sema/SemaHLSL.cpp @@ -4012,6 +4012,18 @@ static bool CheckWavePrefix(Sema *S, CallExpr *TheCall) { return false; } +static bool CheckWaveMultiPrefixMask(Sema *S, CallExpr *TheCall) { + QualType MaskTy = TheCall->getArg(1)->getType(); + QualType UInt4Ty = S->Context.getExtVectorType(S->Context.UnsignedIntTy, 4); + if (S->Context.hasSameUnqualifiedType(MaskTy, UInt4Ty)) + return false; + + S->Diag(TheCall->getArg(1)->getBeginLoc(), + diag::err_typecheck_convert_incompatible) + << MaskTy << UInt4Ty << 1 << 0 << 0; + return true; +} + static bool CheckBoolSelect(Sema *S, CallExpr *TheCall) { assert(TheCall->getNumArgs() == 3); Expr *Arg1 = TheCall->getArg(1); @@ -5302,8 +5314,11 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { return true; break; } - case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: { - if (SemaRef.checkArgCount(TheCall, 1)) + case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: + case Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits: { + bool IsMultiPrefix = + BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits; + if (SemaRef.checkArgCount(TheCall, IsMultiPrefix ? 2 : 1)) return true; QualType ArgType = TheCall->getArg(0)->getType(); @@ -5322,6 +5337,20 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { return true; } + if (IsMultiPrefix) { + if (CheckWaveMultiPrefixMask(&SemaRef, TheCall)) + return true; + + // Alias validation uses calls with an invalid source location. + if (TheCall->getBeginLoc().isValid() && + !getASTContext().getTargetInfo().getTriple().isDXIL()) { + SemaRef.Diag(TheCall->getBeginLoc(), + diag::err_builtin_target_unsupported) + << TheCall->getSourceRange(); + return true; + } + } + break; } case Builtin::BI__builtin_hlsl_wave_read_lane_at: { @@ -5395,15 +5424,8 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { return true; } - QualType MaskTy = TheCall->getArg(1)->getType(); - QualType UInt4Ty = - getASTContext().getExtVectorType(getASTContext().UnsignedIntTy, 4); - if (!getASTContext().hasSameUnqualifiedType(MaskTy, UInt4Ty)) { - SemaRef.Diag(TheCall->getArg(1)->getBeginLoc(), - diag::err_typecheck_convert_incompatible) - << MaskTy << UInt4Ty << 1 << 0 << 0; + if (CheckWaveMultiPrefixMask(&SemaRef, TheCall)) return true; - } TheCall->setType(ValueTy); break; diff --git a/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl new file mode 100644 index 0000000000000..4b7c8445d5c7e --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl @@ -0,0 +1,45 @@ +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \ +// RUN: -disable-llvm-passes -o - | FileCheck %s +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \ +// RUN: -disable-llvm-passes -o - | \ +// RUN: opt -S -strip-convergence-intrinsics -dxil-op-lower | \ +// RUN: FileCheck %s --check-prefix=DXIL + +// CHECK-LABEL: test_bool +// DXIL-LABEL: test_bool +uint test_bool(bool value, uint4 mask) { + // CHECK: %[[TOKEN:.*]] = call token @llvm.experimental.convergence.entry() + // CHECK: %[[VALUE:.*]] = icmp ne i32 %{{.*}}, 0 + // CHECK: %[[MASK:.*]] = load <4 x i32> + // CHECK: %[[M0:.*]] = extractelement <4 x i32> %[[MASK]], i64 0 + // CHECK: %[[M1:.*]] = extractelement <4 x i32> %[[MASK]], i64 1 + // CHECK: %[[M2:.*]] = extractelement <4 x i32> %[[MASK]], i64 2 + // CHECK: %[[M3:.*]] = extractelement <4 x i32> %[[MASK]], i64 3 + // CHECK: %[[RET:.*]] = call i32 @llvm.dx.wave.multi.prefix.bit.count(i1 %[[VALUE]], i32 %[[M0]], i32 %[[M1]], i32 %[[M2]], i32 %[[M3]]) [ "convergencectrl"(token %[[TOKEN]]) ] + // CHECK: ret i32 %[[RET]] + // DXIL: call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) + return WaveMultiPrefixCountBits(value, mask); +} + +// CHECK: declare i32 @llvm.dx.wave.multi.prefix.bit.count(i1, i32, i32, i32, i32) #[[ATTR:[0-9]+]] + +// CHECK-LABEL: test_true +// DXIL-LABEL: test_true +uint test_true() { + // CHECK: call i32 @llvm.dx.wave.multi.prefix.bit.count(i1 true, i32 1, i32 2, i32 4, i32 8) + // DXIL: call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 true, i32 1, i32 2, i32 4, i32 8) + return WaveMultiPrefixCountBits(true, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_false +// DXIL-LABEL: test_false +uint test_false() { + // CHECK: call i32 @llvm.dx.wave.multi.prefix.bit.count(i1 false, i32 16, i32 32, i32 64, i32 128) + // DXIL: call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 false, i32 16, i32 32, i32 64, i32 128) + return __builtin_hlsl_wave_multi_prefix_count_bits( + false, uint4(16, 32, 64, 128)); +} + +// CHECK: attributes #[[ATTR]] = { convergent {{.*}}memory(none) } diff --git a/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl new file mode 100644 index 0000000000000..4b0944d349395 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl @@ -0,0 +1,63 @@ +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.5-library -verify %s +// RUN: %clang_cc1 -finclude-default-header -DTEST_AVAILABILITY \ +// RUN: -triple dxil-pc-shadermodel6.4-library -verify=availability %s +// RUN: %clang_cc1 -finclude-default-header -DTEST_UNSUPPORTED \ +// RUN: -triple spirv-pc-vulkan-library -verify=spirv %s + +#ifdef TEST_AVAILABILITY +[shader("compute")] +[numthreads(1, 1, 1)] +void main() { + (void)WaveMultiPrefixCountBits(true, uint4(1, 2, 4, 8)); + // availability-error@-1 {{only available on Shader Model 6.5 or newer}} + // availability-note@*:* {{introduced in Shader Model 6.5}} +} +#elif defined(TEST_UNSUPPORTED) +uint test_public(bool value, uint4 mask) { + return WaveMultiPrefixCountBits(value, mask); + // spirv-error@-1 {{builtin is not supported on this target}} +} + +uint test_builtin(bool value, uint4 mask) { + return __builtin_hlsl_wave_multi_prefix_count_bits(value, mask); + // spirv-error@-1 {{builtin is not supported on this target}} +} +#else +void test_arg_count(bool value, uint4 mask) { + __builtin_hlsl_wave_multi_prefix_count_bits(value); + // expected-error@-1 {{too few arguments to function call, expected 2}} + __builtin_hlsl_wave_multi_prefix_count_bits(value, mask, mask); + // expected-error@-1 {{too many arguments to function call, expected 2}} +} + +struct S { uint value; }; + +void test_value_types(float f, int i, bool2 bv, S s, uint4 mask) { + __builtin_hlsl_wave_multi_prefix_count_bits(f, mask); + // expected-error@-1 {{invalid operand of type 'float'}} + __builtin_hlsl_wave_multi_prefix_count_bits(i, mask); + // expected-error@-1 {{invalid operand of type 'int'}} + __builtin_hlsl_wave_multi_prefix_count_bits(bv, mask); + // expected-error@-1 {{invalid operand of type 'bool2'}} + __builtin_hlsl_wave_multi_prefix_count_bits(s, mask); + // expected-error@-1 {{invalid operand of type 'S'}} +} + +void test_mask_types(bool value, uint scalar, uint3 short_mask, + int4 signed_mask, float4 float_mask, + uint64_t4 wide_mask, S struct_mask) { + __builtin_hlsl_wave_multi_prefix_count_bits(value, scalar); + // expected-error@-1 {{passing 'uint' (aka 'unsigned int')}} + __builtin_hlsl_wave_multi_prefix_count_bits(value, short_mask); + // expected-error@-1 {{passing 'uint3' (aka 'vector<uint, 3>')}} + __builtin_hlsl_wave_multi_prefix_count_bits(value, signed_mask); + // expected-error@-1 {{passing 'int4' (aka 'vector<int, 4>')}} + __builtin_hlsl_wave_multi_prefix_count_bits(value, float_mask); + // expected-error@-1 {{passing 'float4' (aka 'vector<float, 4>')}} + __builtin_hlsl_wave_multi_prefix_count_bits(value, wide_mask); + // expected-error@-1 {{passing 'uint64_t4'}} + __builtin_hlsl_wave_multi_prefix_count_bits(value, struct_mask); + // expected-error@-1 {{passing 'S' to parameter of incompatible type}} +} +#endif diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index c292a0a46b2ae..6343cd2332bd2 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -488,6 +488,12 @@ let TargetPrefix = "dx" in { [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>; + def int_dx_wave_multi_prefix_bit_count + : DefaultAttrsIntrinsic<[llvm_i32_ty], + [llvm_i1_ty, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem]>; + def int_dx_wave_prefix_usum : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index cf3d6404f6210..508680d51603c 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -1589,6 +1589,15 @@ def WaveMultiPrefixOp : DXILOp<166, waveMultiPrefixOp> { let attributes = [Attributes<DXIL1_5, []>]; } +def WaveMultiPrefixBitCount : DXILOp<167, waveMultiPrefixBitCount> { + let Doc = "returns the prefix count for lanes selected by a mask"; + let intrinsics = [IntrinSelect<int_dx_wave_multi_prefix_bit_count>]; + let arguments = [Int1Ty, Int32Ty, Int32Ty, Int32Ty, Int32Ty]; + let result = Int32Ty; + let stages = [Stages<DXIL1_5, [all_stages]>]; + let attributes = [Attributes<DXIL1_5, []>]; +} + def AnnotateHandle : DXILOp<216, annotateHandle> { let Doc = "annotate handle with resource properties"; let arguments = [HandleTy, ResPropsTy]; diff --git a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp index 1f37844dc5f19..7cefe50ea67cc 100644 --- a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp +++ b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp @@ -113,6 +113,7 @@ static bool checkWaveOps(Intrinsic::ID IID) { case Intrinsic::dx_wave_prefix_uproduct: case Intrinsic::dx_wave_multi_prefix_bit_and: case Intrinsic::dx_wave_multi_prefix_bit_xor: + case Intrinsic::dx_wave_multi_prefix_bit_count: // Quad Op Variants case Intrinsic::dx_quad_read_across_x: case Intrinsic::dx_quad_read_across_y: diff --git a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll index 833b5fee3d946..9c9d9dc24d933 100644 --- a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll +++ b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll @@ -84,6 +84,13 @@ define i32 @wave_multi_prefix_bit_xor(i32 %x) { ret i32 %ret } +define i32 @wave_multi_prefix_count_bits(i1 %x) { + ; CHECK: Function wave_multi_prefix_count_bits : [[WAVE_FLAG]] + %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count( + i1 %x, i32 1, i32 2, i32 4, i32 8) + ret i32 %ret +} + define noundef i1 @wave_all_equal(i1 %x) { entry: ; CHECK: Function wave_all_equal : [[WAVE_FLAG]] diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll new file mode 100644 index 0000000000000..17ef364e41111 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll @@ -0,0 +1,20 @@ +; RUN: opt -S -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.5-library < %s | FileCheck %s + +define i32 @test_dynamic(i1 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) { + ; CHECK-LABEL: define i32 @test_dynamic( + ; CHECK: %[[RET:.*]] = call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ; CHECK: ret i32 %[[RET]] + %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count( + i1 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret i32 %ret +} + +define i32 @test_constant() { + ; CHECK-LABEL: define i32 @test_constant( + ; CHECK: %[[RET:.*]] = call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 true, i32 1, i32 2, i32 4, i32 8) + ; CHECK: ret i32 %[[RET]] + %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count( + i1 true, i32 1, i32 2, i32 4, i32 8) + ret i32 %ret +} diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll new file mode 100644 index 0000000000000..74b9c30732adc --- /dev/null +++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll @@ -0,0 +1,9 @@ +; RUN: not opt -S -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.4-library < %s 2>&1 | FileCheck %s + +; CHECK: Cannot create WaveMultiPrefixBitCount operation: No valid stage for DXIL version 1.4 +define i32 @test_version(i1 %value) { + %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count( + i1 %value, i32 1, i32 2, i32 4, i32 8) + ret i32 %ret +} `````````` </details> https://github.com/llvm/llvm-project/pull/238132 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
