https://github.com/llvmbot updated https://github.com/llvm/llvm-project/pull/210264
>From a7fc1e90f2f318b087760f1e212dd7941bf194aa Mon Sep 17 00:00:00 2001 From: Shanzhi Chen <[email protected]> Date: Fri, 17 Jul 2026 09:21:32 +0800 Subject: [PATCH] [AArch64][SVE] Support lowering masked loads/stores of <4 x bf16> and <8 x bf16> (#208744) Add support for lowering masked loads/stores of <4 x bf16> and <8 x bf16> when target features contain "+sve" and "+bf16". Fixes: #201149 (cherry picked from commit b9869c8c920a7dfa983e215bc30729b13d8b353b) --- .../Target/AArch64/AArch64ISelLowering.cpp | 20 ++- llvm/lib/Target/AArch64/AArch64ISelLowering.h | 5 +- ...sve-fixed-length-masked-64-128bit-loads.ll | 148 ++++++++++++++++++ ...ve-fixed-length-masked-64-128bit-stores.ll | 143 +++++++++++++++++ 4 files changed, 308 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index d6f2633297e51..b206869732d5e 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1792,9 +1792,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, // NEON doesn't support masked loads/stores, but SME and SVE do. for (auto VT : - {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, MVT::v1f64, - MVT::v2f64, MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, - MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) { + {MVT::v4f16, MVT::v8f16, MVT::v4bf16, MVT::v8bf16, MVT::v2f32, + MVT::v4f32, MVT::v1f64, MVT::v2f64, MVT::v8i8, MVT::v16i8, MVT::v4i16, + MVT::v8i16, MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) { setOperationAction(ISD::MLOAD, VT, Custom); setOperationAction(ISD::MSTORE, VT, Custom); } @@ -2522,7 +2522,7 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) { // Mark floating-point truncating stores/extending loads as having custom // lowering - if (VT.isFloatingPoint()) { + if (VT.getScalarType() == MVT::f32 || VT.getScalarType() == MVT::f64) { MVT InnerVT = VT.changeVectorElementType(MVT::f16); while (InnerVT != VT) { setTruncStoreAction(VT, InnerVT, Custom); @@ -7537,7 +7537,8 @@ SDValue AArch64TargetLowering::LowerMLOAD(SDValue Op, SelectionDAG &DAG) const { assert(LoadNode && "Expected custom lowering of a masked load node"); EVT VT = Op->getValueType(0); - if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true)) + if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true, + /*AllowBF16=*/true)) return LowerFixedLengthVectorMLoadToSVE(Op, DAG); SDValue PassThru = LoadNode->getPassThru(); @@ -8851,8 +8852,9 @@ bool AArch64TargetLowering::mergeStoresAfterLegalization(EVT VT) const { return !Subtarget->useSVEForFixedLengthVectors(); } -bool AArch64TargetLowering::useSVEForFixedLengthVectorVT( - EVT VT, bool OverrideNEON) const { +bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(EVT VT, + bool OverrideNEON, + bool AllowBF16) const { if (!VT.isFixedLengthVector() || !VT.isSimple()) return false; @@ -8863,6 +8865,10 @@ bool AArch64TargetLowering::useSVEForFixedLengthVectorVT( case MVT::i1: default: return false; + case MVT::bf16: + if (!AllowBF16) + return false; + break; case MVT::i8: case MVT::i16: case MVT::i32: diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index 6e395e004f519..5cc4e9fa1b063 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -588,7 +588,10 @@ class AArch64TargetLowering : public TargetLowering { // Normally SVE is only used for byte size vectors that do not fit within a // NEON vector. This changes when OverrideNEON is true, allowing SVE to be // used for 64bit and 128bit vectors as well. - bool useSVEForFixedLengthVectorVT(EVT VT, bool OverrideNEON = false) const; + // FIXME: AllowBF16 is used to incrementally enable SVE code generation for + // all the fixed-length vectors of bf16 and will be removed in the future. + bool useSVEForFixedLengthVectorVT(EVT VT, bool OverrideNEON = false, + bool AllowBF16 = false) const; // Follow NEON ABI rules even when using SVE for fixed length vectors. MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll index 91af81c617c41..14431f826a75b 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll @@ -35,6 +35,83 @@ define <8 x half> @masked_load_v8f16(ptr %src, <8 x i1> %mask) { ret <8 x half> %load } +define <8 x bfloat> @masked_load_v8bf16_without_bf16_attr(ptr %src, <8 x i1> %mask) { +; CHECK-LABEL: masked_load_v8bf16_without_bf16_attr: +; CHECK: // %bb.0: +; CHECK-NEXT: shl v0.8b, v0.8b, #7 +; CHECK-NEXT: adrp x8, .LCPI2_0 +; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI2_0] +; CHECK-NEXT: cmlt v0.8b, v0.8b, #0 +; CHECK-NEXT: and v0.8b, v0.8b, v1.8b +; CHECK-NEXT: addv b1, v0.8b +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: tbnz w8, #0, .LBB2_9 +; CHECK-NEXT: // %bb.1: // %else +; CHECK-NEXT: tbnz w8, #1, .LBB2_10 +; CHECK-NEXT: .LBB2_2: // %else2 +; CHECK-NEXT: tbnz w8, #2, .LBB2_11 +; CHECK-NEXT: .LBB2_3: // %else5 +; CHECK-NEXT: tbnz w8, #3, .LBB2_12 +; CHECK-NEXT: .LBB2_4: // %else8 +; CHECK-NEXT: tbnz w8, #4, .LBB2_13 +; CHECK-NEXT: .LBB2_5: // %else11 +; CHECK-NEXT: tbnz w8, #5, .LBB2_14 +; CHECK-NEXT: .LBB2_6: // %else14 +; CHECK-NEXT: tbnz w8, #6, .LBB2_15 +; CHECK-NEXT: .LBB2_7: // %else17 +; CHECK-NEXT: tbnz w8, #7, .LBB2_16 +; CHECK-NEXT: .LBB2_8: // %else20 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB2_9: // %cond.load +; CHECK-NEXT: ldr h0, [x0] +; CHECK-NEXT: tbz w8, #1, .LBB2_2 +; CHECK-NEXT: .LBB2_10: // %cond.load1 +; CHECK-NEXT: add x9, x0, #2 +; CHECK-NEXT: ld1 { v0.h }[1], [x9] +; CHECK-NEXT: tbz w8, #2, .LBB2_3 +; CHECK-NEXT: .LBB2_11: // %cond.load4 +; CHECK-NEXT: add x9, x0, #4 +; CHECK-NEXT: ld1 { v0.h }[2], [x9] +; CHECK-NEXT: tbz w8, #3, .LBB2_4 +; CHECK-NEXT: .LBB2_12: // %cond.load7 +; CHECK-NEXT: add x9, x0, #6 +; CHECK-NEXT: ld1 { v0.h }[3], [x9] +; CHECK-NEXT: tbz w8, #4, .LBB2_5 +; CHECK-NEXT: .LBB2_13: // %cond.load10 +; CHECK-NEXT: add x9, x0, #8 +; CHECK-NEXT: ld1 { v0.h }[4], [x9] +; CHECK-NEXT: tbz w8, #5, .LBB2_6 +; CHECK-NEXT: .LBB2_14: // %cond.load13 +; CHECK-NEXT: add x9, x0, #10 +; CHECK-NEXT: ld1 { v0.h }[5], [x9] +; CHECK-NEXT: tbz w8, #6, .LBB2_7 +; CHECK-NEXT: .LBB2_15: // %cond.load16 +; CHECK-NEXT: add x9, x0, #12 +; CHECK-NEXT: ld1 { v0.h }[6], [x9] +; CHECK-NEXT: tbz w8, #7, .LBB2_8 +; CHECK-NEXT: .LBB2_16: // %cond.load19 +; CHECK-NEXT: add x8, x0, #14 +; CHECK-NEXT: ld1 { v0.h }[7], [x8] +; CHECK-NEXT: ret + %load = call <8 x bfloat> @llvm.masked.load.v8bf16(ptr %src, i32 8, <8 x i1> %mask, <8 x bfloat> zeroinitializer) + ret <8 x bfloat> %load +} + +define <8 x bfloat> @masked_load_v8bf16_with_bf16_attr(ptr %src, <8 x i1> %mask) #0 { +; CHECK-LABEL: masked_load_v8bf16_with_bf16_attr: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: shl v0.8h, v0.8h, #15 +; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %load = call <8 x bfloat> @llvm.masked.load.v8bf16(ptr %src, i32 8, <8 x i1> %mask, <8 x bfloat> zeroinitializer) + ret <8 x bfloat> %load +} + define <4 x float> @masked_load_v4f32(ptr %src, <4 x i1> %mask) { ; CHECK-LABEL: masked_load_v4f32: ; CHECK: // %bb.0: @@ -134,3 +211,74 @@ define <4 x half> @masked_load_v4f16(ptr %ap, ptr %bp) { %load = call <4 x half> @llvm.masked.load.v4f16(ptr %ap, i32 2, <4 x i1> %mask, <4 x half> zeroinitializer) ret <4 x half> %load } + +define <4 x bfloat> @masked_load_v4bf16_without_bf16_attr(ptr %ap, ptr %bp) { +; CHECK-LABEL: masked_load_v4bf16_without_bf16_attr: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr d0, [x0] +; CHECK-NEXT: ldr d1, [x1] +; CHECK-NEXT: adrp x8, .LCPI10_0 +; CHECK-NEXT: shll v1.4s, v1.4h, #16 +; CHECK-NEXT: shll v0.4s, v0.4h, #16 +; CHECK-NEXT: fcmeq v0.4s, v0.4s, v1.4s +; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI10_0] +; CHECK-NEXT: xtn v0.4h, v0.4s +; CHECK-NEXT: and v0.8b, v0.8b, v1.8b +; CHECK-NEXT: addv h1, v0.4h +; CHECK-NEXT: movi d0, #0000000000000000 +; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: tbnz w8, #0, .LBB10_5 +; CHECK-NEXT: // %bb.1: // %else +; CHECK-NEXT: tbnz w8, #1, .LBB10_6 +; CHECK-NEXT: .LBB10_2: // %else2 +; CHECK-NEXT: tbnz w8, #2, .LBB10_7 +; CHECK-NEXT: .LBB10_3: // %else5 +; CHECK-NEXT: tbnz w8, #3, .LBB10_8 +; CHECK-NEXT: .LBB10_4: // %else8 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB10_5: // %cond.load +; CHECK-NEXT: ldr h0, [x0] +; CHECK-NEXT: tbz w8, #1, .LBB10_2 +; CHECK-NEXT: .LBB10_6: // %cond.load1 +; CHECK-NEXT: add x9, x0, #2 +; CHECK-NEXT: ld1 { v0.h }[1], [x9] +; CHECK-NEXT: tbz w8, #2, .LBB10_3 +; CHECK-NEXT: .LBB10_7: // %cond.load4 +; CHECK-NEXT: add x9, x0, #4 +; CHECK-NEXT: ld1 { v0.h }[2], [x9] +; CHECK-NEXT: tbz w8, #3, .LBB10_4 +; CHECK-NEXT: .LBB10_8: // %cond.load7 +; CHECK-NEXT: add x8, x0, #6 +; CHECK-NEXT: ld1 { v0.h }[3], [x8] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %a = load <4 x bfloat>, ptr %ap + %b = load <4 x bfloat>, ptr %bp + %mask = fcmp oeq <4 x bfloat> %a, %b + %load = call <4 x bfloat> @llvm.masked.load.v4bf16(ptr %ap, i32 2, <4 x i1> %mask, <4 x bfloat> zeroinitializer) + ret <4 x bfloat> %load +} + +define <4 x bfloat> @masked_load_v4bf16_with_bf16_attr(ptr %ap, ptr %bp) #0 { +; CHECK-LABEL: masked_load_v4bf16_with_bf16_attr: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr d0, [x0] +; CHECK-NEXT: ldr d1, [x1] +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: shll v1.4s, v1.4h, #16 +; CHECK-NEXT: shll v0.4s, v0.4h, #16 +; CHECK-NEXT: fcmeq v0.4s, v0.4s, v1.4s +; CHECK-NEXT: xtn v0.4h, v0.4s +; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %a = load <4 x bfloat>, ptr %ap + %b = load <4 x bfloat>, ptr %bp + %mask = fcmp oeq <4 x bfloat> %a, %b + %load = call <4 x bfloat> @llvm.masked.load.v4bf16(ptr %ap, i32 2, <4 x i1> %mask, <4 x bfloat> zeroinitializer) + ret <4 x bfloat> %load +} + +attributes #0 = { "target-features"="+bf16" } diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll index 439c61339a631..f1b0a0fafb9cd 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll @@ -35,6 +35,83 @@ define void @masked_store_v8f16(ptr %dst, <8 x i1> %mask) { ret void } +define void @masked_store_v8bf16_without_bf16_attr(ptr %dst, <8 x i1> %mask) { +; CHECK-LABEL: masked_store_v8bf16_without_bf16_attr: +; CHECK: // %bb.0: +; CHECK-NEXT: shl v0.8b, v0.8b, #7 +; CHECK-NEXT: adrp x8, .LCPI2_0 +; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI2_0] +; CHECK-NEXT: cmlt v0.8b, v0.8b, #0 +; CHECK-NEXT: and v0.8b, v0.8b, v1.8b +; CHECK-NEXT: addv b0, v0.8b +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: tbnz w8, #0, .LBB2_9 +; CHECK-NEXT: // %bb.1: // %else +; CHECK-NEXT: tbnz w8, #1, .LBB2_10 +; CHECK-NEXT: .LBB2_2: // %else2 +; CHECK-NEXT: tbnz w8, #2, .LBB2_11 +; CHECK-NEXT: .LBB2_3: // %else4 +; CHECK-NEXT: tbnz w8, #3, .LBB2_12 +; CHECK-NEXT: .LBB2_4: // %else6 +; CHECK-NEXT: tbnz w8, #4, .LBB2_13 +; CHECK-NEXT: .LBB2_5: // %else8 +; CHECK-NEXT: tbnz w8, #5, .LBB2_14 +; CHECK-NEXT: .LBB2_6: // %else10 +; CHECK-NEXT: tbnz w8, #6, .LBB2_15 +; CHECK-NEXT: .LBB2_7: // %else12 +; CHECK-NEXT: tbnz w8, #7, .LBB2_16 +; CHECK-NEXT: .LBB2_8: // %else14 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB2_9: // %cond.store +; CHECK-NEXT: movi d0, #0000000000000000 +; CHECK-NEXT: str h0, [x0] +; CHECK-NEXT: tbz w8, #1, .LBB2_2 +; CHECK-NEXT: .LBB2_10: // %cond.store1 +; CHECK-NEXT: movi d0, #0000000000000000 +; CHECK-NEXT: str h0, [x0, #2] +; CHECK-NEXT: tbz w8, #2, .LBB2_3 +; CHECK-NEXT: .LBB2_11: // %cond.store3 +; CHECK-NEXT: movi d0, #0000000000000000 +; CHECK-NEXT: str h0, [x0, #4] +; CHECK-NEXT: tbz w8, #3, .LBB2_4 +; CHECK-NEXT: .LBB2_12: // %cond.store5 +; CHECK-NEXT: movi d0, #0000000000000000 +; CHECK-NEXT: str h0, [x0, #6] +; CHECK-NEXT: tbz w8, #4, .LBB2_5 +; CHECK-NEXT: .LBB2_13: // %cond.store7 +; CHECK-NEXT: movi d0, #0000000000000000 +; CHECK-NEXT: str h0, [x0, #8] +; CHECK-NEXT: tbz w8, #5, .LBB2_6 +; CHECK-NEXT: .LBB2_14: // %cond.store9 +; CHECK-NEXT: movi d0, #0000000000000000 +; CHECK-NEXT: str h0, [x0, #10] +; CHECK-NEXT: tbz w8, #6, .LBB2_7 +; CHECK-NEXT: .LBB2_15: // %cond.store11 +; CHECK-NEXT: movi d0, #0000000000000000 +; CHECK-NEXT: str h0, [x0, #12] +; CHECK-NEXT: tbz w8, #7, .LBB2_8 +; CHECK-NEXT: .LBB2_16: // %cond.store13 +; CHECK-NEXT: movi d0, #0000000000000000 +; CHECK-NEXT: str h0, [x0, #14] +; CHECK-NEXT: ret + call void @llvm.masked.store.v8bf16(<8 x bfloat> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask) + ret void +} + +define void @masked_store_v8bf16_with_bf16_attr(ptr %dst, <8 x i1> %mask) #0 { +; CHECK-LABEL: masked_store_v8bf16_with_bf16_attr: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: shl v0.8h, v0.8h, #15 +; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0 +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: st1h { z0.h }, p1, [x0] +; CHECK-NEXT: ret + call void @llvm.masked.store.v8bf16(<8 x bfloat> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask) + ret void +} + define void @masked_store_v4f32(ptr %dst, <4 x i1> %mask) { ; CHECK-LABEL: masked_store_v4f32: ; CHECK: // %bb.0: @@ -116,3 +193,69 @@ define void @masked_store_v4f16(ptr %ap, ptr %bp) { ret void } +define void @masked_store_v4bf16_without_bf16_attr(ptr %ap, ptr %bp) { +; CHECK-LABEL: masked_store_v4bf16_without_bf16_attr: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr d0, [x0] +; CHECK-NEXT: ldr d1, [x1] +; CHECK-NEXT: adrp x8, .LCPI9_0 +; CHECK-NEXT: shll v1.4s, v1.4h, #16 +; CHECK-NEXT: shll v2.4s, v0.4h, #16 +; CHECK-NEXT: fcmeq v1.4s, v2.4s, v1.4s +; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI9_0] +; CHECK-NEXT: xtn v1.4h, v1.4s +; CHECK-NEXT: and v1.8b, v1.8b, v2.8b +; CHECK-NEXT: addv h1, v1.4h +; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: tbnz w8, #0, .LBB9_5 +; CHECK-NEXT: // %bb.1: // %else +; CHECK-NEXT: tbnz w8, #1, .LBB9_6 +; CHECK-NEXT: .LBB9_2: // %else2 +; CHECK-NEXT: tbnz w8, #2, .LBB9_7 +; CHECK-NEXT: .LBB9_3: // %else4 +; CHECK-NEXT: tbnz w8, #3, .LBB9_8 +; CHECK-NEXT: .LBB9_4: // %else6 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB9_5: // %cond.store +; CHECK-NEXT: st1 { v0.h }[0], [x1] +; CHECK-NEXT: tbz w8, #1, .LBB9_2 +; CHECK-NEXT: .LBB9_6: // %cond.store1 +; CHECK-NEXT: add x9, x1, #2 +; CHECK-NEXT: st1 { v0.h }[1], [x9] +; CHECK-NEXT: tbz w8, #2, .LBB9_3 +; CHECK-NEXT: .LBB9_7: // %cond.store3 +; CHECK-NEXT: add x9, x1, #4 +; CHECK-NEXT: st1 { v0.h }[2], [x9] +; CHECK-NEXT: tbz w8, #3, .LBB9_4 +; CHECK-NEXT: .LBB9_8: // %cond.store5 +; CHECK-NEXT: add x8, x1, #6 +; CHECK-NEXT: st1 { v0.h }[3], [x8] +; CHECK-NEXT: ret + %a = load <4 x bfloat>, ptr %ap + %b = load <4 x bfloat>, ptr %bp + %mask = fcmp oeq <4 x bfloat> %a, %b + call void @llvm.masked.store.v4bf16(<4 x bfloat> %a, ptr %bp, i32 2, <4 x i1> %mask) + ret void +} + +define void @masked_store_v4bf16_with_bf16_attr(ptr %ap, ptr %bp) #0 { +; CHECK-LABEL: masked_store_v4bf16_with_bf16_attr: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr d0, [x0] +; CHECK-NEXT: ldr d1, [x1] +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: shll v1.4s, v1.4h, #16 +; CHECK-NEXT: shll v2.4s, v0.4h, #16 +; CHECK-NEXT: fcmeq v1.4s, v2.4s, v1.4s +; CHECK-NEXT: xtn v1.4h, v1.4s +; CHECK-NEXT: cmpne p1.h, p0/z, z1.h, #0 +; CHECK-NEXT: st1h { z0.h }, p1, [x1] +; CHECK-NEXT: ret + %a = load <4 x bfloat>, ptr %ap + %b = load <4 x bfloat>, ptr %bp + %mask = fcmp oeq <4 x bfloat> %a, %b + call void @llvm.masked.store.v4bf16(<4 x bfloat> %a, ptr %bp, i32 2, <4 x i1> %mask) + ret void +} + +attributes #0 = { "target-features"="+bf16" } _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
