https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232527
>From 5c4cf2ff515f5a098f4a9fa27c0598f0ebabed23 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Thu, 15 Oct 2026 22:23:45 +0100 Subject: [PATCH] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _m forms Replace NOT followed by merging AND with BIC for expressions such as: return svand_u32_m(pg, a, svnot_u32_x(pg, b)); Combine when NOT is the second operand and its predicate matches the AND predicate or is all-active. Use predicated BIC to preserve the AND's first operand in inactive lanes. Use unpredicated BIC when the AND predicate is all-active. Do not commute merging AND operands, since this would change inactive results. Leave pairs with incompatible predicates unchanged. This applies to all integer element widths. Update the pre-commit LLVM expectations, including cases where combining would change the result. --- .../Target/AArch64/AArch64ISelLowering.cpp | 32 ++++++---- .../CodeGen/AArch64/sve-intrinsics-and-not.ll | 60 ++++--------------- 2 files changed, 34 insertions(+), 58 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index bc3ded382f070..9859924fc0736 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -25395,23 +25395,34 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N, } static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { + bool IsMerging = getIntrinsicID(N) == Intrinsic::aarch64_sve_and; + SDLoc DL(N); + EVT VT = N->getValueType(0); SDValue Pg = N->getOperand(1); - // Fold matching predicates before lowering AND_U discards its predicate. - // NOT's inactive lanes cannot affect AND_U's active lanes. - for (unsigned I = 0; I != 2; ++I) { - unsigned NotOperandIdx = 2 + I; - unsigned OtherOperandIdx = 3 - I; + // Only commute AND_U: a merging AND must preserve its first operand in + // inactive lanes. NOT must cover all of AND's active lanes. + for (unsigned I = 0, E = IsMerging ? 1 : 2; I != E; ++I) { + unsigned NotOperandIdx = 3 - I; + unsigned OtherOperandIdx = 2 + I; SDValue Not = N->getOperand(NotOperandIdx); if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not || - Not.getOperand(2) != Pg) + (Not.getOperand(2) != Pg && + !isAllActivePredicate(DAG, Not.getOperand(2)))) continue; - return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), - N->getOperand(OtherOperandIdx), Not.getOperand(3)); + SDValue A = N->getOperand(OtherOperandIdx); + SDValue B = Not.getOperand(3); + if (!IsMerging || isAllActivePredicate(DAG, Pg)) + return DAG.getNode(AArch64ISD::BIC, DL, VT, A, B); + + return DAG.getNode( + ISD::INTRINSIC_WO_CHAIN, DL, VT, + DAG.getConstant(Intrinsic::aarch64_sve_bic, DL, MVT::i32), Pg, A, B); } - return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2), - N->getOperand(3)); + if (IsMerging) + return SDValue(); + return DAG.getNode(ISD::AND, DL, VT, N->getOperand(2), N->getOperand(3)); } static SDValue performIntrinsicCombine(SDNode *N, @@ -25632,6 +25643,7 @@ static SDValue performIntrinsicCombine(SDNode *N, N->getOperand(3)); case Intrinsic::aarch64_sve_subr: return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); + case Intrinsic::aarch64_sve_and: case Intrinsic::aarch64_sve_and_u: return performSVEAndNotIntrinsicCombine(N, DAG); case Intrinsic::aarch64_sve_bic_u: diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll index 66ecfa31e01f2..c988622be8407 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll @@ -84,8 +84,7 @@ define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x define <vscale x 4 x i32> @merging_and_rhs(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { ; CHECK-LABEL: merging_and_rhs: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.s, p0/m, z1.s -; CHECK-NEXT: and z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: bic z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) @@ -95,8 +94,7 @@ define <vscale x 4 x i32> @merging_and_rhs(<vscale x 4 x i32> %a, <vscale x 4 x define <vscale x 16 x i8> @merging_and_rhs_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %inactive, <vscale x 16 x i1> %pg) { ; CHECK-LABEL: merging_and_rhs_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.b, p0/m, z1.b -; CHECK-NEXT: and z0.b, p0/m, z0.b, z2.b +; CHECK-NEXT: bic z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret %not = call <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8> %inactive, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) %and = call <vscale x 16 x i8> @llvm.aarch64.sve.and.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %not) @@ -106,8 +104,7 @@ define <vscale x 16 x i8> @merging_and_rhs_i8(<vscale x 16 x i8> %a, <vscale x 1 define <vscale x 8 x i16> @merging_and_rhs_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %inactive, <vscale x 8 x i1> %pg) { ; CHECK-LABEL: merging_and_rhs_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.h, p0/m, z1.h -; CHECK-NEXT: and z0.h, p0/m, z0.h, z2.h +; CHECK-NEXT: bic z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %not = call <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16> %inactive, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) %and = call <vscale x 8 x i16> @llvm.aarch64.sve.and.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %not) @@ -117,8 +114,7 @@ define <vscale x 8 x i16> @merging_and_rhs_i16(<vscale x 8 x i16> %a, <vscale x define <vscale x 2 x i64> @merging_and_rhs_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %inactive, <vscale x 2 x i1> %pg) { ; CHECK-LABEL: merging_and_rhs_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.d, p0/m, z1.d -; CHECK-NEXT: and z0.d, p0/m, z0.d, z2.d +; CHECK-NEXT: bic z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %not = call <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64> %inactive, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) %and = call <vscale x 2 x i64> @llvm.aarch64.sve.and.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %not) @@ -126,52 +122,20 @@ define <vscale x 2 x i64> @merging_and_rhs_i64(<vscale x 2 x i64> %a, <vscale x } define <vscale x 4 x i32> @merging_and_all_active_not(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { -; SVE-LABEL: merging_and_all_active_not: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p1.s -; SVE-NEXT: not z1.s, p1/m, z1.s -; SVE-NEXT: and z0.s, p0/m, z0.s, z1.s -; SVE-NEXT: ret -; -; SVE2P2-LABEL: merging_and_all_active_not: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p1.s -; SVE2P2-NEXT: not z1.s, p1/z, z1.s -; SVE2P2-NEXT: and z0.s, p0/m, z0.s, z1.s -; SVE2P2-NEXT: ret -; -; SME-LABEL: merging_and_all_active_not: -; SME: // %bb.0: -; SME-NEXT: ptrue p1.s -; SME-NEXT: not z1.s, p1/m, z1.s -; SME-NEXT: and z0.s, p0/m, z0.s, z1.s -; SME-NEXT: ret +; CHECK-LABEL: merging_and_all_active_not: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and } define <vscale x 4 x i32> @merging_and_all_active(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) { -; SVE-LABEL: merging_and_all_active: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.s, p0/m, z0.s, z1.s -; SVE-NEXT: ret -; -; SVE2P2-LABEL: merging_and_all_active: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.s, p0/m, z0.s, z1.s -; SVE2P2-NEXT: ret -; -; SME-LABEL: merging_and_all_active: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.s, p0/m, z0.s, z1.s -; SME-NEXT: ret +; CHECK-LABEL: merging_and_all_active: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
