https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232526
>From 7f9a45470a0b53261ef7732f5c290b8bfac9901f Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Wed, 14 Oct 2026 23:22:09 +0100 Subject: [PATCH 1/6] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms Replace NOT followed by AND with BIC for expressions such as: return svand_u32_x(svptrue_b32(), a, svnot_u32_x(svptrue_b32(), b)); Combine ordinary AND with NOT when the NOT predicate is all-active or its inactive values are undefined. Also combine matching-predicate SVE AND_x-NOT pairs before lowering discards the AND predicate. AND_x ignores inactive results, so the NOT's inactive values do not need preserving. This applies to all integer element widths. Update the pre-commit LLVM expectations, including cases where combining would change the result. --- .../Target/AArch64/AArch64ISelLowering.cpp | 37 +++++- .../CodeGen/AArch64/sve-intrinsics-and-not.ll | 123 ++++-------------- 2 files changed, 57 insertions(+), 103 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index c1cd81ba0c259..060ecf1614fcd 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -22134,7 +22134,21 @@ static SDValue performReinterpretCastCombine(SDNode *N) { static SDValue performSVEAndCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { + assert(N->getOpcode() == ISD::AND && "Expected AND instruction"); SelectionDAG &DAG = DCI.DAG; + + // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not + // matter, leaving standalone NOTs available for predicated selection. + for (unsigned I = 0; I != 2; ++I) { + SDValue Not = N->getOperand(I); + if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not) + continue; + if (Not.getOperand(1).isUndef() || + isAllActivePredicate(DAG, Not.getOperand(2))) + return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), + N->getOperand(1 - I), Not.getOperand(3)); + } + SDValue Src = N->getOperand(0); unsigned Opc = Src->getOpcode(); @@ -25442,6 +25456,26 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N, DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64)); } +static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { + SDValue Pg = N->getOperand(1); + + // Fold matching predicates before lowering AND_U discards its predicate. + // NOT's inactive lanes cannot affect AND_U's active lanes. + for (unsigned I = 0; I != 2; ++I) { + unsigned NotOperandIdx = 2 + I; + unsigned OtherOperandIdx = 3 - I; + SDValue Not = N->getOperand(NotOperandIdx); + if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not || + Not.getOperand(2) != Pg) + continue; + + return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), + N->getOperand(OtherOperandIdx), Not.getOperand(3)); + } + return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2), + N->getOperand(3)); +} + static SDValue performIntrinsicCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) { @@ -25661,8 +25695,7 @@ static SDValue performIntrinsicCombine(SDNode *N, case Intrinsic::aarch64_sve_subr: return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); case Intrinsic::aarch64_sve_and_u: - return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2), - N->getOperand(3)); + return performSVEAndNotIntrinsicCombine(N, DAG); case Intrinsic::aarch64_sve_bic_u: return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll index 8eac86cb93479..dd06c3d38c54d 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll @@ -6,26 +6,10 @@ ; Ordinary LLVM AND with SVE NOT. define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) { -; SVE-LABEL: and_not_ptrue_all: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_ptrue_all: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_ptrue_all: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_ptrue_all: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31) %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = and <vscale x 4 x i32> %a, %not @@ -33,49 +17,20 @@ define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4 } define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { -; SVE-LABEL: and_not_commuted: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z1.d, z0.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_commuted: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z1.d, z0.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_commuted: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z1.d, z0.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_commuted: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = and <vscale x 4 x i32> %not, %a ret <vscale x 4 x i32> %and } define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg) { -; SVE-LABEL: and_not_poison_i32: -; SVE: // %bb.0: -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_poison_i32: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_poison_i32: -; SME: // %bb.0: -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_poison_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = and <vscale x 4 x i32> %a, %not ret <vscale x 4 x i32> %and @@ -107,26 +62,10 @@ define <vscale x 4 x i32> @and_u_not_commuted_i32(<vscale x 4 x i1> %pg, <vscale ; irrelevant to AND_U. An all-active NOT can also fold with any AND predicate. define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { -; SVE-LABEL: and_not_all_i32: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_all_i32: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_all_i32: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_all_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and @@ -135,8 +74,7 @@ define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { ; CHECK-LABEL: matching_defined: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.s, p0/m, z1.s -; CHECK-NEXT: and z0.d, z0.d, z2.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) @@ -146,8 +84,7 @@ define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { ; CHECK-LABEL: matching_commuted: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.s, p0/m, z1.s -; CHECK-NEXT: and z0.d, z2.d, z0.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %not, <vscale x 4 x i32> %a) @@ -155,26 +92,10 @@ define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4 } define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { -; SVE-LABEL: all_not_partial_and: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: all_not_partial_and: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: all_not_partial_and: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: all_not_partial_and: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and >From 93831a21b6b8fd93719c16a2ae98a2a233de8e0a Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Wed, 21 Oct 2026 12:55:10 +0100 Subject: [PATCH 2/6] fixup! [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 12 ------------ llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 060ecf1614fcd..3b728e3e64d63 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -22137,18 +22137,6 @@ static SDValue performSVEAndCombine(SDNode *N, assert(N->getOpcode() == ISD::AND && "Expected AND instruction"); SelectionDAG &DAG = DCI.DAG; - // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not - // matter, leaving standalone NOTs available for predicated selection. - for (unsigned I = 0; I != 2; ++I) { - SDValue Not = N->getOperand(I); - if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not) - continue; - if (Not.getOperand(1).isUndef() || - isAllActivePredicate(DAG, Not.getOperand(2))) - return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), - N->getOperand(1 - I), Not.getOperand(3)); - } - SDValue Src = N->getOperand(0); unsigned Opc = Src->getOpcode(); diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 9e56956dab730..9d92c34c7fbb7 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -753,6 +753,20 @@ let Predicates = [HasSVE_or_SME] in { defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>; defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>; + // Select BIC when NOT's inactive values are undefined or all lanes are active. + multiclass SVEAndNotPats<ValueType VT, ValueType PT> { + def : Pat<(and VT:$a, + (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))), + (BIC_ZZZ $a, $b)>; + def : Pat<(and VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))), + (BIC_ZZZ $a, $b)>; + } + defm : SVEAndNotPats<nxv16i8, nxv16i1>; + defm : SVEAndNotPats<nxv8i16, nxv8i1>; + defm : SVEAndNotPats<nxv4i32, nxv4i1>; + defm : SVEAndNotPats<nxv2i64, nxv2i1>; + defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>; defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">; defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>; >From 9722027e70997cd9c4ac1891b2cc1c436d5c8b16 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Wed, 21 Oct 2026 13:43:41 +0100 Subject: [PATCH 3/6] fixup! Add tests for extra widths --- .../CodeGen/AArch64/sve-intrinsics-and-not.ll | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll index dd06c3d38c54d..1a1b53a90a746 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll @@ -26,6 +26,26 @@ define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x ret <vscale x 4 x i32> %and } +define <vscale x 16 x i8> @and_not_poison_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i1> %pg) { +; CHECK-LABEL: and_not_poison_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %not = call <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) + %and = and <vscale x 16 x i8> %a, %not + ret <vscale x 16 x i8> %and +} + +define <vscale x 8 x i16> @and_not_poison_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i1> %pg) { +; CHECK-LABEL: and_not_poison_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %not = call <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) + %and = and <vscale x 8 x i16> %a, %not + ret <vscale x 8 x i16> %and +} + define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg) { ; CHECK-LABEL: and_not_poison_i32: ; CHECK: // %bb.0: @@ -36,6 +56,16 @@ define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 ret <vscale x 4 x i32> %and } +define <vscale x 2 x i64> @and_not_poison_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i1> %pg) { +; CHECK-LABEL: and_not_poison_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %not = call <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) + %and = and <vscale x 2 x i64> %a, %not + ret <vscale x 2 x i64> %and +} + ; SVE AND_U with ordinary LLVM NOT. define <vscale x 4 x i32> @and_u_not_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { >From cad17f3a124e8d2880e6f040b44e6294793533ae Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Wed, 21 Oct 2026 16:27:26 +0100 Subject: [PATCH 4/6] fixup! Replace performSVEAndNotIntrinsicCombine() with tablegen Move AND_U and NOT matching to TableGen, using a shared pattern fragment for both operand orders. Retain register and immediate AND selection for other AND_U operations. --- .../Target/AArch64/AArch64ISelLowering.cpp | 38 ++++++++++------ .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 44 ++++++++++++++++++- 2 files changed, 67 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 3b728e3e64d63..011c074992c99 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -22134,9 +22134,7 @@ static SDValue performReinterpretCastCombine(SDNode *N) { static SDValue performSVEAndCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { - assert(N->getOpcode() == ISD::AND && "Expected AND instruction"); SelectionDAG &DAG = DCI.DAG; - SDValue Src = N->getOperand(0); unsigned Opc = Src->getOpcode(); @@ -25444,22 +25442,34 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N, DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64)); } -static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { +static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { SDValue Pg = N->getOperand(1); - // Fold matching predicates before lowering AND_U discards its predicate. - // NOT's inactive lanes cannot affect AND_U's active lanes. - for (unsigned I = 0; I != 2; ++I) { - unsigned NotOperandIdx = 2 + I; - unsigned OtherOperandIdx = 3 - I; - SDValue Not = N->getOperand(NotOperandIdx); - if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not || - Not.getOperand(2) != Pg) + // Preserve matching predicates so TableGen can select: + // and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b) + for (unsigned I = 2; I != 4; ++I) { + SDValue Not = N->getOperand(I); + if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not) continue; - return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), - N->getOperand(OtherOperandIdx), Not.getOperand(3)); + SDValue NotPg = Not.getOperand(2); + if (NotPg != Pg) + continue; + + // We can discard AND_U's predicate if NOT's inactive lanes are + // undefined or NOT's predicate is all-active: ordinary AND-NOT + // patterns can select BIC. + SDValue Inactive = Not.getOperand(1); + if (Inactive.isUndef() || isAllActivePredicate(DAG, NotPg)) + continue; + + // Keep AND_U unchanged so TableGen can match the predicates. + return SDValue(); } + + // Otherwise expose an ordinary AND so generic DAG combines can simplify + // it, for example: + // and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15) return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); } @@ -25683,7 +25693,7 @@ static SDValue performIntrinsicCombine(SDNode *N, case Intrinsic::aarch64_sve_subr: return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); case Intrinsic::aarch64_sve_and_u: - return performSVEAndNotIntrinsicCombine(N, DAG); + return performSVEAndUIntrinsicCombine(N, DAG); case Intrinsic::aarch64_sve_bic_u: return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 9d92c34c7fbb7..26c88729e1bdf 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -559,6 +559,11 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2), (and node:$op1, (xor node:$op2, (SVEAllActive))), (AArch64bic_node node:$op1, node:$op2)]>; +// AND_U permits either operand order. +def SVEAndU : PatFrags<(ops node:$pg, node:$a, node:$b), + [(int_aarch64_sve_and_u node:$pg, node:$a, node:$b), + (int_aarch64_sve_and_u node:$pg, node:$b, node:$a)]>; + def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2), [(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2), (and node:$pg, (and node:$op1, node:$op2))]>; @@ -753,14 +758,40 @@ let Predicates = [HasSVE_or_SME] in { defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>; defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>; - // Select BIC when NOT's inactive values are undefined or all lanes are active. multiclass SVEAndNotPats<ValueType VT, ValueType PT> { + // Ordinary AND: NOT's undefined inactive lanes may take the value ~b. def : Pat<(and VT:$a, (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))), (BIC_ZZZ $a, $b)>; + // Ordinary AND: an all-active NOT produces ~b in every lane. def : Pat<(and VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))), (BIC_ZZZ $a, $b)>; + + // AND_U: matching predicates ensure NOT produces ~b in every active AND lane. + def : Pat<(VT (SVEAndU PT:$pg, VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // AND_U: NOT's undefined inactive lanes may take the value ~b, + // even when the predicates differ. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // AND_U with ordinary LLVM NOT, which produces ~b in every lane. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))), + (BIC_ZZZ $a, $b)>; + + // Fall back to register AND when no BIC pattern matches. + // AND_U allows any result in inactive lanes. + def : Pat<(VT (int_aarch64_sve_and_u (PT srcvalue), VT:$a, VT:$b)), + (AND_ZZZ $a, $b)>; } defm : SVEAndNotPats<nxv16i8, nxv16i1>; defm : SVEAndNotPats<nxv8i16, nxv8i1>; @@ -825,6 +856,17 @@ let Predicates = [HasSVE_or_SME] in { defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>; defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>; + // Allow retained AND_U intrinsics to select AND-immediate instructions. + class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR, + ValueType IT, ComplexPattern CP> + : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a), + (VT (splat_vector (IT (CP i64:$imm)))))), + (AND_ZI_PSEUDO $a, i64:$imm)>; + def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>; + def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>; + def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>; + def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>; + defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>; defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>; defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>; >From ca00132647c0ad5fa2a6653bdf8f0b300ff546f8 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Thu, 22 Oct 2026 16:36:25 +0100 Subject: [PATCH 5/6] fixup! [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms Remove the AND_U pattern for NOT with undefined inactive lanes and an arbitrary predicate. Keep the original fix focused; ordinary cases still lower to AND and use the existing AND-NOT patterns. --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 6 ------ 1 file changed, 6 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 26c88729e1bdf..af65a5d39b31d 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -773,12 +773,6 @@ let Predicates = [HasSVE_or_SME] in { (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), (BIC_ZZZ $a, $b)>; - // AND_U: NOT's undefined inactive lanes may take the value ~b, - // even when the predicates differ. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, - (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))), - (BIC_ZZZ $a, $b)>; - // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate. def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), >From 96015be30d9e1c07422e142d9e0e3a472e815e31 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Thu, 22 Oct 2026 16:44:55 +0100 Subject: [PATCH 6/6] fixup! [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms Remove the extra AND_U patterns for all-active NOT, ordinary LLVM NOT and register AND fallback. The C++ helper lowers ordinary cases to AND and retains matching-predicate AND-NOT pairs for the remaining BIC pattern. --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index af65a5d39b31d..8ffdb8cb9c05f 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -772,20 +772,6 @@ let Predicates = [HasSVE_or_SME] in { def : Pat<(VT (SVEAndU PT:$pg, VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), (BIC_ZZZ $a, $b)>; - - // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, - (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), - (BIC_ZZZ $a, $b)>; - - // AND_U with ordinary LLVM NOT, which produces ~b in every lane. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))), - (BIC_ZZZ $a, $b)>; - - // Fall back to register AND when no BIC pattern matches. - // AND_U allows any result in inactive lanes. - def : Pat<(VT (int_aarch64_sve_and_u (PT srcvalue), VT:$a, VT:$b)), - (AND_ZZZ $a, $b)>; } defm : SVEAndNotPats<nxv16i8, nxv16i1>; defm : SVEAndNotPats<nxv8i16, nxv8i1>; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
