https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232526
>From 655f40cdb4224f0ff0b0604c430f654926fbc26e Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Wed, 14 Oct 2026 23:22:09 +0100 Subject: [PATCH 01/10] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms Replace NOT followed by AND with BIC for expressions such as: return svand_u32_x(svptrue_b32(), a, svnot_u32_x(svptrue_b32(), b)); Combine ordinary AND with NOT when the NOT predicate is all-active or its inactive values are undefined. Also combine matching-predicate SVE AND_x-NOT pairs before lowering discards the AND predicate. AND_x ignores inactive results, so the NOT's inactive values do not need preserving. This applies to all integer element widths. Update the pre-commit LLVM expectations, including cases where combining would change the result. --- .../Target/AArch64/AArch64ISelLowering.cpp | 37 +++++- .../CodeGen/AArch64/sve-intrinsics-and-not.ll | 123 ++++-------------- 2 files changed, 57 insertions(+), 103 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index dbdc98611f5340..bd915200219d12 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -22364,7 +22364,21 @@ static SDValue performReinterpretCastCombine(SDNode *N) { static SDValue performSVEAndCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { + assert(N->getOpcode() == ISD::AND && "Expected AND instruction"); SelectionDAG &DAG = DCI.DAG; + + // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not + // matter, leaving standalone NOTs available for predicated selection. + for (unsigned I = 0; I != 2; ++I) { + SDValue Not = N->getOperand(I); + if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not) + continue; + if (Not.getOperand(1).isUndef() || + isAllActivePredicate(DAG, Not.getOperand(2))) + return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), + N->getOperand(1 - I), Not.getOperand(3)); + } + SDValue Src = N->getOperand(0); unsigned Opc = Src->getOpcode(); @@ -25623,6 +25637,26 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N, DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64)); } +static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { + SDValue Pg = N->getOperand(1); + + // Fold matching predicates before lowering AND_U discards its predicate. + // NOT's inactive lanes cannot affect AND_U's active lanes. + for (unsigned I = 0; I != 2; ++I) { + unsigned NotOperandIdx = 2 + I; + unsigned OtherOperandIdx = 3 - I; + SDValue Not = N->getOperand(NotOperandIdx); + if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not || + Not.getOperand(2) != Pg) + continue; + + return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), + N->getOperand(OtherOperandIdx), Not.getOperand(3)); + } + return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2), + N->getOperand(3)); +} + static SDValue performIntrinsicCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) { @@ -25842,8 +25876,7 @@ static SDValue performIntrinsicCombine(SDNode *N, case Intrinsic::aarch64_sve_subr: return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); case Intrinsic::aarch64_sve_and_u: - return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2), - N->getOperand(3)); + return performSVEAndNotIntrinsicCombine(N, DAG); case Intrinsic::aarch64_sve_bic_u: return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll index 736c0ee0171c54..bc59860c715f39 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll @@ -6,26 +6,10 @@ ; Ordinary LLVM AND with SVE NOT. define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) { -; SVE-LABEL: and_not_ptrue_all: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_ptrue_all: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_ptrue_all: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_ptrue_all: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31) %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = and <vscale x 4 x i32> %a, %not @@ -33,26 +17,10 @@ define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4 } define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { -; SVE-LABEL: and_not_commuted: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z1.d, z0.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_commuted: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z1.d, z0.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_commuted: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z1.d, z0.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_commuted: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = and <vscale x 4 x i32> %not, %a ret <vscale x 4 x i32> %and @@ -105,23 +73,10 @@ define <vscale x 8 x i16> @and_not_poison_i16(<vscale x 8 x i16> %a, <vscale x 8 } define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg) { -; SVE-LABEL: and_not_poison_i32: -; SVE: // %bb.0: -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_poison_i32: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_poison_i32: -; SME: // %bb.0: -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_poison_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = and <vscale x 4 x i32> %a, %not ret <vscale x 4 x i32> %and @@ -176,26 +131,10 @@ define <vscale x 4 x i32> @and_u_not_commuted_i32(<vscale x 4 x i1> %pg, <vscale ; irrelevant to AND_U. An all-active NOT can also fold with any AND predicate. define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { -; SVE-LABEL: and_not_all_i32: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_all_i32: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_all_i32: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_all_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and @@ -204,8 +143,7 @@ define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { ; CHECK-LABEL: matching_defined: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.s, p0/m, z1.s -; CHECK-NEXT: and z0.d, z0.d, z2.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) @@ -215,8 +153,7 @@ define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { ; CHECK-LABEL: matching_commuted: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.s, p0/m, z1.s -; CHECK-NEXT: and z0.d, z2.d, z0.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %not, <vscale x 4 x i32> %a) @@ -249,26 +186,10 @@ define <vscale x 4 x i32> @different_predicates_poison(<vscale x 4 x i32> %a, <v } define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { -; SVE-LABEL: all_not_partial_and: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: all_not_partial_and: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: all_not_partial_and: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: all_not_partial_and: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and >From 85888670ae88764747be627c1351b30d8bf2da8a Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Wed, 21 Oct 2026 12:55:10 +0100 Subject: [PATCH 02/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 12 ------------ llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index bd915200219d12..540da867dc9678 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -22367,18 +22367,6 @@ static SDValue performSVEAndCombine(SDNode *N, assert(N->getOpcode() == ISD::AND && "Expected AND instruction"); SelectionDAG &DAG = DCI.DAG; - // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not - // matter, leaving standalone NOTs available for predicated selection. - for (unsigned I = 0; I != 2; ++I) { - SDValue Not = N->getOperand(I); - if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not) - continue; - if (Not.getOperand(1).isUndef() || - isAllActivePredicate(DAG, Not.getOperand(2))) - return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), - N->getOperand(1 - I), Not.getOperand(3)); - } - SDValue Src = N->getOperand(0); unsigned Opc = Src->getOpcode(); diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index a3a4fdef30ef1b..958835f1858608 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -753,6 +753,20 @@ let Predicates = [HasSVE_or_SME] in { defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>; defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>; + // Select BIC when NOT's inactive values are undefined or all lanes are active. + multiclass SVEAndNotPats<ValueType VT, ValueType PT> { + def : Pat<(and VT:$a, + (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))), + (BIC_ZZZ $a, $b)>; + def : Pat<(and VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))), + (BIC_ZZZ $a, $b)>; + } + defm : SVEAndNotPats<nxv16i8, nxv16i1>; + defm : SVEAndNotPats<nxv8i16, nxv8i1>; + defm : SVEAndNotPats<nxv4i32, nxv4i1>; + defm : SVEAndNotPats<nxv2i64, nxv2i1>; + defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>; defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">; defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>; >From f647b2e2af470675fe5802e3bb5d928ea0de3987 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Wed, 21 Oct 2026 13:43:41 +0100 Subject: [PATCH 03/10] fixup! Add tests for extra widths --- .../CodeGen/AArch64/sve-intrinsics-and-not.ll | 63 ++++--------------- 1 file changed, 12 insertions(+), 51 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll index bc59860c715f39..4d8d595f242c24 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll @@ -27,46 +27,20 @@ define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x } define <vscale x 16 x i8> @and_not_poison_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i1> %pg) { -; SVE-LABEL: and_not_poison_i8: -; SVE: // %bb.0: -; SVE-NEXT: not z1.b, p0/m, z1.b -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_poison_i8: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: not z1.b, p0/z, z1.b -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_poison_i8: -; SME: // %bb.0: -; SME-NEXT: not z1.b, p0/m, z1.b -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_poison_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) %and = and <vscale x 16 x i8> %a, %not ret <vscale x 16 x i8> %and } define <vscale x 8 x i16> @and_not_poison_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i1> %pg) { -; SVE-LABEL: and_not_poison_i16: -; SVE: // %bb.0: -; SVE-NEXT: not z1.h, p0/m, z1.h -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_poison_i16: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: not z1.h, p0/z, z1.h -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_poison_i16: -; SME: // %bb.0: -; SME-NEXT: not z1.h, p0/m, z1.h -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_poison_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) %and = and <vscale x 8 x i16> %a, %not ret <vscale x 8 x i16> %and @@ -83,23 +57,10 @@ define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 } define <vscale x 2 x i64> @and_not_poison_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i1> %pg) { -; SVE-LABEL: and_not_poison_i64: -; SVE: // %bb.0: -; SVE-NEXT: not z1.d, p0/m, z1.d -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_poison_i64: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: not z1.d, p0/z, z1.d -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_poison_i64: -; SME: // %bb.0: -; SME-NEXT: not z1.d, p0/m, z1.d -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_poison_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) %and = and <vscale x 2 x i64> %a, %not ret <vscale x 2 x i64> %and >From 4253f8737eca97a87063aa2952ab7884969f245c Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Wed, 21 Oct 2026 16:27:26 +0100 Subject: [PATCH 04/10] fixup! Replace performSVEAndNotIntrinsicCombine() with tablegen Move AND_U and NOT matching to TableGen, using a shared pattern fragment for both operand orders. Retain register and immediate AND selection for other AND_U operations. --- .../Target/AArch64/AArch64ISelLowering.cpp | 38 ++++++++++------ .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 44 ++++++++++++++++++- 2 files changed, 67 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 540da867dc9678..b0d8dda0a6d668 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -22364,9 +22364,7 @@ static SDValue performReinterpretCastCombine(SDNode *N) { static SDValue performSVEAndCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { - assert(N->getOpcode() == ISD::AND && "Expected AND instruction"); SelectionDAG &DAG = DCI.DAG; - SDValue Src = N->getOperand(0); unsigned Opc = Src->getOpcode(); @@ -25625,22 +25623,34 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N, DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64)); } -static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { +static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { SDValue Pg = N->getOperand(1); - // Fold matching predicates before lowering AND_U discards its predicate. - // NOT's inactive lanes cannot affect AND_U's active lanes. - for (unsigned I = 0; I != 2; ++I) { - unsigned NotOperandIdx = 2 + I; - unsigned OtherOperandIdx = 3 - I; - SDValue Not = N->getOperand(NotOperandIdx); - if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not || - Not.getOperand(2) != Pg) + // Preserve matching predicates so TableGen can select: + // and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b) + for (unsigned I = 2; I != 4; ++I) { + SDValue Not = N->getOperand(I); + if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not) continue; - return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), - N->getOperand(OtherOperandIdx), Not.getOperand(3)); + SDValue NotPg = Not.getOperand(2); + if (NotPg != Pg) + continue; + + // We can discard AND_U's predicate if NOT's inactive lanes are + // undefined or NOT's predicate is all-active: ordinary AND-NOT + // patterns can select BIC. + SDValue Inactive = Not.getOperand(1); + if (Inactive.isUndef() || isAllActivePredicate(DAG, NotPg)) + continue; + + // Keep AND_U unchanged so TableGen can match the predicates. + return SDValue(); } + + // Otherwise expose an ordinary AND so generic DAG combines can simplify + // it, for example: + // and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15) return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); } @@ -25864,7 +25874,7 @@ static SDValue performIntrinsicCombine(SDNode *N, case Intrinsic::aarch64_sve_subr: return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); case Intrinsic::aarch64_sve_and_u: - return performSVEAndNotIntrinsicCombine(N, DAG); + return performSVEAndUIntrinsicCombine(N, DAG); case Intrinsic::aarch64_sve_bic_u: return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 958835f1858608..3204884ee66915 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -559,6 +559,11 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2), (and node:$op1, (xor node:$op2, (SVEAllActive))), (AArch64bic_node node:$op1, node:$op2)]>; +// AND_U permits either operand order. +def SVEAndU : PatFrags<(ops node:$pg, node:$a, node:$b), + [(int_aarch64_sve_and_u node:$pg, node:$a, node:$b), + (int_aarch64_sve_and_u node:$pg, node:$b, node:$a)]>; + def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2), [(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2), (and node:$pg, (and node:$op1, node:$op2))]>; @@ -753,14 +758,40 @@ let Predicates = [HasSVE_or_SME] in { defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>; defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>; - // Select BIC when NOT's inactive values are undefined or all lanes are active. multiclass SVEAndNotPats<ValueType VT, ValueType PT> { + // Ordinary AND: NOT's undefined inactive lanes may take the value ~b. def : Pat<(and VT:$a, (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))), (BIC_ZZZ $a, $b)>; + // Ordinary AND: an all-active NOT produces ~b in every lane. def : Pat<(and VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))), (BIC_ZZZ $a, $b)>; + + // AND_U: matching predicates ensure NOT produces ~b in every active AND lane. + def : Pat<(VT (SVEAndU PT:$pg, VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // AND_U: NOT's undefined inactive lanes may take the value ~b, + // even when the predicates differ. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // AND_U with ordinary LLVM NOT, which produces ~b in every lane. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))), + (BIC_ZZZ $a, $b)>; + + // Fall back to register AND when no BIC pattern matches. + // AND_U allows any result in inactive lanes. + def : Pat<(VT (int_aarch64_sve_and_u (PT srcvalue), VT:$a, VT:$b)), + (AND_ZZZ $a, $b)>; } defm : SVEAndNotPats<nxv16i8, nxv16i1>; defm : SVEAndNotPats<nxv8i16, nxv8i1>; @@ -825,6 +856,17 @@ let Predicates = [HasSVE_or_SME] in { defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>; defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>; + // Allow retained AND_U intrinsics to select AND-immediate instructions. + class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR, + ValueType IT, ComplexPattern CP> + : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a), + (VT (splat_vector (IT (CP i64:$imm)))))), + (AND_ZI_PSEUDO $a, i64:$imm)>; + def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>; + def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>; + def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>; + def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>; + defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>; defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>; defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>; >From 061db66e1f75f92e8894f5f3569eff2ec575d886 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Thu, 22 Oct 2026 16:36:25 +0100 Subject: [PATCH 05/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms Remove the AND_U pattern for NOT with undefined inactive lanes and an arbitrary predicate. Keep the original fix focused; ordinary cases still lower to AND and use the existing AND-NOT patterns. --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 6 ------ 1 file changed, 6 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 3204884ee66915..eba913eb05762d 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -773,12 +773,6 @@ let Predicates = [HasSVE_or_SME] in { (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), (BIC_ZZZ $a, $b)>; - // AND_U: NOT's undefined inactive lanes may take the value ~b, - // even when the predicates differ. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, - (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))), - (BIC_ZZZ $a, $b)>; - // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate. def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), >From 8cf58a119ef3dfcb317fb809d7213cf983a1ef64 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Thu, 22 Oct 2026 16:44:55 +0100 Subject: [PATCH 06/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms Remove the extra AND_U patterns for all-active NOT, ordinary LLVM NOT and register AND fallback. The C++ helper lowers ordinary cases to AND and retains matching-predicate AND-NOT pairs for the remaining BIC pattern. --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index eba913eb05762d..005a2e795a4215 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -772,20 +772,6 @@ let Predicates = [HasSVE_or_SME] in { def : Pat<(VT (SVEAndU PT:$pg, VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), (BIC_ZZZ $a, $b)>; - - // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, - (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), - (BIC_ZZZ $a, $b)>; - - // AND_U with ordinary LLVM NOT, which produces ~b in every lane. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))), - (BIC_ZZZ $a, $b)>; - - // Fall back to register AND when no BIC pattern matches. - // AND_U allows any result in inactive lanes. - def : Pat<(VT (int_aarch64_sve_and_u (PT srcvalue), VT:$a, VT:$b)), - (AND_ZZZ $a, $b)>; } defm : SVEAndNotPats<nxv16i8, nxv16i1>; defm : SVEAndNotPats<nxv8i16, nxv8i1>; >From 16aaf5244136bd772cfd3587cd03bc6dc2ef8302 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Thu, 22 Oct 2026 17:23:16 +0100 Subject: [PATCH 07/10] fixup! Remove patterns Remove these patterns, as they're no longer needed since I've added the C++ code in performSVEAndUIntrinsicCombine() --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 005a2e795a4215..9fb57945a06bf8 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -836,17 +836,6 @@ let Predicates = [HasSVE_or_SME] in { defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>; defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>; - // Allow retained AND_U intrinsics to select AND-immediate instructions. - class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR, - ValueType IT, ComplexPattern CP> - : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a), - (VT (splat_vector (IT (CP i64:$imm)))))), - (AND_ZI_PSEUDO $a, i64:$imm)>; - def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>; - def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>; - def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>; - def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>; - defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>; defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>; defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>; >From af73492a4e1e7afb198bd7cce0296bd89419d698 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Thu, 22 Oct 2026 18:49:10 +0100 Subject: [PATCH 08/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms Lower AND_U to an AArch64 target DAG node and match that node in TableGen. Keep the predicate for matching AND-NOT pairs, and combine other cases to ordinary AND so generic simplifications remain available. Name the data operands and iterate over them directly. --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 17 ++++++++++------- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 7 ++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index b0d8dda0a6d668..02a7bb7001194d 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -25623,13 +25623,14 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N, DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64)); } -static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { - SDValue Pg = N->getOperand(1); +static SDValue performSVEAndUCombine(SDNode *N, SelectionDAG &DAG) { + SDValue Pg = N->getOperand(0); + SDValue A = N->getOperand(1); + SDValue B = N->getOperand(2); // Preserve matching predicates so TableGen can select: // and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b) - for (unsigned I = 2; I != 4; ++I) { - SDValue Not = N->getOperand(I); + for (SDValue Not : {A, B}) { if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not) continue; @@ -25651,8 +25652,7 @@ static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { // Otherwise expose an ordinary AND so generic DAG combines can simplify // it, for example: // and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15) - return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2), - N->getOperand(3)); + return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), A, B); } static SDValue performIntrinsicCombine(SDNode *N, @@ -25874,7 +25874,8 @@ static SDValue performIntrinsicCombine(SDNode *N, case Intrinsic::aarch64_sve_subr: return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); case Intrinsic::aarch64_sve_and_u: - return performSVEAndUIntrinsicCombine(N, DAG); + return DAG.getNode(AArch64ISD::AND_U, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2), N->getOperand(3)); case Intrinsic::aarch64_sve_bic_u: return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); @@ -32201,6 +32202,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, return performORCombine(N, DCI); case ISD::AND: return performANDCombine(N, DCI); + case AArch64ISD::AND_U: + return performSVEAndUCombine(N, DAG); case ISD::FADD: return performFADDCombine(N, DCI); case AArch64ISD::FADD_PRED: diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 9fb57945a06bf8..099edf459391c9 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -251,6 +251,7 @@ def SDT_AArch64FMA : SDTypeProfile<1, 4, [ ]>; // Predicated operations with the result of inactive lanes being unspecified. +def AArch64and_u : SDNode<"AArch64ISD::AND_U", SDT_AArch64Arith>; def AArch64asr_p : SDNode<"AArch64ISD::SRA_PRED", SDT_AArch64Arith>; def AArch64lsl_p : SDNode<"AArch64ISD::SHL_PRED", SDT_AArch64Arith>; def AArch64lsr_p : SDNode<"AArch64ISD::SRL_PRED", SDT_AArch64Arith>; @@ -559,10 +560,10 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2), (and node:$op1, (xor node:$op2, (SVEAllActive))), (AArch64bic_node node:$op1, node:$op2)]>; -// AND_U permits either operand order. +// Match AND_U with either data operand order, keeping the predicate unchanged. def SVEAndU : PatFrags<(ops node:$pg, node:$a, node:$b), - [(int_aarch64_sve_and_u node:$pg, node:$a, node:$b), - (int_aarch64_sve_and_u node:$pg, node:$b, node:$a)]>; + [(AArch64and_u node:$pg, node:$a, node:$b), + (AArch64and_u node:$pg, node:$b, node:$a)]>; def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2), [(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2), >From a43532f6fa965c1aa9c9cd0d9eafcde53317832a Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Fri, 23 Oct 2026 15:28:12 +0100 Subject: [PATCH 09/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms Keep AND_U as an AArch64 target node until instruction selection. Remove the C++ combine and restore the TableGen patterns for BIC, register AND and immediate AND selection. --- .../Target/AArch64/AArch64ISelLowering.cpp | 34 ------------------- .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 31 +++++++++++++++++ 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 02a7bb7001194d..f806a3223ed4eb 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -25623,38 +25623,6 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N, DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64)); } -static SDValue performSVEAndUCombine(SDNode *N, SelectionDAG &DAG) { - SDValue Pg = N->getOperand(0); - SDValue A = N->getOperand(1); - SDValue B = N->getOperand(2); - - // Preserve matching predicates so TableGen can select: - // and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b) - for (SDValue Not : {A, B}) { - if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not) - continue; - - SDValue NotPg = Not.getOperand(2); - if (NotPg != Pg) - continue; - - // We can discard AND_U's predicate if NOT's inactive lanes are - // undefined or NOT's predicate is all-active: ordinary AND-NOT - // patterns can select BIC. - SDValue Inactive = Not.getOperand(1); - if (Inactive.isUndef() || isAllActivePredicate(DAG, NotPg)) - continue; - - // Keep AND_U unchanged so TableGen can match the predicates. - return SDValue(); - } - - // Otherwise expose an ordinary AND so generic DAG combines can simplify - // it, for example: - // and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15) - return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), A, B); -} - static SDValue performIntrinsicCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) { @@ -32202,8 +32170,6 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, return performORCombine(N, DCI); case ISD::AND: return performANDCombine(N, DCI); - case AArch64ISD::AND_U: - return performSVEAndUCombine(N, DAG); case ISD::FADD: return performFADDCombine(N, DCI); case AArch64ISD::FADD_PRED: diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 099edf459391c9..889c5c4f7eee30 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -773,6 +773,26 @@ let Predicates = [HasSVE_or_SME] in { def : Pat<(VT (SVEAndU PT:$pg, VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), (BIC_ZZZ $a, $b)>; + + // AND_U: NOT's undefined inactive lanes may take the value ~b, + // even when the predicates differ. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // AND_U with ordinary LLVM NOT, which produces ~b in every lane. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))), + (BIC_ZZZ $a, $b)>; + + // Fall back to register AND when no BIC pattern matches. + // AND_U allows any result in inactive lanes. + def : Pat<(VT (AArch64and_u (PT srcvalue), VT:$a, VT:$b)), + (AND_ZZZ $a, $b)>; } defm : SVEAndNotPats<nxv16i8, nxv16i1>; defm : SVEAndNotPats<nxv8i16, nxv8i1>; @@ -837,6 +857,17 @@ let Predicates = [HasSVE_or_SME] in { defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>; defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>; + // Allow AND_U nodes to select AND-immediate instructions. + class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR, + ValueType IT, ComplexPattern CP> + : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a), + (VT (splat_vector (IT (CP i64:$imm)))))), + (AND_ZI_PSEUDO $a, i64:$imm)>; + def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>; + def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>; + def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>; + def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>; + defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>; defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>; defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>; >From 6f119bf071f0b86fb93464f83fabe176ce0d1b20 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Fri, 23 Oct 2026 15:45:24 +0100 Subject: [PATCH 10/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms Check BIC selection for AND_U with poison NOT passthrough and different predicates. Cover the pattern independently of the matching-predicate case. --- .../CodeGen/AArch64/sve-intrinsics-and-not.ll | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll index 4d8d595f242c24..13b4baa8c05549 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll @@ -124,23 +124,10 @@ define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4 ; Poison NOT passthrough permits BIC even when the predicates differ. define <vscale x 4 x i32> @different_predicates_poison(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg, <vscale x 4 x i1> %not_pg) { -; SVE-LABEL: different_predicates_poison: -; SVE: // %bb.0: -; SVE-NEXT: not z1.s, p1/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: different_predicates_poison: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: not z1.s, p1/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: different_predicates_poison: -; SME: // %bb.0: -; SME-NEXT: not z1.s, p1/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: different_predicates_poison: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %not_pg, <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
