https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/226516
>From bbfbc89c2c7df8e336578286ee05009acdf5af07 Mon Sep 17 00:00:00 2001 From: Luke Lau <[email protected]> Date: Wed, 23 Sep 2026 16:58:44 +0100 Subject: [PATCH] [VPlan] Use vputils::findCanonicalIVIncrement in more places. NFC In a future patch we want to enable more scenarios with early exits and tail folding. In these scenarios the terminator will no longer be a BranchOnCount/BranchOnCond but a BranchOnTwoConds, so a bunch of canonical IV matchers break. Use vputils::findCanonicalIVIncrement instead since it searches the users of the canonical IV instead of looking at the terminator. --- .../Transforms/Vectorize/LoopVectorize.cpp | 11 ++-------- .../Vectorize/VPlanConstruction.cpp | 10 ++-------- .../Vectorize/VPlanEVLTailFolding.cpp | 20 +++++++------------ 3 files changed, 11 insertions(+), 30 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 8a4800a80a22f4..9e35bc510892bb 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6389,15 +6389,8 @@ LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan, VFRange &Range, // count is >= increment and a multiple of the increment. VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion(); bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None; - if (!HasNUW) { - auto *IVInc = - LoopRegion->getExitingBasicBlock()->getTerminator()->getOperand(0); - assert(match(IVInc, - m_VPInstruction<Instruction::Add>( - m_Specific(LoopRegion->getCanonicalIV()), m_VPValue())) && - "Did not find the canonical IV increment"); - LoopRegion->clearCanonicalIVNUW(cast<VPInstruction>(IVInc)); - } + if (!HasNUW) + LoopRegion->clearCanonicalIVNUW(vputils::findCanonicalIVIncrement(*Plan)); // --------------------------------------------------------------------------- // Pre-construction: record ingredients whose recipes we'll need to further diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index db47293ca8476d..ff009b49a79fa6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -1393,14 +1393,8 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) { Builder.createNaryOp(VPInstruction::BranchOnCond, HeaderMask); VPBasicBlock *OrigLatch = LoopRegion->getExitingBasicBlock(); - VPValue *IVInc; - [[maybe_unused]] bool TermBranchOnCount = - match(OrigLatch->getTerminator(), - m_BranchOnCount(m_VPValue(IVInc), - m_Specific(&Plan.getVectorTripCount()))); - assert(TermBranchOnCount && - match(IVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()), - m_Specific(&Plan.getVFxUF()))) && + VPValue *IVInc = vputils::findCanonicalIVIncrement(Plan); + assert(IVInc && std::next(IVInc->getDefiningRecipe()->getIterator()) == OrigLatch->getTerminator()->getIterator() && "Unexpected canonical iv increment"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp index 956b92eb07a21e..62ef733d2899dc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp @@ -632,19 +632,13 @@ void VPlanTransforms::convertEVLExitCond(VPlan &Plan) { if (match(LatchBr, m_BranchOnCond(m_True()))) return; - VPValue *CanIVInc; - [[maybe_unused]] bool FoundIncrement = match( - LatchBr, - m_BranchOnCond(m_SpecificCmp(CmpInst::ICMP_EQ, m_VPValue(CanIVInc), - m_Specific(&Plan.getVectorTripCount())))); - assert(FoundIncrement && - match(CanIVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()), - m_Specific(&Plan.getVFxUF()))) && - "Expected BranchOnCond with ICmp comparing CanIV + VFxUF with vector " - "trip count"); + VPValue *CanIVInc = vputils::findCanonicalIVIncrement(Plan); + auto *ExitCond = findSingleUser<VPInstruction>( + CanIVInc, m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(CanIVInc), + m_Specific(&Plan.getVectorTripCount()))); Type *AVLTy = AVLNext->getScalarType(); - VPBuilder Builder(LatchBr); - LatchBr->setOperand( - 0, Builder.createICmp(CmpInst::ICMP_EQ, AVLNext, Plan.getZero(AVLTy))); + VPBuilder Builder(ExitCond); + ExitCond->replaceAllUsesWith( + Builder.createICmp(CmpInst::ICMP_EQ, AVLNext, Plan.getZero(AVLTy))); } _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
