https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/207165
>From 8abccc81a7555d67a0261df4121b501d979b4db3 Mon Sep 17 00:00:00 2001 From: Hassnaa Hamdi <[email protected]> Date: Wed, 1 Jul 2026 22:18:09 +0100 Subject: [PATCH] Patch 4: [LV][NFC]: Filter between vplans by tail-folding status --- .../Vectorize/LoopVectorizationPlanner.h | 7 ++-- .../Transforms/Vectorize/LoopVectorize.cpp | 24 ++++++++----- llvm/lib/Transforms/Vectorize/VPlan.cpp | 34 ++++++++++++++++--- llvm/lib/Transforms/Vectorize/VPlan.h | 9 +++++ 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 4829506b3aa75..b4220d284b9ae 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -883,7 +883,7 @@ class LoopVectorizationPlanner { /// Return the VPlan for \p VF. At the moment, there is always a single VPlan /// for each VF. - VPlan &getPlanFor(ElementCount VF) const; + VPlan &getPlanFor(ElementCount VF, bool TF) const; /// Compute and return the most profitable vectorization factor and the /// corresponding best VPlan. Also collect all profitable VFs in @@ -927,10 +927,7 @@ class LoopVectorizationPlanner { /// Look through the existing plans and return true if we have one with /// vectorization factor \p VF. - bool hasPlanWithVF(ElementCount VF) const { - return any_of(VPlans, - [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); }); - } + bool hasPlanWithVF(ElementCount VF, bool TF) const; /// Test a \p Predicate on a \p Range of VF's. Return the value of applying /// \p Predicate on Range.Start, possibly decreasing Range.End such that the diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 6901454770479..3590353932540 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3490,8 +3490,9 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan( LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n"); ElementCount ForcedEC = ElementCount::getFixed(EpilogueVectorizationForceVF); - if (hasPlanWithVF(ForcedEC)) { - std::unique_ptr<VPlan> Clone(getPlanFor(ForcedEC).duplicate()); + if (hasPlanWithVF(ForcedEC, CM.foldTailByMasking())) { + std::unique_ptr<VPlan> Clone( + getPlanFor(ForcedEC, CM.foldTailByMasking()).duplicate()); Clone->setVF(ForcedEC); return Clone; } @@ -3582,10 +3583,10 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan( VPlan *BestPlan = nullptr; for (auto &NextVF : ProfitableVFs) { // Skip candidate VFs without a corresponding VPlan. - if (!hasPlanWithVF(NextVF.Width)) + if (!hasPlanWithVF(NextVF.Width, CM.foldTailByMasking())) continue; - VPlan &CurrentPlan = getPlanFor(NextVF.Width); + VPlan &CurrentPlan = getPlanFor(NextVF.Width, CM.foldTailByMasking()); ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width); // Skip candidate VFs with widths >= the (estimated) runtime VF (scalable // vectors) or > the VF of the main loop (fixed vectors). @@ -5831,13 +5832,15 @@ std::pair<VectorizationFactor, VPlan *> LoopVectorizationPlanner::computeBestVF( if (VPlans.size() == 1) { // For outer loops, the plan has a single vector VF determined by the // heuristic. - assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) || + assert((FirstPlan.hasScalarVFOnly() || + hasPlanWithVF(UserVF, CM.foldTailByMasking()) || FirstPlan.isOuterLoop()) && "must have a single scalar VF, UserVF or an outer loop"); return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan}; } - if (hasPlanWithVF(UserVF) && EpilogueVectorizationForceVF > 1) { + if (hasPlanWithVF(UserVF, CM.foldTailByMasking()) && + EpilogueVectorizationForceVF > 1) { assert(VPlans.size() == 2 && "Must have exactly 2 VPlans built"); assert(VPlans[0]->getSingleVF() == ElementCount::getFixed(EpilogueVectorizationForceVF) && @@ -5908,6 +5911,8 @@ std::pair<VectorizationFactor, VPlan *> LoopVectorizationPlanner::computeBestVF( continue; } + assert(P->isCompatibleWithTF(CM.foldTailByMasking()) && + "All vplans must be compatible with the CM"); InstructionCost Cost = cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr, CM); VectorizationFactor CurrentFactor(VF, Cost, ScalarCost); @@ -5921,8 +5926,8 @@ std::pair<VectorizationFactor, VPlan *> LoopVectorizationPlanner::computeBestVF( if (isMoreProfitable(CurrentFactor, ScalarFactor, P->hasScalarTail())) ProfitableVFs.push_back(CurrentFactor); - // Get the costs for the EpilogueTailFoldingCM: if (EpilogueTailFoldingCM) { + // Get the costs for the EpilogueTailFoldingCM: LLVM_DEBUG(dbgs() << "LV: Predicated CM, calculate costs for VF: " << VF << "\n"); cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr, @@ -8133,7 +8138,7 @@ bool LoopVectorizePass::processLoop(Loop *L) { GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind, CM.maskPartialAliasing()); - if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) { + if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking())) { // Select the interleave count. IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost, CM); @@ -8195,7 +8200,8 @@ bool LoopVectorizePass::processLoop(Loop *L) { "Ignoring user-specified interleave count due to possibly " "unsafe dependencies in the loop."}; InterleaveLoop = false; - } else if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) { + } else if (!LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking()) && + UserIC > 1) { // Tell the user interleaving was avoided up-front, despite being explicitly // requested. LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and " diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 292e3e85da9d5..bf93d8000cad9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1310,6 +1310,25 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) { return VPIRBB; } +bool VPlan::isCompatibleWithTF(bool TF) { + bool HasHeaderMask = (vputils::findHeaderMask(*this) != nullptr); + bool NoTail = !hasScalarTail(); + // Check if TC is not divisible by the VF to make sure that the vplan doesn't + // have a tail because the tail is folded not because there is no remaining + // iterations. + bool TCNotDivisibleByVF = true; + if (auto *ConstTC = dyn_cast<VPConstantInt>(getTripCount())) { + const APInt &TC = + cast<ConstantInt>(ConstTC->getLiveInIRValue())->getValue(); + TCNotDivisibleByVF = any_of(vectorFactors(), [&TC](ElementCount VF) { + APInt VFVal(TC.getBitWidth(), VF.getKnownMinValue()); + return TC.urem(VFVal) != 0; + }); + } + NoTail &= TCNotDivisibleByVF; + return (TF == (HasHeaderMask | NoTail)); +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) Twine VPlanPrinter::getUID(const VPBlockBase *Block) { @@ -1678,19 +1697,26 @@ VPBuilder::createConsecutiveVectorPointer(VPValue *Ptr, Type *SourceElementTy, return createVectorPointer(Ptr, SourceElementTy, StrideOne, Flags, DL); } -VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const { +VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF, bool TF) const { assert(count_if(VPlans, - [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) == - 1 && + [VF, TF](const VPlanPtr &Plan) { + return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF); + }) == 1 && "Multiple VPlans for VF."); for (const VPlanPtr &Plan : VPlans) { - if (Plan->hasVF(VF)) + if (Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF)) return *Plan.get(); } llvm_unreachable("No plan found!"); } +bool LoopVectorizationPlanner::hasPlanWithVF(ElementCount VF, bool TF) const { + return any_of(VPlans, [VF, TF](const VPlanPtr &Plan) { + return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF); + }); +} + static void addRuntimeUnrollDisableMetaData(Loop *L) { SmallVector<Metadata *, 4> MDs; // Reserve first location for self reference to the LoopID metadata node. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index b2d179d8e6d98..ea4d4614dd80d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -5145,6 +5145,15 @@ class VPlan { is_contained(ScalarPH->getPredecessors(), getMiddleBlock()); } + /// Returns true if this VPlan's tail-folding matches \p TF. + /// A plan is tail-folded if it has a header mask, or if it has no scalar + /// tail while the TC is not evenly divisible by VF (in which case the tail + /// must be folded by the vector loop itself). + /// + /// This function is used to disambiguate VPlan lookup by VF when multiple + /// plans share the same VF but differ in tail-folding status. + bool isCompatibleWithTF(bool TF); + /// The type of the canonical induction variable of the vector loop. Type *getIndexType() const { return VF.getType(); } }; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
