https://github.com/hassnaaHamdi created https://github.com/llvm/llvm-project/pull/207165
None >From cc4cfdbb5bf2324817ed43951d47218b4f5c44df Mon Sep 17 00:00:00 2001 From: Hassnaa Hamdi <[email protected]> Date: Wed, 1 Jul 2026 22:18:09 +0100 Subject: [PATCH] Patch 4: [LV][NFC]: Filter between vplans by tail-folding status --- .../Vectorize/LoopVectorizationPlanner.h | 7 ++-- .../Transforms/Vectorize/LoopVectorize.cpp | 20 +++++++----- llvm/lib/Transforms/Vectorize/VPlan.cpp | 32 +++++++++++++++++-- llvm/lib/Transforms/Vectorize/VPlan.h | 2 ++ 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 249b00a6da196..20dde28e75b4e 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -872,7 +872,7 @@ class LoopVectorizationPlanner { /// Return the VPlan for \p VF. At the moment, there is always a single VPlan /// for each VF. - VPlan &getPlanFor(ElementCount VF) const; + VPlan &getPlanFor(ElementCount VF, bool TF) const; /// Compute and return the most profitable vectorization factor and the /// corresponding best VPlan. Also collect all profitable VFs in @@ -916,10 +916,7 @@ class LoopVectorizationPlanner { /// Look through the existing plans and return true if we have one with /// vectorization factor \p VF. - bool hasPlanWithVF(ElementCount VF) const { - return any_of(VPlans, - [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); }); - } + bool hasPlanWithVF(ElementCount VF, bool TF) const; /// Test a \p Predicate on a \p Range of VF's. Return the value of applying /// \p Predicate on Range.Start, possibly decreasing Range.End such that the diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 9f5025ff30642..fbb19849ed0dc 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3147,6 +3147,8 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks( using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>; SmallVector<RecipeVFPair> InvalidCosts; for (const auto &Plan : VPlans) { + if (!Plan->isCompatibleWithTF(CM.foldTailByMasking())) + continue; for (ElementCount VF : Plan->vectorFactors()) { // The VPlan-based cost model is designed for computing vector cost. // Querying VPlan-based cost model with a scarlar VF will cause some @@ -3490,8 +3492,8 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan( LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n"); ElementCount ForcedEC = ElementCount::getFixed(EpilogueVectorizationForceVF); - if (hasPlanWithVF(ForcedEC)) { - std::unique_ptr<VPlan> Clone(getPlanFor(ForcedEC).duplicate()); + if (hasPlanWithVF(ForcedEC, CM.foldTailByMasking())) { + std::unique_ptr<VPlan> Clone(getPlanFor(ForcedEC, CM.foldTailByMasking()).duplicate()); Clone->setVF(ForcedEC); return Clone; } @@ -3582,10 +3584,10 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan( VPlan *BestPlan = nullptr; for (auto &NextVF : ProfitableVFs) { // Skip candidate VFs without a corresponding VPlan. - if (!hasPlanWithVF(NextVF.Width)) + if (!hasPlanWithVF(NextVF.Width, CM.foldTailByMasking())) continue; - VPlan &CurrentPlan = getPlanFor(NextVF.Width); + VPlan &CurrentPlan = getPlanFor(NextVF.Width, CM.foldTailByMasking()); ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width); // Skip candidate VFs with widths >= the (estimated) runtime VF (scalable // vectors) or > the VF of the main loop (fixed vectors). @@ -5835,13 +5837,13 @@ std::pair<VectorizationFactor, VPlan *> LoopVectorizationPlanner::computeBestVF( if (VPlans.size() == 1) { // For outer loops, the plan has a single vector VF determined by the // heuristic. - assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) || + assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF, CM.foldTailByMasking()) || FirstPlan.isOuterLoop()) && "must have a single scalar VF, UserVF or an outer loop"); return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan}; } - if (hasPlanWithVF(UserVF) && EpilogueVectorizationForceVF > 1) { + if (hasPlanWithVF(UserVF, CM.foldTailByMasking()) && EpilogueVectorizationForceVF > 1) { assert(VPlans.size() == 2 && "Must have exactly 2 VPlans built"); assert(VPlans[0]->getSingleVF() == ElementCount::getFixed(EpilogueVectorizationForceVF) && @@ -5882,6 +5884,8 @@ std::pair<VectorizationFactor, VPlan *> LoopVectorizationPlanner::computeBestVF( VPlan *PlanForBestVF = &FirstPlan; for (auto &P : VPlans) { + if (!P->isCompatibleWithTF(CM.foldTailByMasking())) + continue; ArrayRef<ElementCount> VFs(P->vectorFactors().begin(), P->vectorFactors().end()); @@ -8158,7 +8162,7 @@ bool LoopVectorizePass::processLoop(Loop *L) { GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind, CM.maskPartialAliasing()); - if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) { + if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking())) { // Select the interleave count. IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost, CM); @@ -8220,7 +8224,7 @@ bool LoopVectorizePass::processLoop(Loop *L) { "Ignoring user-specified interleave count due to possibly " "unsafe dependencies in the loop."}; InterleaveLoop = false; - } else if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) { + } else if (!LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking()) && UserIC > 1) { // Tell the user interleaving was avoided up-front, despite being explicitly // requested. LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and " diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 9c0bec2350c97..c1d168d51e61a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1308,6 +1308,26 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) { return VPIRBB; } +bool VPlan::isCompatibleWithTF(bool TF) { + bool HasHeaderMask = (vputils::findHeaderMask(*this) != nullptr); + bool NoTail = !hasScalarTail(); + // Check if TC is not divisible by the VF to make sure that the plan doesn't + // have a tail because the tail is folded not because there is no remaining + // iterations. For scalable VFs, divisibility cannot be determined at + // compile-time, so conservatively keep TCNotDivisibleByVF = true. + bool TCNotDivisibleByVF = true; + if (auto *ConstTC = dyn_cast<VPConstantInt>(getTripCount())) { + const APInt &TC = cast<ConstantInt>(ConstTC->getLiveInIRValue())->getValue(); + if (none_of(vectorFactors(), [](ElementCount VF) { return VF.isScalable(); })) + TCNotDivisibleByVF = any_of(vectorFactors(), [&TC](ElementCount VF) { + APInt VFVal(TC.getBitWidth(), VF.getFixedValue()); + return TC.urem(VFVal) != 0; + }); + } + NoTail &= TCNotDivisibleByVF; + return (TF == (HasHeaderMask | NoTail)); +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) Twine VPlanPrinter::getUID(const VPBlockBase *Block) { @@ -1655,19 +1675,25 @@ bool LoopVectorizationPlanner::getDecisionAndClampRange( return PredicateAtRangeStart; } -VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const { +VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF, bool TF) const { assert(count_if(VPlans, - [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) == + [VF, TF](const VPlanPtr &Plan) { return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF); }) == 1 && "Multiple VPlans for VF."); for (const VPlanPtr &Plan : VPlans) { - if (Plan->hasVF(VF)) + if (Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF)) return *Plan.get(); } llvm_unreachable("No plan found!"); } +bool LoopVectorizationPlanner::hasPlanWithVF(ElementCount VF, bool TF) const { + return any_of(VPlans, [VF, TF](const VPlanPtr &Plan) { + return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF); + }); +} + static void addRuntimeUnrollDisableMetaData(Loop *L) { SmallVector<Metadata *, 4> MDs; // Reserve first location for self reference to the LoopID metadata node. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index eaf9d1433aff7..6cb238af9804e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -5155,6 +5155,8 @@ class VPlan { is_contained(ScalarPH->getPredecessors(), getMiddleBlock()); } + bool isCompatibleWithTF(bool TF); + /// The type of the canonical induction variable of the vector loop. Type *getIndexType() const { return VF.getType(); } }; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
