https://github.com/madhur13490 updated https://github.com/llvm/llvm-project/pull/222613
>From 6c218b69627f8d05967997d2ab252e87fe719a8e Mon Sep 17 00:00:00 2001 From: Madhur Amilkanthwar <[email protected]> Date: Thu, 10 Sep 2026 04:12:20 -0700 Subject: [PATCH] [SLP][modularisation][NFC] Move getReductionInstr/getAggregateSize to SLPReductionUtils Move the BoUpSLP-independent helpers getReductionInstr and getAggregateSize out of SLPVectorizer.cpp into the self-contained SLPVectorizer/SLPReductionUtils.{h,cpp} module. Part of the SLPVectorizer.cpp modularization effort: https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 79 ------------------- .../SLPVectorizer/SLPReductionUtils.cpp | 77 ++++++++++++++++++ .../SLPVectorizer/SLPReductionUtils.h | 15 ++++ 3 files changed, 92 insertions(+), 79 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index a4ba4b3bf6ae43..6a2fc84d528024 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -34810,34 +34810,6 @@ class HorizontalReduction { static RecurKind getRdxKind(Value *V) { return HorizontalReduction::getRdxKind(V); } -static std::optional<unsigned> getAggregateSize(Instruction *InsertInst) { - if (auto *IE = dyn_cast<InsertElementInst>(InsertInst)) - return cast<FixedVectorType>(IE->getType())->getNumElements(); - - unsigned AggregateSize = 1; - auto *IV = cast<InsertValueInst>(InsertInst); - Type *CurrentType = IV->getType(); - do { - if (auto *ST = dyn_cast<StructType>(CurrentType)) { - for (auto *Elt : ST->elements()) - if (Elt != ST->getElementType(0)) // check homogeneity - return std::nullopt; - AggregateSize *= ST->getNumElements(); - CurrentType = ST->getElementType(0); - } else if (auto *AT = dyn_cast<ArrayType>(CurrentType)) { - AggregateSize *= AT->getNumElements(); - CurrentType = AT->getElementType(); - } else if (auto *VT = dyn_cast<FixedVectorType>(CurrentType)) { - AggregateSize *= VT->getNumElements(); - return AggregateSize; - } else if (CurrentType->isSingleValueType()) { - return AggregateSize; - } else { - return std::nullopt; - } - } while (true); -} - static void findBuildAggregateRec(Instruction *LastInsertInst, TargetTransformInfo *TTI, SmallVectorImpl<Value *> &BuildVectorOpds, @@ -34905,57 +34877,6 @@ static bool findBuildAggregate(Instruction *LastInsertInst, return false; } -/// Try and get a reduction instruction from a phi node. -/// -/// Given a phi node \p P in a block \p ParentBB, consider possible reductions -/// if they come from either \p ParentBB or a containing loop latch. -/// -/// \returns A candidate reduction value if possible, or \code nullptr \endcode -/// if not possible. -static Instruction *getReductionInstr(const DominatorTree *DT, PHINode *P, - BasicBlock *ParentBB, LoopInfo *LI) { - // There are situations where the reduction value is not dominated by the - // reduction phi. Vectorizing such cases has been reported to cause - // miscompiles. See PR25787. - auto DominatedReduxValue = [&](Value *R) { - return isa<Instruction>(R) && - DT->dominates(P->getParent(), cast<Instruction>(R)->getParent()); - }; - - Instruction *Rdx = nullptr; - - // Return the incoming value if it comes from the same BB as the phi node. - if (P->getIncomingBlock(0) == ParentBB) { - Rdx = dyn_cast<Instruction>(P->getIncomingValue(0)); - } else if (P->getIncomingBlock(1) == ParentBB) { - Rdx = dyn_cast<Instruction>(P->getIncomingValue(1)); - } - - if (Rdx && DominatedReduxValue(Rdx)) - return Rdx; - - // Otherwise, check whether we have a loop latch to look at. - Loop *BBL = LI->getLoopFor(ParentBB); - if (!BBL) - return nullptr; - BasicBlock *BBLatch = BBL->getLoopLatch(); - if (!BBLatch) - return nullptr; - - // There is a loop latch, return the incoming value if it comes from - // that. This reduction pattern occasionally turns up. - if (P->getIncomingBlock(0) == BBLatch) { - Rdx = dyn_cast<Instruction>(P->getIncomingValue(0)); - } else if (P->getIncomingBlock(1) == BBLatch) { - Rdx = dyn_cast<Instruction>(P->getIncomingValue(1)); - } - - if (Rdx && DominatedReduxValue(Rdx)) - return Rdx; - - return nullptr; -} - /// We could have an initial reduction that is not an add. /// r *= v1 + v2 + v3 + v4 /// In such a case start looking for a tree rooted in the first '+'. diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPReductionUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPReductionUtils.cpp index e8626c9a00b480..70189faaafad34 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPReductionUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPReductionUtils.cpp @@ -14,15 +14,20 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallBitVector.h" #include "llvm/Analysis/IVDescriptors.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/PatternMatch.h" #include "llvm/IR/Type.h" +#include <optional> + using namespace llvm; using namespace llvm::PatternMatch; @@ -169,4 +174,76 @@ Value *tryEmitBoolReduxBitcastCmp(IRBuilderBase &Builder, return Res; } +std::optional<unsigned> getAggregateSize(Instruction *InsertInst) { + if (auto *IE = dyn_cast<InsertElementInst>(InsertInst)) + return cast<FixedVectorType>(IE->getType())->getNumElements(); + + unsigned AggregateSize = 1; + auto *IV = cast<InsertValueInst>(InsertInst); + Type *CurrentType = IV->getType(); + do { + if (auto *ST = dyn_cast<StructType>(CurrentType)) { + for (auto *Elt : ST->elements()) + if (Elt != ST->getElementType(0)) // check homogeneity + return std::nullopt; + AggregateSize *= ST->getNumElements(); + CurrentType = ST->getElementType(0); + } else if (auto *AT = dyn_cast<ArrayType>(CurrentType)) { + AggregateSize *= AT->getNumElements(); + CurrentType = AT->getElementType(); + } else if (auto *VT = dyn_cast<FixedVectorType>(CurrentType)) { + AggregateSize *= VT->getNumElements(); + return AggregateSize; + } else if (CurrentType->isSingleValueType()) { + return AggregateSize; + } else { + return std::nullopt; + } + } while (true); +} + +Instruction *getReductionInstr(const DominatorTree *DT, PHINode *P, + BasicBlock *ParentBB, LoopInfo *LI) { + // There are situations where the reduction value is not dominated by the + // reduction phi. Vectorizing such cases has been reported to cause + // miscompiles. See PR25787. + auto DominatedReduxValue = [&](Value *R) { + return isa<Instruction>(R) && + DT->dominates(P->getParent(), cast<Instruction>(R)->getParent()); + }; + + Instruction *Rdx = nullptr; + + // Return the incoming value if it comes from the same BB as the phi node. + if (P->getIncomingBlock(0) == ParentBB) { + Rdx = dyn_cast<Instruction>(P->getIncomingValue(0)); + } else if (P->getIncomingBlock(1) == ParentBB) { + Rdx = dyn_cast<Instruction>(P->getIncomingValue(1)); + } + + if (Rdx && DominatedReduxValue(Rdx)) + return Rdx; + + // Otherwise, check whether we have a loop latch to look at. + Loop *BBL = LI->getLoopFor(ParentBB); + if (!BBL) + return nullptr; + BasicBlock *BBLatch = BBL->getLoopLatch(); + if (!BBLatch) + return nullptr; + + // There is a loop latch, return the incoming value if it comes from + // that. This reduction pattern occasionally turns up. + if (P->getIncomingBlock(0) == BBLatch) { + Rdx = dyn_cast<Instruction>(P->getIncomingValue(0)); + } else if (P->getIncomingBlock(1) == BBLatch) { + Rdx = dyn_cast<Instruction>(P->getIncomingValue(1)); + } + + if (Rdx && DominatedReduxValue(Rdx)) + return Rdx; + + return nullptr; +} + } // namespace llvm::slpvectorizer diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPReductionUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPReductionUtils.h index 6e55bfdad861c8..e6022c234efd2e 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPReductionUtils.h +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPReductionUtils.h @@ -19,11 +19,16 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include <optional> + namespace llvm { +class BasicBlock; class DataLayout; +class DominatorTree; class FastMathFlags; class IRBuilderBase; class Instruction; +class LoopInfo; class PHINode; class Type; class Value; @@ -81,6 +86,16 @@ Value *tryEmitBoolReduxBitcastCmp(IRBuilderBase &Builder, RecurKind RdxKind, Value *Vec, const Value *Root, FastMathFlags FMF, TargetTransformInfo::TargetCostKind CostKind); +/// \returns the number of elements of the homogeneous aggregate built by +/// \p InsertInst (insertelement or insertvalue), or std::nullopt if it is not +/// a homogeneous aggregate. +std::optional<unsigned> getAggregateSize(Instruction *InsertInst); + +/// Try to get a reduction instruction from phi node \p P in block \p ParentBB, +/// considering incoming values from \p ParentBB or the containing loop latch. +/// \returns a candidate reduction value, or nullptr if none. +Instruction *getReductionInstr(const DominatorTree *DT, PHINode *P, + BasicBlock *ParentBB, LoopInfo *LI); } // namespace llvm::slpvectorizer _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
