Author: Florian Hahn Date: 2021-01-25T14:21:56Z New Revision: 3201274dea63abbac5467f3f992f0280cbf3b100
URL: https://github.com/llvm/llvm-project/commit/3201274dea63abbac5467f3f992f0280cbf3b100 DIFF: https://github.com/llvm/llvm-project/commit/3201274dea63abbac5467f3f992f0280cbf3b100.diff LOG: [VPlan] Handle scalarized values in VPTransformState. This patch adds plumbing to handle scalarized values directly in VPTransformState. Reviewed By: gilr Differential Revision: https://reviews.llvm.org/D92282 Added: Modified: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp llvm/lib/Transforms/Vectorize/VPlan.cpp llvm/lib/Transforms/Vectorize/VPlan.h Removed: ################################################################################ diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 9373d40458f4..ea0d7673edf6 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7718,9 +7718,15 @@ void LoopVectorizationPlanner::executePlan(InnerLoopVectorizer &ILV, assert(BestVF.hasValue() && "Vectorization Factor is missing"); - VPTransformState State{*BestVF, BestUF, LI, - DT, ILV.Builder, ILV.VectorLoopValueMap, - &ILV, CallbackILV}; + VPTransformState State{*BestVF, + BestUF, + OrigLoop, + LI, + DT, + ILV.Builder, + ILV.VectorLoopValueMap, + &ILV, + CallbackILV}; State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton(); State.TripCount = ILV.getOrCreateTripCount(nullptr); State.CanonicalIV = ILV.Induction; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 120562b6db5f..b26399e0ae58 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -216,6 +216,27 @@ VPBasicBlock::iterator VPBasicBlock::getFirstNonPhi() { return It; } +Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) { + if (!Def->getDef() && OrigLoop->isLoopInvariant(Def->getLiveInIRValue())) + return Def->getLiveInIRValue(); + + if (hasScalarValue(Def, Instance)) + return Data.PerPartScalars[Def][Instance.Part][Instance.Lane]; + + if (hasVectorValue(Def, Instance.Part)) { + assert(Data.PerPartOutput.count(Def)); + auto *VecPart = Data.PerPartOutput[Def][Instance.Part]; + if (!VecPart->getType()->isVectorTy()) { + assert(Instance.Lane == 0 && "cannot get lane > 0 for scalar"); + return VecPart; + } + // TODO: Cache created scalar values. + return Builder.CreateExtractElement(VecPart, + Builder.getInt32(Instance.Lane)); + } + return Callback.getOrCreateScalarValue(VPValue2Value[Def], Instance); +} + BasicBlock * VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) { // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 8d907dbc857d..2cce127cd4ce 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -246,12 +246,12 @@ struct VPCallback { /// VPTransformState holds information passed down when "executing" a VPlan, /// needed for generating the output IR. struct VPTransformState { - VPTransformState(ElementCount VF, unsigned UF, LoopInfo *LI, + VPTransformState(ElementCount VF, unsigned UF, Loop *OrigLoop, LoopInfo *LI, DominatorTree *DT, IRBuilder<> &Builder, VectorizerValueMap &ValueMap, InnerLoopVectorizer *ILV, VPCallback &Callback) - : VF(VF), UF(UF), Instance(), LI(LI), DT(DT), Builder(Builder), - ValueMap(ValueMap), ILV(ILV), Callback(Callback) {} + : VF(VF), UF(UF), Instance(), OrigLoop(OrigLoop), LI(LI), DT(DT), + Builder(Builder), ValueMap(ValueMap), ILV(ILV), Callback(Callback) {} /// The chosen Vectorization and Unroll Factors of the loop being vectorized. ElementCount VF; @@ -269,6 +269,9 @@ struct VPTransformState { typedef SmallVector<Value *, 2> PerPartValuesTy; DenseMap<VPValue *, PerPartValuesTy> PerPartOutput; + + using ScalarsPerPartValuesTy = SmallVector<SmallVector<Value *, 4>, 2>; + DenseMap<VPValue *, ScalarsPerPartValuesTy> PerPartScalars; } Data; /// Get the generated Value for a given VPValue and a given Part. Note that @@ -285,24 +288,21 @@ struct VPTransformState { } /// Get the generated Value for a given VPValue and given Part and Lane. - Value *get(VPValue *Def, const VPIteration &Instance) { - // If the Def is managed directly by VPTransformState, extract the lane from - // the relevant part. Note that currently only VPInstructions and external - // defs are managed by VPTransformState. Other Defs are still created by ILV - // and managed in its ValueMap. For those this method currently just - // delegates the call to ILV below. - if (Data.PerPartOutput.count(Def)) { - auto *VecPart = Data.PerPartOutput[Def][Instance.Part]; - if (!VecPart->getType()->isVectorTy()) { - assert(Instance.Lane == 0 && "cannot get lane > 0 for scalar"); - return VecPart; - } - // TODO: Cache created scalar values. - return Builder.CreateExtractElement(VecPart, - Builder.getInt32(Instance.Lane)); - } + Value *get(VPValue *Def, const VPIteration &Instance); - return Callback.getOrCreateScalarValue(VPValue2Value[Def], Instance); + bool hasVectorValue(VPValue *Def, unsigned Part) { + auto I = Data.PerPartOutput.find(Def); + return I != Data.PerPartOutput.end() && Part < I->second.size() && + I->second[Part]; + } + + bool hasScalarValue(VPValue *Def, VPIteration Instance) { + auto I = Data.PerPartScalars.find(Def); + if (I == Data.PerPartScalars.end()) + return false; + return Instance.Part < I->second.size() && + Instance.Lane < I->second[Instance.Part].size() && + I->second[Instance.Part][Instance.Lane]; } /// Set the generated Value for a given VPValue and a given Part. @@ -315,6 +315,17 @@ struct VPTransformState { } void set(VPValue *Def, Value *IRDef, Value *V, unsigned Part); + void set(VPValue *Def, Value *V, const VPIteration &Instance) { + auto Iter = Data.PerPartScalars.insert({Def, {}}); + auto &PerPartVec = Iter.first->second; + while (PerPartVec.size() <= Instance.Part) + PerPartVec.emplace_back(); + auto &Scalars = PerPartVec[Instance.Part]; + while (Scalars.size() <= Instance.Lane) + Scalars.push_back(nullptr); + Scalars[Instance.Lane] = V; + } + /// Hold state information used when constructing the CFG of the output IR, /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks. struct CFGState { @@ -340,6 +351,9 @@ struct VPTransformState { CFGState() = default; } CFG; + /// Hold a pointer to the original loop. + Loop *OrigLoop; + /// Hold a pointer to LoopInfo to register new basic blocks in the loop. LoopInfo *LI; _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
