Author: Vitaly Buka Date: 2026-10-30T18:32:13-07:00 New Revision: c24601a78f98b957c3e0e284e1fcda94c92b2689
URL: https://github.com/llvm/llvm-project/commit/c24601a78f98b957c3e0e284e1fcda94c92b2689 DIFF: https://github.com/llvm/llvm-project/commit/c24601a78f98b957c3e0e284e1fcda94c92b2689.diff LOG: Revert "[LV] Model memory runtime checks for inner loops of loop nests in VPl…" This reverts commit 0d387de3856273874ef65aa2e90cd07a7a9a2fd0. Added: Modified: llvm/include/llvm/Transforms/Utils/LoopUtils.h llvm/lib/Transforms/Utils/LoopUtils.cpp llvm/lib/Transforms/Vectorize/LoopVectorize.cpp llvm/lib/Transforms/Vectorize/VPlan.h llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp llvm/lib/Transforms/Vectorize/VPlanLowering.cpp llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp llvm/lib/Transforms/Vectorize/VPlanTransforms.h llvm/lib/Transforms/Vectorize/VPlanUtils.cpp llvm/lib/Transforms/Vectorize/VPlanUtils.h llvm/test/Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop-diff-checks.ll llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll Removed: ################################################################################ diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h index 5b539ee515ca78..ae51fcecc02583 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -661,21 +661,6 @@ LLVM_ABI void appendLoopsToWorklist(LoopInfo &, LLVM_ABI Loop *cloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM, LoopInfo *LI, LPPassManager *LPM); -/// SCEV bounds of a pointer group: the first accessed byte, the last accessed -/// byte plus one, and a stride that must be checked to be non-negative, if any. -struct RuntimeCheckBounds { - const SCEV *Low; - const SCEV *High; - const SCEV *StrideToCheck; -}; - -/// Return the bounds of \p CG in \p TheLoop. If \p HoistRuntimeChecks is true, -/// bounds varying in the parent loop of \p TheLoop may be widened to cover the -/// whole parent loop, so that checks using them can be hoisted out of it. -LLVM_ABI RuntimeCheckBounds -getRuntimeCheckBounds(const RuntimeCheckingPtrGroup *CG, const Loop *TheLoop, - ScalarEvolution &SE, bool HoistRuntimeChecks); - /// Add code that checks at runtime if the accessed arrays in \p PointerChecks /// overlap. Returns the final comparator value or NULL if no check is needed. LLVM_ABI Value * diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index 2968f4600fd9a7..c1a7362f77beca 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -2120,10 +2120,16 @@ struct PointerBounds { Value *StrideToCheck; }; -RuntimeCheckBounds -llvm::getRuntimeCheckBounds(const RuntimeCheckingPtrGroup *CG, - const Loop *TheLoop, ScalarEvolution &SE, - bool HoistRuntimeChecks) { +/// Expand code for the lower and upper bound of the pointer group \p CG +/// in \p TheLoop. \return the values for the bounds. +static PointerBounds expandBounds(const RuntimeCheckingPtrGroup *CG, + Loop *TheLoop, Instruction *Loc, + SCEVExpander &Exp, bool HoistRuntimeChecks) { + LLVMContext &Ctx = Loc->getContext(); + Type *PtrArithTy = PointerType::get(Ctx, CG->AddressSpace); + + Value *Start = nullptr, *End = nullptr; + LLVM_DEBUG(dbgs() << "LAA: Adding RT check for range:\n"); const SCEV *Low = CG->Low, *High = CG->High, *Stride = nullptr; // If the Low and High values are themselves loop-variant, then we may want @@ -2141,6 +2147,7 @@ llvm::getRuntimeCheckBounds(const RuntimeCheckingPtrGroup *CG, auto *HighAR = cast<SCEVAddRecExpr>(High); auto *LowAR = cast<SCEVAddRecExpr>(Low); const Loop *OuterLoop = TheLoop->getParentLoop(); + ScalarEvolution &SE = *Exp.getSE(); const SCEV *Recur = LowAR->getStepRecurrence(SE); if (Recur == HighAR->getStepRecurrence(SE) && HighAR->getLoop() == OuterLoop && LowAR->getLoop() == OuterLoop) { @@ -2168,21 +2175,6 @@ llvm::getRuntimeCheckBounds(const RuntimeCheckingPtrGroup *CG, } } } - return {Low, High, Stride}; -} - -/// Expand code for the lower and upper bound of the pointer group \p CG -/// in \p TheLoop. \return the values for the bounds. -static PointerBounds expandBounds(const RuntimeCheckingPtrGroup *CG, - Loop *TheLoop, Instruction *Loc, - SCEVExpander &Exp, bool HoistRuntimeChecks) { - LLVMContext &Ctx = Loc->getContext(); - Type *PtrArithTy = PointerType::get(Ctx, CG->AddressSpace); - - Value *Start = nullptr, *End = nullptr; - LLVM_DEBUG(dbgs() << "LAA: Adding RT check for range:\n"); - auto [Low, High, Stride] = - getRuntimeCheckBounds(CG, TheLoop, *Exp.getSE(), HoistRuntimeChecks); Start = Exp.expandCodeFor(Low, PtrArithTy, Loc); End = Exp.expandCodeFor(High, PtrArithTy, Loc); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index fce481ec049bfa..c17abe52fb1c36 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6737,11 +6737,17 @@ void LoopVectorizationPlanner::attachRuntimeChecks( "(e.g., adding 'restrict')."; }); } + // The VPlan expander cannot hoist bounds out of an enclosing loop. const auto &RtPtrChecking = *Legal->getRuntimePointerChecking(); + if (OrigLoop->getParentLoop()) + return RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan, + MemCheckCond, MemCheckBlock, HasBranchWeights); + // Erase the temporary IR before recipe expansion can reuse its values. RTChecks.eraseMemCheckBlock(); RUN_VPLAN_PASS(VPlanTransforms::attachMemoryChecks, Plan, RtPtrChecking, - OrigLoop, *PSE.getSE(), VF, UF, HasBranchWeights); + *PSE.getSE(), VF, UF, OrigLoop->getStartLoc(), + HasBranchWeights); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 7764ec9b45a947..279f93b2fc90fa 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -4090,18 +4090,12 @@ struct LLVM_ABI_FOR_TEST VPWidenStoreEVLRecipe final class VPExpandSCEVRecipe : public VPSingleDefRecipe { const SCEV *Expr; - /// Whether \p Expr must be expanded in IR, which hoists it out of the loops - /// it is invariant in, instead of as VPInstructions. - bool ExpandInIR; - public: - VPExpandSCEVRecipe(const SCEV *Expr, bool ExpandInIR = false); + VPExpandSCEVRecipe(const SCEV *Expr); ~VPExpandSCEVRecipe() override = default; - VPExpandSCEVRecipe *clone() override { - return new VPExpandSCEVRecipe(Expr, ExpandInIR); - } + VPExpandSCEVRecipe *clone() override { return new VPExpandSCEVRecipe(Expr); } VP_CLASSOF_IMPL(VPRecipeBase::VPExpandSCEVSC) @@ -4118,8 +4112,6 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe { const SCEV *getSCEV() const { return Expr; } - bool expandInIR() const { return ExpandInIR; } - protected: #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the recipe. diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 6daa320993d661..a62f1ddae9769b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -1583,53 +1583,39 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond, attachVPCheckBlock(Plan, CondVPV, CheckBlockVPBB, AddBranchWeights); } -/// Generate recipes for the memory runtime checks \p Checks of \p TheLoop -/// using \p Builder and return the condition that is true if any check fails. +/// Generate recipes for the memory runtime checks \p Checks using \p Builder +/// and return the condition that is true if any check fails. static VPValue *createRuntimeChecks(VPlan &Plan, VPBuilder &Builder, VPSCEVExpander &Expander, ArrayRef<RuntimePointerCheck> Checks, - const Loop *TheLoop, ScalarEvolution &SE, DebugLoc DL) { // Expand each group's bounds once so all checks reuse the same frozen values. SmallDenseMap<const RuntimeCheckingPtrGroup *, - std::tuple<VPValue *, VPValue *, const SCEV *>, 8> + std::pair<VPValue *, VPValue *>, 8> GroupToBounds; - for (const auto &[A, B] : Checks) { + for (const auto &[A, B] : Checks) for (const RuntimeCheckingPtrGroup *CG : {A, B}) { - if (GroupToBounds.contains(CG)) + auto &[Start, End] = GroupToBounds[CG]; + if (Start) continue; - auto [Low, High, Stride] = getRuntimeCheckBounds( - CG, TheLoop, SE, VectorizerParams::HoistRuntimeChecks); - VPValue *Start = Expander.expand(Low); - VPValue *End = Expander.expand(High); + Start = Expander.expand(CG->Low); + End = Expander.expand(CG->High); if (CG->NeedsFreeze) { Start = Builder.createFreeze(Start, DL); End = Builder.createFreeze(End, DL); } - GroupToBounds[CG] = {Start, End, Stride}; } - } VPValue *Cond = Plan.getFalse(); - SmallPtrSet<const SCEV *, 4> CheckedStrides; for (const auto &[A, B] : Checks) { - auto [AStart, AEnd, AStride] = GroupToBounds[A]; - auto [BStart, BEnd, BStride] = GroupToBounds[B]; + auto [AStart, AEnd] = GroupToBounds[A]; + auto [BStart, BEnd] = GroupToBounds[B]; VPValue *Bound0 = Builder.createICmp(CmpInst::ICMP_ULT, AStart, BEnd, DL, "bound0"); VPValue *Bound1 = Builder.createICmp(CmpInst::ICMP_ULT, BStart, AEnd, DL, "bound1"); VPValue *IsConflict = Builder.createAnd(Bound0, Bound1, DL, "found.conflict"); - for (const SCEV *Stride : {AStride, BStride}) { - if (!Stride || !CheckedStrides.insert(Stride).second) - continue; - VPValue *StrideV = Expander.expand(Stride); - VPValue *IsNegative = Builder.createICmp( - CmpInst::ICMP_SLT, StrideV, Plan.getZero(StrideV->getScalarType()), - DL, "stride.check"); - IsConflict = Builder.createOr(IsConflict, IsNegative, DL); - } Cond = Builder.createOr(Cond, IsConflict, DL, "conflict.rdx"); } return Cond; @@ -1678,21 +1664,19 @@ static VPValue *createDiffRuntimeChecks(VPlan &Plan, VPBuilder &Builder, void VPlanTransforms::attachMemoryChecks( VPlan &Plan, const RuntimePointerChecking &RtPtrChecking, - const Loop *TheLoop, ScalarEvolution &SE, ElementCount VF, unsigned UF, + ScalarEvolution &SE, ElementCount VF, unsigned UF, DebugLoc DL, bool AddBranchWeights) { assert(!RtPtrChecking.getChecks().empty() && "No checks to generate"); - DebugLoc DL = TheLoop->getStartLoc(); auto *MemCheckVPBB = Plan.createVPBasicBlock("vector.memcheck"); VPBuilder Builder(MemCheckVPBB); - VPSCEVExpander Expander(Builder, SE, DL, TheLoop->getParentLoop()); + VPSCEVExpander Expander(Builder, SE, DL); auto DiffChecks = RtPtrChecking.getDiffChecks(); - VPValue *Cond = - DiffChecks - ? createDiffRuntimeChecks(Plan, Builder, Expander, *DiffChecks, SE, - VF, UF, DL) - : createRuntimeChecks(Plan, Builder, Expander, - RtPtrChecking.getChecks(), TheLoop, SE, DL); + VPValue *Cond = DiffChecks + ? createDiffRuntimeChecks(Plan, Builder, Expander, + *DiffChecks, SE, VF, UF, DL) + : createRuntimeChecks(Plan, Builder, Expander, + RtPtrChecking.getChecks(), DL); attachVPCheckBlock(Plan, Cond, MemCheckVPBB, AddBranchWeights); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp index e26f9b2ba0c27d..5335f00013869f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp @@ -1063,9 +1063,7 @@ void VPlanTransforms::expandSCEVsToVPInstructions(VPlan &Plan, if (ExpSCEV.user_empty()) continue; Builder.setInsertPoint(&ExpSCEV); - VPValue *Expanded = Expander.expand(ExpSCEV); - if (Expanded == &ExpSCEV) - continue; + VPValue *Expanded = Expander.expand(ExpSCEV.getSCEV()); ExpSCEV.replaceAllUsesWith(Expanded); // TripCount should not be used after expansion to VPInstructions. Reset to // poison to avoid dangling references. diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 957990ceb90256..5d9fb8f21d3be1 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -471,9 +471,9 @@ void VPRecipeBase::print(raw_ostream &O, const Twine &Indent, } #endif -VPExpandSCEVRecipe::VPExpandSCEVRecipe(const SCEV *Expr, bool ExpandInIR) +VPExpandSCEVRecipe::VPExpandSCEVRecipe(const SCEV *Expr) : VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, {}, Expr->getType()), - Expr(Expr), ExpandInIR(ExpandInIR) {} + Expr(Expr) {} static SmallVector<unsigned> getExtractValueIndices(ArrayRef<VPValue *> Ops) { return map_to_vector(drop_begin(Ops), [](VPValue *Op) { @@ -5205,8 +5205,6 @@ void VPExpandSCEVRecipe::printRecipe(raw_ostream &O, const Twine &Indent, O << Indent << "EMIT "; printAsOperand(O, SlotTracker); O << " = EXPAND SCEV " << *Expr; - if (ExpandInIR) - O << " (in IR)"; } #endif diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 3497d07d29ec9b..854d5c7dd4c249 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1274,11 +1274,6 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) { VPValue *A, *B; - // icmp Pred X, X -> true if Pred is true when equal, false otherwise. - CmpPredicate Pred; - if (match(Def, m_ICmp(Pred, m_VPValue(A), m_Deferred(A)))) - return ICmpInst::isTrueWhenEqual(Pred) ? Plan.getTrue() : Plan.getFalse(); - if (match(Def, m_c_Add(m_VPValue(A), m_ZeroInt()))) return A; @@ -2399,13 +2394,9 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> { if (auto *LL = dyn_cast<VPWidenMemoryRecipe>(L)) if (LL->isConsecutive() != cast<VPWidenMemoryRecipe>(R)->isConsecutive()) return false; - if (auto *LExp = dyn_cast<VPExpandSCEVRecipe>(L)) { - auto *RExp = cast<VPExpandSCEVRecipe>(R); - if (LExp->getSCEV() != RExp->getSCEV()) + if (auto *LExp = dyn_cast<VPExpandSCEVRecipe>(L)) + if (LExp->getSCEV() != cast<VPExpandSCEVRecipe>(R)->getSCEV()) return false; - assert(LExp->expandInIR() == RExp->expandInIR() && - "cannot CSE recipes expanded in IR and as VPInstructions"); - } // Phi recipes can only be equal if they are in the same VPBB, as they // implicitly depend on their predecessors. if (isa<VPWidenPHIRecipe>(L) && L->getParent() != R->getParent()) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 5f7c26897bd076..3d6bda8641f6ea 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -214,13 +214,13 @@ struct VPlanTransforms { static void attachCheckBlock(VPlan &Plan, Value *Cond, BasicBlock *CheckBlock, bool AddBranchWeights); - /// Generate the memory runtime checks of \p RtPtrChecking for \p TheLoop as - /// recipes and attach the check block to \p Plan. \p VF and \p UF are used - /// to compute the thresholds of diff erence checks. + /// Generate the memory runtime checks of \p RtPtrChecking as recipes and + /// attach the check block to \p Plan. \p VF and \p UF are used to compute + /// the thresholds of diff erence checks. static void attachMemoryChecks(VPlan &Plan, const RuntimePointerChecking &RtPtrChecking, - const Loop *TheLoop, ScalarEvolution &SE, - ElementCount VF, unsigned UF, + ScalarEvolution &SE, ElementCount VF, + unsigned UF, DebugLoc DL, bool AddBranchWeights); /// Model the blocks the executed \p MainPlan generated for the main vector diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp index 5a6d0a3d0e2823..52c70621ae1db0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp @@ -80,8 +80,7 @@ bool vputils::onlyScalarValuesUsed(const VPValue *Def) { [Def](const VPUser *U) { return U->usesScalars(Def); }); } -VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, - bool ExpandInIR) { +VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) { if (auto *E = dyn_cast<SCEVConstant>(Expr)) return Plan.getOrAddLiveIn(E->getValue()); // Skip SCEV expansion if Expr is a SCEVUnknown wrapping a non-instruction @@ -91,7 +90,7 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, auto *U = dyn_cast<SCEVUnknown>(Expr); if (U && !isa<Instruction>(U->getValue())) return Plan.getOrAddLiveIn(U->getValue()); - auto *Expanded = new VPExpandSCEVRecipe(Expr, ExpandInIR); + auto *Expanded = new VPExpandSCEVRecipe(Expr); VPBasicBlock *EntryVPBB = Plan.getEntry(); auto Iter = EntryVPBB->getFirstNonPhi(); while (Iter != EntryVPBB->end() && isa<VPIRInstruction>(*Iter)) @@ -987,11 +986,6 @@ VPValue *VPSCEVExpander::expandImpl(const SCEV *S) { if (VPValue *V = tryToReuseIRValue(S)) return V; - if (HoistOutOf && !SafeUDivMode && SE.isLoopInvariant(S, HoistOutOf) && - SE.isGuaranteedNotToCauseUB(S)) - return vputils::getOrCreateVPValueForSCEVExpr(Builder.getPlan(), S, - /*ExpandInIR=*/true); - switch (S->getSCEVType()) { case scConstant: return Builder.getPlan().getOrAddLiveIn(cast<SCEVConstant>(S)->getValue()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h index 94676d87434010..b66dad723af5d0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h @@ -43,9 +43,8 @@ bool onlyScalarValuesUsed(const VPValue *Def); /// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in /// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p Plan's /// pre-header already contains a recipe expanding \p Expr, return it. If not, -/// create a new one. \p ExpandInIR is forwarded to the VPExpandSCEVRecipe. -VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, - bool ExpandInIR = false); +/// create a new one. +VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr); /// Return the SCEV expression for \p V. Returns SCEVCouldNotCompute if no /// SCEV expression could be constructed. @@ -282,10 +281,6 @@ class VPSCEVExpander { /// zero, matching SCEVExpander's SafeUDivMode. bool SafeUDivMode = false; - /// If set, expressions invariant in this loop are expanded in IR via - /// VPExpandSCEVRecipes, which hoists them out of it. - const Loop *HoistOutOf; - DenseMap<PointerIntPair<const SCEV *, 1, bool>, VPValue *> InsertedExpressions; @@ -300,18 +295,11 @@ class VPSCEVExpander { VPValue *expandImpl(const SCEV *S); public: - VPSCEVExpander(VPBuilder &Builder, ScalarEvolution &SE, DebugLoc DL, - const Loop *HoistOutOf = nullptr) - : Builder(Builder), SE(SE), DL(DL), HoistOutOf(HoistOutOf) {} + VPSCEVExpander(VPBuilder &Builder, ScalarEvolution &SE, DebugLoc DL) + : Builder(Builder), SE(SE), DL(DL) {} /// Expand \p S into recipes and live-ins using the builder. VPValue *expand(const SCEV *S); - - /// Expand the expression of \p R, or return \p R if it must be expanded in - /// IR. - VPValue *expand(VPExpandSCEVRecipe &R) { - return R.expandInIR() ? &R : expand(R.getSCEV()); - } }; //===----------------------------------------------------------------------===// // Utilities for modifying predecessors and successors of VPlan blocks. diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll index 7998f85cd551d7..3f0a689a76dff7 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll @@ -37,7 +37,7 @@ while.body: %i = phi i64 [ 1023, %entry ], [ %i.next, %if.end ] %arrayidx = getelementptr inbounds i32, ptr %data1, i64 %i %ld = load i32, ptr %arrayidx, align 4 - %cmp = icmp sgt i32 %ld, 0 + %cmp = icmp sgt i32 %ld, %ld br i1 %cmp, label %if.then, label %if.end if.then: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop- diff -checks.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop- diff -checks.ll index 255126a88dacc5..050547a89f29c5 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop- diff -checks.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop- diff -checks.ll @@ -10,10 +10,10 @@ define void @foo(ptr %dst, ptr %src, i32 %m, i32 %n) vscale_range(1,16) { ; CHECK-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64 ; CHECK-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64 ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64 -; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[DST1]], [[SRC2]] ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP0]], 5 ; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[TMP11]], -1 +; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[DST1]], [[SRC2]] ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i32 [ [[OUTER_IV_NEXT:%.*]], %[[INNER_EXIT:.*]] ], [ 0, %[[ENTRY]] ] diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll index d00e93f3f34eee..2c2ef39597e648 100644 --- a/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll +++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll @@ -35,15 +35,15 @@ define void @ diff _checks(ptr nocapture noundef writeonly %dst, ptr nocapture nou ; CHECK-SAME: ptr noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[ADD5:%.*]] = add nsw i32 [[N]], 1 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[M]], -1 ; CHECK-NEXT: [[TMP1:%.*]] = shl i32 [[N]], 2 ; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 4 -; CHECK-NEXT: [[TMP5:%.*]] = mul i32 [[N]], [[M]] -; CHECK-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], 2 -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i32 [[TMP6]] -; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[M]], -1 ; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[TMP0]], [[TMP2]] ; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], [[TMP1]] ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i32 [[TMP4]] +; CHECK-NEXT: [[TMP5:%.*]] = mul i32 [[N]], [[M]] +; CHECK-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], 2 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i32 [[TMP6]] ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[I_023_US:%.*]] = phi i32 [ [[INC10_US:%.*]], %[[INNER_LOOP_EXIT:.*]] ], [ 0, %[[ENTRY]] ] diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll index e3e55d10f8b437..1e65b4ff539662 100644 --- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll +++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll @@ -351,6 +351,8 @@ define void @multiple_uniform_stores(ptr nocapture %var1, ptr nocapture readonly ; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2 ; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 4 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[VAR1:%.*]], i64 [[TMP3]] +; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[VAR2:%.*]], i64 4 +; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[ITR]], -1 ; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[ITR]], -1 ; CHECK-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] ; CHECK: for.cond1.preheader: @@ -369,11 +371,12 @@ define void @multiple_uniform_stores(ptr nocapture %var1, ptr nocapture readonly ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP4]], 2 -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[VAR2:%.*]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP7]], 2 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[VAR2]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = sub i32 [[TMP5]], [[J_022]] +; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP11]] to i64 +; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 2 ; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP9]], [[TMP13]] -; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[TMP14]], 4 -; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[VAR2]], i64 [[TMP25]] +; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SCEVGEP2]], i64 [[TMP14]] ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[VAR1]], [[SCEVGEP3]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] diff --git a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll index 3a898474d59637..7687357aa8199b 100644 --- a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll +++ b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll @@ -31,23 +31,23 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" define void @Test(ptr nocapture %obj, i64 %z) #0 { ; CHECK-LABEL: @Test( ; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[Z:%.*]], 2 -; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ:%.*]], i64 [[TMP1]] ; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 256 +; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ:%.*]], i64 [[TMP1]] ; CHECK-NEXT: br label [[DOTOUTER_PREHEADER:%.*]] ; CHECK: .outer.preheader: ; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[I_NEXT:%.*]], [[DOTOUTER:%.*]] ] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[OBJ]], i64 0, i32 1, i64 [[I]] -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[Z]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] -; CHECK: vector.memcheck: ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[I]], 7 -; CHECK-NEXT: [[TMP4:%.*]] = add nuw i64 256, [[TMP3]] +; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], 256 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[OBJ]], i64 [[TMP4]] ; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP2]], [[TMP3]] ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP5]] ; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[I]], 2 -; CHECK-NEXT: [[TMP8:%.*]] = add i64 132, [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP6]], 132 ; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[OBJ]], i64 0, i32 1, i64 [[I]] +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[Z]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] +; CHECK: vector.memcheck: ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[OBJ]], [[SCEVGEP1]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] @@ -102,11 +102,12 @@ define void @Test(ptr nocapture %obj, i64 %z) #0 { ; CHECK-NEXT: br i1 [[EXITCOND_INNER]], label [[DOTOUTER]], label [[DOTINNER]], !llvm.loop [[LOOP11:![0-9]+]] ; ; CHECK-HOIST-LABEL: @Test( +; CHECK-HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[OBJ:%.*]], i64 256 ; CHECK-HOIST-NEXT: [[TMP1:%.*]] = shl i64 [[Z:%.*]], 2 -; CHECK-HOIST-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ:%.*]], i64 [[TMP1]] ; CHECK-HOIST-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 4224 ; CHECK-HOIST-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP2]] -; CHECK-HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[OBJ]], i64 256 +; CHECK-HOIST-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP1]] +; CHECK-HOIST-NEXT: [[SCEVGEP3:%.*]] = getelementptr nuw i8, ptr [[OBJ]], i64 128 ; CHECK-HOIST-NEXT: br label [[DOTOUTER_PREHEADER:%.*]] ; CHECK-HOIST: .outer.preheader: ; CHECK-HOIST-NEXT: [[I:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[I_NEXT:%.*]], [[DOTOUTER:%.*]] ] @@ -114,10 +115,11 @@ define void @Test(ptr nocapture %obj, i64 %z) #0 { ; CHECK-HOIST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[Z]], 4 ; CHECK-HOIST-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK-HOIST: vector.memcheck: -; CHECK-HOIST-NEXT: [[BOUND08:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] -; CHECK-HOIST-NEXT: [[BOUND19:%.*]] = icmp ult ptr [[OBJ]], [[SCEVGEP1]] -; CHECK-HOIST-NEXT: [[FOUND_CONFLICT10:%.*]] = and i1 [[BOUND08]], [[BOUND19]] -; CHECK-HOIST-NEXT: br i1 [[FOUND_CONFLICT10]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] +; CHECK-HOIST-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] +; CHECK-HOIST-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[OBJ]], [[SCEVGEP1]] +; CHECK-HOIST-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; CHECK-HOIST-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SCEVGEP3]], [[SCEVGEP1]] +; CHECK-HOIST-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] ; CHECK-HOIST: vector.ph: ; CHECK-HOIST-NEXT: [[N_MOD_VF:%.*]] = and i64 [[Z]], 3 ; CHECK-HOIST-NEXT: [[N_VEC:%.*]] = sub i64 [[Z]], [[N_MOD_VF]] diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks- diff erence-rmw-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks- diff erence-rmw-hoist.ll index 5ca0e8c289af10..0c829e26090be0 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-checks- diff erence-rmw-hoist.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-checks- diff erence-rmw-hoist.ll @@ -7,9 +7,9 @@ define void @rmw_variant_distance(ptr %dst, ptr %src, i64 %rows) { ; HOIST-LABEL: define void @rmw_variant_distance( ; HOIST-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[ROWS:%.*]]) { ; HOIST-NEXT: [[ENTRY:.*]]: -; HOIST-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 64 ; HOIST-NEXT: [[TMP0:%.*]] = shl i64 [[ROWS]], 6 ; HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]] +; HOIST-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 64 ; HOIST-NEXT: br label %[[OUTER_HEADER:.*]] ; HOIST: [[OUTER_HEADER]]: ; HOIST-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ] @@ -30,13 +30,13 @@ define void @rmw_variant_distance(ptr %dst, ptr %src, i64 %rows) { ; NO-HOIST-NEXT: br label %[[OUTER_HEADER:.*]] ; NO-HOIST: [[OUTER_HEADER]]: ; NO-HOIST-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ] +; NO-HOIST-NEXT: [[TMP1:%.*]] = shl i64 [[OUTER_IV]], 6 +; NO-HOIST-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 64 +; NO-HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]] ; NO-HOIST-NEXT: [[OFFSET:%.*]] = mul i64 [[OUTER_IV]], 16 ; NO-HOIST-NEXT: [[DST_BASE:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[OFFSET]] ; NO-HOIST-NEXT: br label %[[VECTOR_MEMCHECK:.*]] ; NO-HOIST: [[VECTOR_MEMCHECK]]: -; NO-HOIST-NEXT: [[TMP0:%.*]] = shl i64 [[OUTER_IV]], 6 -; NO-HOIST-NEXT: [[TMP1:%.*]] = add i64 64, [[TMP0]] -; NO-HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]] ; NO-HOIST-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST_BASE]], [[SCEVGEP1]] ; NO-HOIST-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]] ; NO-HOIST-NEXT: [[DIFF_CHECK:%.*]] = and i1 [[BOUND0]], [[BOUND1]] @@ -79,8 +79,8 @@ define void @rmw_invariant_distance(ptr %dst, ptr %src, i64 %rows) { ; HOIST-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[ROWS:%.*]]) { ; HOIST-NEXT: [[ENTRY:.*]]: ; HOIST-NEXT: [[TMP0:%.*]] = shl i64 [[ROWS]], 6 -; HOIST-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]] ; HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]] +; HOIST-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]] ; HOIST-NEXT: br label %[[OUTER_HEADER:.*]] ; HOIST: [[OUTER_HEADER]]: ; HOIST-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ] @@ -101,15 +101,15 @@ define void @rmw_invariant_distance(ptr %dst, ptr %src, i64 %rows) { ; NO-HOIST-NEXT: br label %[[OUTER_HEADER:.*]] ; NO-HOIST: [[OUTER_HEADER]]: ; NO-HOIST-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ] +; NO-HOIST-NEXT: [[TMP0:%.*]] = shl i64 [[OUTER_IV]], 6 +; NO-HOIST-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 64 +; NO-HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]] +; NO-HOIST-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]] ; NO-HOIST-NEXT: [[OFFSET:%.*]] = mul i64 [[OUTER_IV]], 16 ; NO-HOIST-NEXT: [[DST_BASE:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[OFFSET]] ; NO-HOIST-NEXT: [[SRC_BASE:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[OFFSET]] ; NO-HOIST-NEXT: br label %[[VECTOR_MEMCHECK:.*]] ; NO-HOIST: [[VECTOR_MEMCHECK]]: -; NO-HOIST-NEXT: [[TMP0:%.*]] = shl i64 [[OUTER_IV]], 6 -; NO-HOIST-NEXT: [[TMP1:%.*]] = add i64 64, [[TMP0]] -; NO-HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]] -; NO-HOIST-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]] ; NO-HOIST-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST_BASE]], [[SCEVGEP1]] ; NO-HOIST-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC_BASE]], [[SCEVGEP]] ; NO-HOIST-NEXT: [[DIFF_CHECK:%.*]] = and i1 [[BOUND0]], [[BOUND1]] diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks- diff erence.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks- diff erence.ll index c8f81b7b7cc712..89c1585382a553 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-checks- diff erence.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-checks- diff erence.ll @@ -326,13 +326,13 @@ define void @nested_loop_outer_iv_addrec_invariant_in_inner1(ptr %a, ptr %b, i64 ; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] ; CHECK: [[OUTER_HEADER]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[OUTER_IV]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 4 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]] ; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OUTER_IV]] ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[OUTER_IV]], 2 -; CHECK-NEXT: [[TMP2:%.*]] = add i64 4, [[TMP1]] -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]] ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[GEP_A]], [[SCEVGEP2]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] @@ -378,13 +378,13 @@ define void @nested_loop_outer_iv_addrec_invariant_in_inner2(ptr %a, ptr %b, i64 ; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] ; CHECK: [[OUTER_HEADER]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[OUTER_IV]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 4 +; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]] ; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OUTER_IV]] ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[OUTER_IV]], 2 -; CHECK-NEXT: [[TMP2:%.*]] = add i64 4, [[TMP1]] -; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]] ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[GEP_A]], [[SCEVGEP]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] @@ -433,8 +433,16 @@ define void @nested_loop_bounds_are_scaled_outer_iv(ptr %a, ptr %b, i32 %n, i32 ; CHECK-NEXT: [[N_EXT:%.*]] = zext nneg i32 [[N]] to i64 ; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] ; CHECK: [[OUTER_HEADER]]: -; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[OUTER_LATCH:%.*]] ], [ 0, %[[OUTER_PH]] ] +; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[OUTER_LATCH:%.*]] ], [ 0, %[[OUTER_PH]] ] ; CHECK-NEXT: [[OUTER_IV1:%.*]] = phi i64 [ 1, %[[OUTER_PH]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH]] ] +; CHECK-NEXT: [[TMP0:%.*]] = shl nuw nsw i64 [[OUTER_IV]], 2 +; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP0]], 4 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[OUTER_IV]], 3 +; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP1]], 8 +; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]] +; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]] +; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]] ; CHECK-NEXT: [[OUTER_OFF_IS:%.*]] = mul nsw i64 [[OUTER_IV1]], [[IS_EXT]] ; CHECK-NEXT: [[OUTER_OFF_JS:%.*]] = mul nsw i64 [[OUTER_IV1]], [[JS_EXT]] ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[OUTER_IV1]], 4 @@ -445,14 +453,6 @@ define void @nested_loop_bounds_are_scaled_outer_iv(ptr %a, ptr %b, i32 %n, i32 ; CHECK-NEXT: [[TMP2:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]] ; CHECK-NEXT: br i1 [[TMP2]], [[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[INDVAR]], 2 -; CHECK-NEXT: [[TMP3:%.*]] = add nuw i64 4, [[TMP1]] -; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]] -; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[INDVAR]], 3 -; CHECK-NEXT: [[TMP5:%.*]] = add i64 8, [[TMP4]] -; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] -; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]] -; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]] ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP4]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP3]], [[SCEVGEP2]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] @@ -518,8 +518,16 @@ define void @nested_loop_only_one_bound_is_scaled_outer_iv(ptr %a, ptr %b, i32 % ; CHECK-NEXT: [[N_EXT:%.*]] = zext nneg i32 [[N]] to i64 ; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] ; CHECK: [[OUTER_HEADER]]: -; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[OUTER_LATCH:%.*]] ], [ 0, %[[OUTER_PH]] ] +; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[OUTER_LATCH:%.*]] ], [ 0, %[[OUTER_PH]] ] ; CHECK-NEXT: [[OUTER_IV1:%.*]] = phi i64 [ 1, %[[OUTER_PH]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH]] ] +; CHECK-NEXT: [[TMP0:%.*]] = mul nuw nsw i64 [[OUTER_IV]], 12 +; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP0]], 12 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i64 [[OUTER_IV]], 24 +; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 16 +; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]] +; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]] +; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]] ; CHECK-NEXT: [[OUTER_OFF_IS:%.*]] = mul nsw i64 [[OUTER_IV1]], [[IS_EXT]] ; CHECK-NEXT: [[OUTER_OFF_JS:%.*]] = mul nsw i64 [[OUTER_IV1]], [[JS_EXT]] ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[OUTER_IV1]], 4 @@ -530,14 +538,6 @@ define void @nested_loop_only_one_bound_is_scaled_outer_iv(ptr %a, ptr %b, i32 % ; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]] ; CHECK-NEXT: br i1 [[TMP3]], [[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i64 [[INDVAR]], 12 -; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 12, [[TMP1]] -; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]] -; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[INDVAR]], 24 -; CHECK-NEXT: [[TMP5:%.*]] = add i64 16, [[TMP4]] -; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] -; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]] -; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]] ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP4]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP3]], [[SCEVGEP2]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] @@ -596,22 +596,22 @@ define void @nested_loop_bound_needs_constant_correction(ptr %a, ptr %b, i64 %n) ; CHECK-LABEL: define void @nested_loop_bound_needs_constant_correction( ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]] ; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 3 ; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]] -; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N]], 2 -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]] ; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] ; CHECK: [[OUTER_HEADER]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ] -; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[N]], [[OUTER_IV]] -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]] -; CHECK: [[VECTOR_MEMCHECK]]: ; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[OUTER_IV]], 2 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]] ; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[OUTER_IV]], 3 -; CHECK-NEXT: [[TMP6:%.*]] = add i64 4, [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], 4 ; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[N]], [[OUTER_IV]] +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]] +; CHECK: [[VECTOR_MEMCHECK]]: ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll index e539bfb7f6c143..492c161868fbdc 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll @@ -37,16 +37,16 @@ define void @ diff _checks(ptr nocapture noundef writeonly %dst, ptr nocapture nou ; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[ADD5]] to i64 ; CHECK-NEXT: [[WIDE_M:%.*]] = zext i32 [[M]] to i64 ; CHECK-NEXT: [[WIDE_N:%.*]] = zext i32 [[N]] to i64 -; CHECK-NEXT: [[TMP7:%.*]] = shl nsw i64 [[TMP1]], 2 -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]] -; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 2 -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP9]] ; CHECK-NEXT: [[TMP2:%.*]] = add nsw i64 [[WIDE_M]], -1 ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], [[TMP1]] ; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 2 ; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2 ; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP4]], [[TMP5]] ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = shl nsw i64 [[TMP1]], 2 +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]] +; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 2 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP9]] ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_OUTER_NEXT:%.*]], %[[INNER_EXIT:.*]] ] @@ -166,8 +166,8 @@ define void @full_checks(ptr nocapture noundef %dst, ptr nocapture noundef reado ; CHECK-NEXT: [[WIDE_N:%.*]] = zext i32 [[N]] to i64 ; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]] ; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[TMP1]], 2 -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]] ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]] ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_EXIT:.*]] ] @@ -284,15 +284,15 @@ define void @full_checks_ diff _strides(ptr nocapture noundef %dst, ptr nocapture ; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[WIDE_M:%.*]] = zext i32 [[M]] to i64 ; CHECK-NEXT: [[WIDE_N:%.*]] = zext i32 [[N]] to i64 -; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]] -; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]] ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[WIDE_M]], -1 ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2 ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 4 ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP0]], [[TMP2]] ; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[TMP1]] ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]] +; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]] ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_EXIT:.*]] ] @@ -409,17 +409,17 @@ define void @ diff _checks_src_start_invariant(ptr nocapture noundef writeonly %ds ; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64 ; CHECK-NEXT: [[WIDE_M:%.*]] = zext i32 [[M]] to i64 ; CHECK-NEXT: [[WIDE_N:%.*]] = zext i32 [[N]] to i64 -; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2 ; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[DST1]], [[SRC2]] +; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[WIDE_N]], 2 ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_OUTER_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ] +; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], [[IV_OUTER]] +; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], [[TMP3]] ; CHECK-NEXT: [[TMP5:%.*]] = mul nsw i64 [[IV_OUTER]], [[TMP0]] ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[IV_OUTER]], [[TMP2]] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], [[TMP13]] ; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[TMP4]], 1 ; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], 15 ; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] @@ -516,11 +516,11 @@ define void @full_checks_src_start_invariant(ptr nocapture noundef %dst, ptr noc ; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64 ; CHECK-NEXT: [[WIDE_M:%.*]] = zext i32 [[M]] to i64 ; CHECK-NEXT: [[WIDE_N:%.*]] = zext i32 [[N]] to i64 -; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2 -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]] ; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]] ; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[TMP1]], 2 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]] +; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]] ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_OUTER_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ] @@ -640,22 +640,24 @@ define void @triple_nested_loop_mixed_access(ptr nocapture noundef %dst, ptr noc ; CHECK-NEXT: [[WIDE_TRIP_COUNT68:%.*]] = zext i32 [[M]] to i64 ; CHECK-NEXT: [[WIDE_TRIP_COUNT60:%.*]] = zext i32 [[N]] to i64 ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[O]] to i64 -; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2 +; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP1]], [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 2 ; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[WIDE_TRIP_COUNT]], [[TMP1]] ; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP1]], [[TMP2]] +; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[WIDE_TRIP_COUNT]], [[TMP1]] ; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP7]], 2 +; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2 ; CHECK-NEXT: br label %[[OUTER_OUTER_LOOP:.*]] ; CHECK: [[OUTER_OUTER_LOOP]]: ; CHECK-NEXT: [[OUTER_OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_OUTER_IV_NEXT:%.*]], %[[OUTER_LOOP_END:.*]] ] -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP6]], [[OUTER_OUTER_IV]] -; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP3]], [[TMP13]] +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP4]], [[OUTER_OUTER_IV]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP6]], [[TMP10]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]] +; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP8]], [[OUTER_OUTER_IV]] ; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP12]] +; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP9]], [[TMP12]] ; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]] -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP8]], [[OUTER_OUTER_IV]] -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP6]], [[TMP10]] -; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]] -; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP14:%.*]] = mul nsw i64 [[OUTER_OUTER_IV]], [[TMP1]] ; CHECK-NEXT: [[TMP15:%.*]] = mul nsw i64 [[TMP14]], [[TMP0]] ; CHECK-NEXT: [[TMP16:%.*]] = mul nsw i64 [[TMP14]], [[TMP2]] @@ -667,8 +669,8 @@ define void @triple_nested_loop_mixed_access(ptr nocapture noundef %dst, ptr noc ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP6]], [[SCEVGEP2]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP3]], [[SCEVGEP5]] +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: @@ -803,24 +805,25 @@ define void @uncomputable_outer_tc(ptr nocapture noundef %dst, ptr nocapture nou ; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[N]] to i64 ; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[ADD6]] to i64 ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64 +; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2 -; CHECK-NEXT: [[OUTER_IV:%.*]] = shl nuw nsw i64 [[TMP2]], 2 +; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[WIDE_TRIP_COUNT]], 2 ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: -; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PREHEADER]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ] -; CHECK-NEXT: [[TMP10:%.*]] = mul nsw i64 [[TMP3]], [[TMP1]] -; CHECK-NEXT: [[TMP11:%.*]] = mul nsw i64 [[TMP3]], [[TMP2]] -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] -; CHECK: [[VECTOR_MEMCHECK]]: +; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PREHEADER]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ] ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP3]], [[OUTER_IV]] ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]] ; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP4]], [[TMP6]] ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]] -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP3]], [[TMP4]] +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP5]], [[OUTER_IV]] ; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP8]] ; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP4]], [[TMP8]] ; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP10:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP1]] +; CHECK-NEXT: [[TMP11:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP2]] +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; CHECK: [[VECTOR_MEMCHECK]]: ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] @@ -862,7 +865,7 @@ define void @uncomputable_outer_tc(ptr nocapture noundef %dst, ptr nocapture nou ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[WIDE_TRIP_COUNT]] ; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[INNER_LOOP_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP45:![0-9]+]] ; CHECK: [[INNER_LOOP_EXIT]]: -; CHECK-NEXT: [[OUTER_IV_NEXT]] = add i64 [[TMP3]], 1 +; CHECK-NEXT: [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1 ; CHECK-NEXT: [[ARRAYIDX_US:%.*]] = getelementptr inbounds i8, ptr [[STR]], i64 [[OUTER_IV_NEXT]] ; CHECK-NEXT: [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX_US]], align 1 ; CHECK-NEXT: [[CMP_NOT_US:%.*]] = icmp eq i8 [[TMP22]], 0 @@ -953,20 +956,20 @@ define void @decreasing_inner_iv(ptr nocapture noundef %dst, ptr nocapture nound ; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[STRIDE2]] to i64 ; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[STRIDE1]] to i64 ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[M]] to i64 -; CHECK-NEXT: [[TMP14:%.*]] = shl nsw i64 [[TMP1]], 2 -; CHECK-NEXT: [[TMP9:%.*]] = shl nsw i64 [[TMP2]], 2 ; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1 -; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]] +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP2]] ; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2 ; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP0]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP5]], [[TMP6]] ; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP7]], 4 -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP3]], [[TMP2]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP9:%.*]] = shl nsw i64 [[TMP2]], 2 +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP3]], [[TMP1]] ; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[TMP10]], 2 ; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], [[TMP6]] ; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], 4 -; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]] +; CHECK-NEXT: [[TMP14:%.*]] = shl nsw i64 [[TMP1]], 2 ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PRE]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ] @@ -1115,21 +1118,21 @@ define void @decreasing_outer_iv(ptr nocapture noundef %dst, ptr nocapture nound ; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[STRIDE1]] to i64 ; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[STRIDE2]] to i64 ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[TMP0]] to i64 -; CHECK-NEXT: [[TMP14:%.*]] = mul nsw i64 [[TMP3]], -4 -; CHECK-NEXT: [[TMP9:%.*]] = mul nsw i64 [[TMP2]], -4 -; CHECK-NEXT: [[TMP6:%.*]] = shl nsw i64 [[TMP3]], 2 +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP2]], [[TMP1]] +; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = shl nsw i64 [[TMP2]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2 ; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP6]], [[TMP7]] -; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP8]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP9:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP3]], [[TMP1]] ; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[TMP10]], 2 ; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP11]] -; CHECK-NEXT: [[TMP12:%.*]] = shl nsw i64 [[TMP2]], 2 +; CHECK-NEXT: [[TMP12:%.*]] = shl nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], [[TMP7]] -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]] -; CHECK-NEXT: [[TMP29:%.*]] = mul i64 [[TMP2]], [[TMP1]] -; CHECK-NEXT: [[TMP30:%.*]] = shl i64 [[TMP29]], 2 -; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP30]] +; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]] +; CHECK-NEXT: [[TMP14:%.*]] = mul nsw i64 [[TMP3]], -4 ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ [[TMP1]], %[[OUTER_LOOP_PRE]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ] @@ -1273,15 +1276,15 @@ define void @unknown_inner_stride(ptr nocapture noundef %dst, ptr nocapture noun ; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[ADD6]] to i64 ; CHECK-NEXT: [[WIDE_TRIP_COUNT39:%.*]] = zext i32 [[M]] to i64 ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64 -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[WIDE_TRIP_COUNT]], [[WIDE_TRIP_COUNT39]] -; CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[TMP9]], 2 -; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP4:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT39]], -1 ; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP3]] ; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2 ; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP6]], [[TMP7]] ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[WIDE_TRIP_COUNT]], [[WIDE_TRIP_COUNT39]] +; CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[TMP9]], 2 +; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP10]] ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PREHEADER]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ] @@ -1491,8 +1494,8 @@ define void @stride_check_known_via_loop_guard(ptr %C, ptr %A, i32 %Acols) { ; CHECK-NEXT: [[PRE_C:%.*]] = icmp ugt i32 [[ACOLS]], 0 ; CHECK-NEXT: br i1 [[PRE_C]], label %[[EXIT:.*]], label %[[OUTER_HEADER_PREHEADER:.*]] ; CHECK: [[OUTER_HEADER_PREHEADER]]: -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 8 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C]], i64 8000 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 8 ; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] ; CHECK: [[OUTER_HEADER]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i32 [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[OUTER_HEADER_PREHEADER]] ] @@ -1607,13 +1610,13 @@ define void @shared_strides(ptr noundef %dst, ptr noundef readonly %src1, ptr no ; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[STRIDE1]] to i64 ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[M]] to i64 ; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1 -; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]] +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP2]] ; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], [[TMP0]] -; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC2]], i64 [[TMP5]] -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC1]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP3]], [[TMP2]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP3]], [[TMP1]] ; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], [[TMP0]] -; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC1]], i64 [[TMP7]] +; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC2]], i64 [[TMP7]] ; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] ; CHECK: [[OUTER_LOOP]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PRE]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ] @@ -1726,256 +1729,3 @@ outer.loop.exit: exit: ret void } - -; Bounds with a udiv by a non-zero, non-poison divisor are hoisted. -define void @udiv_known_nonzero(ptr %dst, ptr %src, i64 %n, i64 noundef %x) { -; CHECK-LABEL: define void @udiv_known_nonzero( -; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]], i64 noundef [[X:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[S:%.*]] = or i64 [[X]], 1 -; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[S]], i64 [[N]]) -; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[UMAX]], -1 -; CHECK-NEXT: [[TMP1:%.*]] = udiv i64 [[TMP0]], [[S]] -; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[TMP1]], 3 -; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 4 -; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]] -; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP1]], 2 -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], 101380 -; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]] -; CHECK-NEXT: br label %[[OUTER:.*]] -; CHECK: [[OUTER]]: -; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ] -; CHECK-NEXT: [[DST_OFF:%.*]] = mul i64 [[OUTER_IV]], 1024 -; CHECK-NEXT: [[DST_O:%.*]] = getelementptr i8, ptr [[DST]], i64 [[DST_OFF]] -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.umax.i64(i64 [[S]], i64 [[N]]) -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], -1 -; CHECK-NEXT: [[TMP8:%.*]] = udiv i64 [[TMP7]], [[S]] -; CHECK-NEXT: [[TMP9:%.*]] = add nuw i64 [[TMP8]], 1 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP9]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]] -; CHECK: [[VECTOR_SCEVCHECK]]: -; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp sle i64 [[S]], 0 -; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]] -; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP5]] -; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] -; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[TMP10:%.*]] = and i64 [[TMP9]], 3 -; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP9]], [[TMP10]] -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[N_VEC]], [[S]] -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1) -; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP13]] -; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP12]], i64 1 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP12]], i64 2 -; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP17]] -; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i64> [[TMP12]], i64 3 -; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP19]] -; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP14]], align 4, !alias.scope [[META86:![0-9]+]] -; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP16]], align 4, !alias.scope [[META86]] -; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP18]], align 4, !alias.scope [[META86]] -; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP20]], align 4, !alias.scope [[META86]] -; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> poison, i32 [[TMP21]], i64 0 -; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP22]], i64 1 -; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP23]], i64 2 -; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> [[TMP27]], i32 [[TMP24]], i64 3 -; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[DST_O]], i64 [[INDEX]] -; CHECK-NEXT: store <4 x i32> [[TMP28]], ptr [[TMP29]], align 4, !alias.scope [[META89:![0-9]+]], !noalias [[META86]] -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) -; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP91:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP9]], [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ] -; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ] -; CHECK-NEXT: br label %[[INNER:.*]] -; CHECK: [[INNER]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[INNER]] ] -; CHECK-NEXT: [[K:%.*]] = phi i64 [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ], [ [[K_NEXT:%.*]], %[[INNER]] ] -; CHECK-NEXT: [[K_2:%.*]] = shl nuw nsw i64 [[K]], 1 -; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[K_2]] -; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4 -; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST_O]], i64 [[K]] -; CHECK-NEXT: store i32 [[L]], ptr [[GEP_DST]], align 4 -; CHECK-NEXT: [[K_NEXT]] = add nuw nsw i64 [[K]], 1 -; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], [[S]] -; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[C]], label %[[INNER]], label %[[OUTER_LATCH]], !llvm.loop [[LOOP92:![0-9]+]] -; CHECK: [[OUTER_LATCH]]: -; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw i64 [[OUTER_IV]], 1 -; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 100 -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[OUTER]] -; CHECK: [[EXIT]]: -; CHECK-NEXT: ret void -; -entry: - %s = or i64 %x, 1 - br label %outer - -outer: - %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ] - %dst.off = mul i64 %outer.iv, 1024 - %dst.o = getelementptr i8, ptr %dst, i64 %dst.off - br label %inner - -inner: - %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ] - %k = phi i64 [ 0, %outer ], [ %k.next, %inner ] - %k.2 = shl nuw nsw i64 %k, 1 - %gep.src = getelementptr inbounds i32, ptr %src, i64 %k.2 - %l = load i32, ptr %gep.src - %gep.dst = getelementptr inbounds i32, ptr %dst.o, i64 %k - store i32 %l, ptr %gep.dst - %k.next = add nuw nsw i64 %k, 1 - %iv.next = add nuw i64 %iv, %s - %c = icmp ult i64 %iv.next, %n - br i1 %c, label %inner, label %outer.latch - -outer.latch: - %outer.iv.next = add nuw i64 %outer.iv, 1 - %ec = icmp eq i64 %outer.iv.next, 100 - br i1 %ec, label %exit, label %outer - -exit: - ret void -} - -; Bounds with a udiv by a non-zero, maybe-poison divisor are not hoisted. -define void @udiv_known_nonzero_maybe_poison(ptr %dst, ptr %src, i64 %n, i64 %x, i1 %g) { -; CHECK-LABEL: define void @udiv_known_nonzero_maybe_poison( -; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]], i64 [[X:%.*]], i1 [[G:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[S:%.*]] = or i64 [[X]], 1 -; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[S]], i64 [[N]]) -; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[UMAX]], -1 -; CHECK-NEXT: br label %[[OUTER:.*]] -; CHECK: [[OUTER]]: -; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ] -; CHECK-NEXT: [[DST_OFF:%.*]] = mul i64 [[OUTER_IV]], 1024 -; CHECK-NEXT: [[DST_O:%.*]] = getelementptr i8, ptr [[DST]], i64 [[DST_OFF]] -; CHECK-NEXT: br i1 [[G]], label %[[INNER_PH:.*]], label %[[OUTER_LATCH]] -; CHECK: [[INNER_PH]]: -; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[S]], i64 [[N]]) -; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], -1 -; CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[TMP2]], [[S]] -; CHECK-NEXT: [[TMP4:%.*]] = add nuw i64 [[TMP3]], 1 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]] -; CHECK: [[VECTOR_SCEVCHECK]]: -; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp sle i64 [[S]], 0 -; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]] -; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[TMP0]], [[S]] -; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 101380 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[TMP5]], 3 -; CHECK-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP9]], 4 -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP10]] -; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[TMP11]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[TMP8]] -; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] -; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[TMP12:%.*]] = and i64 [[TMP4]], 3 -; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[TMP12]] -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[N_VEC]], [[S]] -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP14:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1) -; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP14]], i64 0 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP14]], i64 1 -; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP17]] -; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i64> [[TMP14]], i64 2 -; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP19]] -; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP14]], i64 3 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP16]], align 4, !alias.scope [[META93:![0-9]+]] -; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP18]], align 4, !alias.scope [[META93]] -; CHECK-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP20]], align 4, !alias.scope [[META93]] -; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP22]], align 4, !alias.scope [[META93]] -; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> poison, i32 [[TMP23]], i64 0 -; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> [[TMP27]], i32 [[TMP24]], i64 1 -; CHECK-NEXT: [[TMP29:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP25]], i64 2 -; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> [[TMP29]], i32 [[TMP26]], i64 3 -; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[DST_O]], i64 [[INDEX]] -; CHECK-NEXT: store <4 x i32> [[TMP30]], ptr [[TMP31]], align 4, !alias.scope [[META96:![0-9]+]], !noalias [[META93]] -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) -; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP98:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label %[[OUTER_LATCH_LOOPEXIT:.*]], label %[[SCALAR_PH]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[INNER_PH]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ] -; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[INNER_PH]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ] -; CHECK-NEXT: br label %[[INNER:.*]] -; CHECK: [[INNER]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[INNER]] ] -; CHECK-NEXT: [[K:%.*]] = phi i64 [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[K_NEXT:%.*]], %[[INNER]] ] -; CHECK-NEXT: [[K_2:%.*]] = shl nuw nsw i64 [[K]], 1 -; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[K_2]] -; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4 -; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST_O]], i64 [[K]] -; CHECK-NEXT: store i32 [[L]], ptr [[GEP_DST]], align 4 -; CHECK-NEXT: [[K_NEXT]] = add nuw nsw i64 [[K]], 1 -; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], [[S]] -; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[C]], label %[[INNER]], label %[[OUTER_LATCH_LOOPEXIT]], !llvm.loop [[LOOP99:![0-9]+]] -; CHECK: [[OUTER_LATCH_LOOPEXIT]]: -; CHECK-NEXT: br label %[[OUTER_LATCH]] -; CHECK: [[OUTER_LATCH]]: -; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw i64 [[OUTER_IV]], 1 -; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 100 -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[OUTER]] -; CHECK: [[EXIT]]: -; CHECK-NEXT: ret void -; -entry: - %s = or i64 %x, 1 - br label %outer - -outer: - %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ] - %dst.off = mul i64 %outer.iv, 1024 - %dst.o = getelementptr i8, ptr %dst, i64 %dst.off - br i1 %g, label %inner.ph, label %outer.latch - -inner.ph: - br label %inner - -inner: - %iv = phi i64 [ 0, %inner.ph ], [ %iv.next, %inner ] - %k = phi i64 [ 0, %inner.ph ], [ %k.next, %inner ] - %k.2 = shl nuw nsw i64 %k, 1 - %gep.src = getelementptr inbounds i32, ptr %src, i64 %k.2 - %l = load i32, ptr %gep.src - %gep.dst = getelementptr inbounds i32, ptr %dst.o, i64 %k - store i32 %l, ptr %gep.dst - %k.next = add nuw nsw i64 %k, 1 - %iv.next = add nuw i64 %iv, %s - %c = icmp ult i64 %iv.next, %n - br i1 %c, label %inner, label %outer.latch - -outer.latch: - %outer.iv.next = add nuw i64 %outer.iv, 1 - %ec = icmp eq i64 %outer.iv.next, 100 - br i1 %ec, label %exit, label %outer - -exit: - ret void -} diff --git a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll index dde04130b24cb2..b17f36a7f5ae31 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll @@ -15,13 +15,11 @@ define void @expand(ptr %src, ptr %dst, i64 %0) { ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]] ; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 8 ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]] +; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 8 ; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP0]], 1 ; CHECK-NEXT: [[SMAX6:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 1000) ; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[SMAX6]], 4 ; CHECK-NEXT: [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[TMP0]], 4 -; CHECK-NEXT: [[SCEVGEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP9]] -; CHECK-NEXT: [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 8 ; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] ; CHECK: [[OUTER_HEADER]]: ; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ] @@ -44,7 +42,7 @@ define void @expand(ptr %src, ptr %dst, i64 %0) { ; CHECK-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP7]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP9]], [[SCEVGEP8]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP5]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll index 4bd8876512ce46..b43d12ecf8b3b3 100644 --- a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll +++ b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll @@ -93,12 +93,12 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK-NEXT: [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP1]] ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[I]], 225 ; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP2]]) -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[CONV6]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[CONV6]] ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i32 [[I]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY4_US_PREHEADER:%.*]], label [[FOR_COND1_PREHEADER_US_PREHEADER:%.*]] ; CHECK: vector.memcheck: -; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]] +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[FOR_BODY4_US_PREHEADER]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: @@ -106,7 +106,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP17]], i64 16 ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP17]], align 8, !alias.scope [[META0:![0-9]+]] ; CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <2 x double>, ptr [[TMP18]], align 8, !alias.scope [[META0]] @@ -115,7 +115,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK-NEXT: [[BROADCAST_SPLAT23:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT22]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP20:%.*]] = fmul <2 x double> [[WIDE_LOAD]], [[BROADCAST_SPLAT23]] ; CHECK-NEXT: [[TMP21:%.*]] = fmul <2 x double> [[WIDE_LOAD21]], [[BROADCAST_SPLAT23]] -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP22]], i64 16 ; CHECK-NEXT: [[WIDE_LOAD24:%.*]] = load <2 x double>, ptr [[TMP22]], align 8, !alias.scope [[META5:![0-9]+]], !noalias [[META0]] ; CHECK-NEXT: [[WIDE_LOAD25:%.*]] = load <2 x double>, ptr [[TMP23]], align 8, !alias.scope [[META5]], !noalias [[META0]] @@ -137,13 +137,13 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK: for.cond1.for.cond.cleanup3_crit_edge: ; CHECK-NEXT: [[TMP31:%.*]] = icmp samesign ult i32 [[I]], 210 ; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP31]]) -; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[CONV6]] +; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[CONV6]] ; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP61]], i64 120 ; CHECK-NEXT: [[MIN_ITERS_CHECK_1:%.*]] = icmp samesign ult i32 [[I]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK_1]], label [[FOR_BODY4_US_PREHEADER_1:%.*]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US:%.*]] ; CHECK: vector.memcheck.1: -; CHECK-NEXT: [[BOUND0_1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]] -; CHECK-NEXT: [[BOUND1_1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]] +; CHECK-NEXT: [[BOUND0_1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]] +; CHECK-NEXT: [[BOUND1_1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]] ; CHECK-NEXT: [[FOUND_CONFLICT_1:%.*]] = and i1 [[BOUND0_1]], [[BOUND1_1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT_1]], label [[FOR_BODY4_US_PREHEADER_1]], label [[VECTOR_PH_1:%.*]] ; CHECK: vector.ph.1: @@ -152,7 +152,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK: vector.body.1: ; CHECK-NEXT: [[INDEX_1:%.*]] = phi i64 [ 0, [[VECTOR_PH_1]] ], [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY_1]] ] ; CHECK-NEXT: [[TMP33:%.*]] = add nuw nsw i64 [[INDEX_1]], 15 -; CHECK-NEXT: [[TMP47:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP33]] +; CHECK-NEXT: [[TMP47:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP33]] ; CHECK-NEXT: [[TMP48:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP47]], i64 16 ; CHECK-NEXT: [[WIDE_LOAD_1:%.*]] = load <2 x double>, ptr [[TMP47]], align 8, !alias.scope [[META0]] ; CHECK-NEXT: [[WIDE_LOAD21_1:%.*]] = load <2 x double>, ptr [[TMP48]], align 8, !alias.scope [[META0]] @@ -161,7 +161,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK-NEXT: [[BROADCAST_SPLAT23_1:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT22_1]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP50:%.*]] = fmul <2 x double> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT23_1]] ; CHECK-NEXT: [[TMP51:%.*]] = fmul <2 x double> [[WIDE_LOAD21_1]], [[BROADCAST_SPLAT23_1]] -; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP33]] +; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP33]] ; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP52]], i64 16 ; CHECK-NEXT: [[WIDE_LOAD24_1:%.*]] = load <2 x double>, ptr [[TMP52]], align 8, !alias.scope [[META5]], !noalias [[META0]] ; CHECK-NEXT: [[WIDE_LOAD25_1:%.*]] = load <2 x double>, ptr [[TMP53]], align 8, !alias.scope [[META5]], !noalias [[META0]] @@ -183,11 +183,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK-NEXT: [[TMP57:%.*]] = add nuw nsw i64 [[INDVARS_IV_1]], 15 ; CHECK-NEXT: [[TMP58:%.*]] = icmp samesign ult i64 [[INDVARS_IV_1]], 210 ; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP58]]) -; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP57]] +; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP57]] ; CHECK-NEXT: [[MATRIXEXT_US_1:%.*]] = load double, ptr [[TMP59]], align 8 ; CHECK-NEXT: [[MATRIXEXT8_US_1:%.*]] = load double, ptr [[TMP32]], align 8 ; CHECK-NEXT: [[MUL_US_1:%.*]] = fmul double [[MATRIXEXT_US_1]], [[MATRIXEXT8_US_1]] -; CHECK-NEXT: [[TMP60:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP57]] +; CHECK-NEXT: [[TMP60:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP57]] ; CHECK-NEXT: [[MATRIXEXT11_US_1:%.*]] = load double, ptr [[TMP60]], align 8 ; CHECK-NEXT: [[SUB_US_1:%.*]] = fsub double [[MATRIXEXT11_US_1]], [[MUL_US_1]] ; CHECK-NEXT: store double [[SUB_US_1]], ptr [[TMP60]], align 8 @@ -197,13 +197,13 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK: for.cond1.for.cond.cleanup3_crit_edge.1: ; CHECK-NEXT: [[TMP62:%.*]] = icmp samesign ult i32 [[I]], 195 ; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP62]]) -; CHECK-NEXT: [[TMP92:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[CONV6]] +; CHECK-NEXT: [[TMP92:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[CONV6]] ; CHECK-NEXT: [[TMP63:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP92]], i64 240 ; CHECK-NEXT: [[MIN_ITERS_CHECK_2:%.*]] = icmp samesign ult i32 [[I]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK_2]], label [[FOR_BODY4_US_PREHEADER_2:%.*]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_1:%.*]] ; CHECK: vector.memcheck.2: -; CHECK-NEXT: [[BOUND0_2:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]] -; CHECK-NEXT: [[BOUND1_2:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]] +; CHECK-NEXT: [[BOUND0_2:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]] +; CHECK-NEXT: [[BOUND1_2:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]] ; CHECK-NEXT: [[FOUND_CONFLICT_2:%.*]] = and i1 [[BOUND0_2]], [[BOUND1_2]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT_2]], label [[FOR_BODY4_US_PREHEADER_2]], label [[VECTOR_PH_2:%.*]] ; CHECK: vector.ph.2: @@ -212,7 +212,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK: vector.body.2: ; CHECK-NEXT: [[INDEX_2:%.*]] = phi i64 [ 0, [[VECTOR_PH_2]] ], [ [[INDEX_NEXT_2:%.*]], [[VECTOR_BODY_2]] ] ; CHECK-NEXT: [[TMP64:%.*]] = add nuw nsw i64 [[INDEX_2]], 30 -; CHECK-NEXT: [[TMP78:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP64]] +; CHECK-NEXT: [[TMP78:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP64]] ; CHECK-NEXT: [[TMP79:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP78]], i64 16 ; CHECK-NEXT: [[WIDE_LOAD_2:%.*]] = load <2 x double>, ptr [[TMP78]], align 8, !alias.scope [[META0]] ; CHECK-NEXT: [[WIDE_LOAD21_2:%.*]] = load <2 x double>, ptr [[TMP79]], align 8, !alias.scope [[META0]] @@ -221,7 +221,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK-NEXT: [[BROADCAST_SPLAT23_2:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT22_2]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP81:%.*]] = fmul <2 x double> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT23_2]] ; CHECK-NEXT: [[TMP82:%.*]] = fmul <2 x double> [[WIDE_LOAD21_2]], [[BROADCAST_SPLAT23_2]] -; CHECK-NEXT: [[TMP83:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP64]] +; CHECK-NEXT: [[TMP83:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP64]] ; CHECK-NEXT: [[TMP84:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP83]], i64 16 ; CHECK-NEXT: [[WIDE_LOAD24_2:%.*]] = load <2 x double>, ptr [[TMP83]], align 8, !alias.scope [[META5]], !noalias [[META0]] ; CHECK-NEXT: [[WIDE_LOAD25_2:%.*]] = load <2 x double>, ptr [[TMP84]], align 8, !alias.scope [[META5]], !noalias [[META0]] @@ -243,11 +243,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK-NEXT: [[TMP88:%.*]] = add nuw nsw i64 [[INDVARS_IV_2]], 30 ; CHECK-NEXT: [[TMP89:%.*]] = icmp samesign ult i64 [[INDVARS_IV_2]], 195 ; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP89]]) -; CHECK-NEXT: [[TMP90:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP88]] +; CHECK-NEXT: [[TMP90:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP88]] ; CHECK-NEXT: [[MATRIXEXT_US_2:%.*]] = load double, ptr [[TMP90]], align 8 ; CHECK-NEXT: [[MATRIXEXT8_US_2:%.*]] = load double, ptr [[TMP63]], align 8 ; CHECK-NEXT: [[MUL_US_2:%.*]] = fmul double [[MATRIXEXT_US_2]], [[MATRIXEXT8_US_2]] -; CHECK-NEXT: [[TMP91:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP88]] +; CHECK-NEXT: [[TMP91:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP88]] ; CHECK-NEXT: [[MATRIXEXT11_US_2:%.*]] = load double, ptr [[TMP91]], align 8 ; CHECK-NEXT: [[SUB_US_2:%.*]] = fsub double [[MATRIXEXT11_US_2]], [[MUL_US_2]] ; CHECK-NEXT: store double [[SUB_US_2]], ptr [[TMP91]], align 8 @@ -257,13 +257,13 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK: for.cond1.for.cond.cleanup3_crit_edge.2: ; CHECK-NEXT: [[TMP93:%.*]] = icmp samesign ult i32 [[I]], 180 ; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP93]]) -; CHECK-NEXT: [[TMP123:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[CONV6]] +; CHECK-NEXT: [[TMP123:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[CONV6]] ; CHECK-NEXT: [[TMP94:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP123]], i64 360 ; CHECK-NEXT: [[MIN_ITERS_CHECK_3:%.*]] = icmp samesign ult i32 [[I]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK_3]], label [[FOR_BODY4_US_PREHEADER_3:%.*]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_2:%.*]] ; CHECK: vector.memcheck.3: -; CHECK-NEXT: [[BOUND0_3:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]] -; CHECK-NEXT: [[BOUND1_3:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]] +; CHECK-NEXT: [[BOUND0_3:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]] +; CHECK-NEXT: [[BOUND1_3:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]] ; CHECK-NEXT: [[FOUND_CONFLICT_3:%.*]] = and i1 [[BOUND0_3]], [[BOUND1_3]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT_3]], label [[FOR_BODY4_US_PREHEADER_3]], label [[VECTOR_PH_3:%.*]] ; CHECK: vector.ph.3: @@ -272,7 +272,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK: vector.body.3: ; CHECK-NEXT: [[INDEX_3:%.*]] = phi i64 [ 0, [[VECTOR_PH_3]] ], [ [[INDEX_NEXT_3:%.*]], [[VECTOR_BODY_3]] ] ; CHECK-NEXT: [[TMP95:%.*]] = add nuw nsw i64 [[INDEX_3]], 45 -; CHECK-NEXT: [[TMP109:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP95]] +; CHECK-NEXT: [[TMP109:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP95]] ; CHECK-NEXT: [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP109]], i64 16 ; CHECK-NEXT: [[WIDE_LOAD_3:%.*]] = load <2 x double>, ptr [[TMP109]], align 8, !alias.scope [[META0]] ; CHECK-NEXT: [[WIDE_LOAD21_3:%.*]] = load <2 x double>, ptr [[TMP110]], align 8, !alias.scope [[META0]] @@ -281,7 +281,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK-NEXT: [[BROADCAST_SPLAT23_3:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT22_3]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP112:%.*]] = fmul <2 x double> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT23_3]] ; CHECK-NEXT: [[TMP113:%.*]] = fmul <2 x double> [[WIDE_LOAD21_3]], [[BROADCAST_SPLAT23_3]] -; CHECK-NEXT: [[TMP114:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP95]] +; CHECK-NEXT: [[TMP114:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP95]] ; CHECK-NEXT: [[TMP115:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP114]], i64 16 ; CHECK-NEXT: [[WIDE_LOAD24_3:%.*]] = load <2 x double>, ptr [[TMP114]], align 8, !alias.scope [[META5]], !noalias [[META0]] ; CHECK-NEXT: [[WIDE_LOAD25_3:%.*]] = load <2 x double>, ptr [[TMP115]], align 8, !alias.scope [[META5]], !noalias [[META0]] @@ -303,11 +303,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK-NEXT: [[TMP119:%.*]] = add nuw nsw i64 [[INDVARS_IV_3]], 45 ; CHECK-NEXT: [[TMP120:%.*]] = icmp samesign ult i64 [[INDVARS_IV_3]], 180 ; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP120]]) -; CHECK-NEXT: [[TMP121:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP119]] +; CHECK-NEXT: [[TMP121:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP119]] ; CHECK-NEXT: [[MATRIXEXT_US_3:%.*]] = load double, ptr [[TMP121]], align 8 ; CHECK-NEXT: [[MATRIXEXT8_US_3:%.*]] = load double, ptr [[TMP94]], align 8 ; CHECK-NEXT: [[MUL_US_3:%.*]] = fmul double [[MATRIXEXT_US_3]], [[MATRIXEXT8_US_3]] -; CHECK-NEXT: [[TMP122:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP119]] +; CHECK-NEXT: [[TMP122:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP119]] ; CHECK-NEXT: [[MATRIXEXT11_US_3:%.*]] = load double, ptr [[TMP122]], align 8 ; CHECK-NEXT: [[SUB_US_3:%.*]] = fsub double [[MATRIXEXT11_US_3]], [[MUL_US_3]] ; CHECK-NEXT: store double [[SUB_US_3]], ptr [[TMP122]], align 8 @@ -318,11 +318,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY4_US]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY4_US_PREHEADER]] ] ; CHECK-NEXT: [[TMP68:%.*]] = icmp samesign ult i64 [[INDVARS_IV]], 225 ; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP68]]) -; CHECK-NEXT: [[TMP69:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[INDVARS_IV]] +; CHECK-NEXT: [[TMP69:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[INDVARS_IV]] ; CHECK-NEXT: [[MATRIXEXT:%.*]] = load double, ptr [[TMP69]], align 8 ; CHECK-NEXT: [[MATRIXEXT8:%.*]] = load double, ptr [[TMP3]], align 8 ; CHECK-NEXT: [[MUL:%.*]] = fmul double [[MATRIXEXT]], [[MATRIXEXT8]] -; CHECK-NEXT: [[TMP70:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[INDVARS_IV]] +; CHECK-NEXT: [[TMP70:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[INDVARS_IV]] ; CHECK-NEXT: [[MATRIXEXT11:%.*]] = load double, ptr [[TMP70]], align 8 ; CHECK-NEXT: [[SUB:%.*]] = fsub double [[MATRIXEXT11]], [[MUL]] ; CHECK-NEXT: store double [[SUB]], ptr [[TMP70]], align 8 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
