Author: Vitaly Buka
Date: 2026-10-30T18:32:13-07:00
New Revision: c24601a78f98b957c3e0e284e1fcda94c92b2689

URL: 
https://github.com/llvm/llvm-project/commit/c24601a78f98b957c3e0e284e1fcda94c92b2689
DIFF: 
https://github.com/llvm/llvm-project/commit/c24601a78f98b957c3e0e284e1fcda94c92b2689.diff

LOG: Revert "[LV] Model memory runtime checks for inner loops of loop nests in 
VPl…"

This reverts commit 0d387de3856273874ef65aa2e90cd07a7a9a2fd0.

Added: 
    

Modified: 
    llvm/include/llvm/Transforms/Utils/LoopUtils.h
    llvm/lib/Transforms/Utils/LoopUtils.cpp
    llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
    llvm/lib/Transforms/Vectorize/VPlan.h
    llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
    llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
    llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
    llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
    llvm/lib/Transforms/Vectorize/VPlanTransforms.h
    llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
    llvm/lib/Transforms/Vectorize/VPlanUtils.h
    
llvm/test/Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll
    llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop-diff-checks.ll
    llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll
    llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
    llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
    llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll
    llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
    llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
    
llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
    llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll

Removed: 
    


################################################################################
diff  --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h 
b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 5b539ee515ca78..ae51fcecc02583 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -661,21 +661,6 @@ LLVM_ABI void appendLoopsToWorklist(LoopInfo &,
 LLVM_ABI Loop *cloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM, LoopInfo 
*LI,
                          LPPassManager *LPM);
 
-/// SCEV bounds of a pointer group: the first accessed byte, the last accessed
-/// byte plus one, and a stride that must be checked to be non-negative, if 
any.
-struct RuntimeCheckBounds {
-  const SCEV *Low;
-  const SCEV *High;
-  const SCEV *StrideToCheck;
-};
-
-/// Return the bounds of \p CG in \p TheLoop. If \p HoistRuntimeChecks is true,
-/// bounds varying in the parent loop of \p TheLoop may be widened to cover the
-/// whole parent loop, so that checks using them can be hoisted out of it.
-LLVM_ABI RuntimeCheckBounds
-getRuntimeCheckBounds(const RuntimeCheckingPtrGroup *CG, const Loop *TheLoop,
-                      ScalarEvolution &SE, bool HoistRuntimeChecks);
-
 /// Add code that checks at runtime if the accessed arrays in \p PointerChecks
 /// overlap. Returns the final comparator value or NULL if no check is needed.
 LLVM_ABI Value *

diff  --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp 
b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 2968f4600fd9a7..c1a7362f77beca 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2120,10 +2120,16 @@ struct PointerBounds {
   Value *StrideToCheck;
 };
 
-RuntimeCheckBounds
-llvm::getRuntimeCheckBounds(const RuntimeCheckingPtrGroup *CG,
-                            const Loop *TheLoop, ScalarEvolution &SE,
-                            bool HoistRuntimeChecks) {
+/// Expand code for the lower and upper bound of the pointer group \p CG
+/// in \p TheLoop.  \return the values for the bounds.
+static PointerBounds expandBounds(const RuntimeCheckingPtrGroup *CG,
+                                  Loop *TheLoop, Instruction *Loc,
+                                  SCEVExpander &Exp, bool HoistRuntimeChecks) {
+  LLVMContext &Ctx = Loc->getContext();
+  Type *PtrArithTy = PointerType::get(Ctx, CG->AddressSpace);
+
+  Value *Start = nullptr, *End = nullptr;
+  LLVM_DEBUG(dbgs() << "LAA: Adding RT check for range:\n");
   const SCEV *Low = CG->Low, *High = CG->High, *Stride = nullptr;
 
   // If the Low and High values are themselves loop-variant, then we may want
@@ -2141,6 +2147,7 @@ llvm::getRuntimeCheckBounds(const RuntimeCheckingPtrGroup 
*CG,
     auto *HighAR = cast<SCEVAddRecExpr>(High);
     auto *LowAR = cast<SCEVAddRecExpr>(Low);
     const Loop *OuterLoop = TheLoop->getParentLoop();
+    ScalarEvolution &SE = *Exp.getSE();
     const SCEV *Recur = LowAR->getStepRecurrence(SE);
     if (Recur == HighAR->getStepRecurrence(SE) &&
         HighAR->getLoop() == OuterLoop && LowAR->getLoop() == OuterLoop) {
@@ -2168,21 +2175,6 @@ llvm::getRuntimeCheckBounds(const 
RuntimeCheckingPtrGroup *CG,
       }
     }
   }
-  return {Low, High, Stride};
-}
-
-/// Expand code for the lower and upper bound of the pointer group \p CG
-/// in \p TheLoop.  \return the values for the bounds.
-static PointerBounds expandBounds(const RuntimeCheckingPtrGroup *CG,
-                                  Loop *TheLoop, Instruction *Loc,
-                                  SCEVExpander &Exp, bool HoistRuntimeChecks) {
-  LLVMContext &Ctx = Loc->getContext();
-  Type *PtrArithTy = PointerType::get(Ctx, CG->AddressSpace);
-
-  Value *Start = nullptr, *End = nullptr;
-  LLVM_DEBUG(dbgs() << "LAA: Adding RT check for range:\n");
-  auto [Low, High, Stride] =
-      getRuntimeCheckBounds(CG, TheLoop, *Exp.getSE(), HoistRuntimeChecks);
 
   Start = Exp.expandCodeFor(Low, PtrArithTy, Loc);
   End = Exp.expandCodeFor(High, PtrArithTy, Loc);

diff  --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp 
b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index fce481ec049bfa..c17abe52fb1c36 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6737,11 +6737,17 @@ void LoopVectorizationPlanner::attachRuntimeChecks(
                   "(e.g., adding 'restrict').";
       });
     }
+    // The VPlan expander cannot hoist bounds out of an enclosing loop.
     const auto &RtPtrChecking = *Legal->getRuntimePointerChecking();
+    if (OrigLoop->getParentLoop())
+      return RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan,
+                            MemCheckCond, MemCheckBlock, HasBranchWeights);
+
     // Erase the temporary IR before recipe expansion can reuse its values.
     RTChecks.eraseMemCheckBlock();
     RUN_VPLAN_PASS(VPlanTransforms::attachMemoryChecks, Plan, RtPtrChecking,
-                   OrigLoop, *PSE.getSE(), VF, UF, HasBranchWeights);
+                   *PSE.getSE(), VF, UF, OrigLoop->getStartLoc(),
+                   HasBranchWeights);
   }
 }
 

diff  --git a/llvm/lib/Transforms/Vectorize/VPlan.h 
b/llvm/lib/Transforms/Vectorize/VPlan.h
index 7764ec9b45a947..279f93b2fc90fa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4090,18 +4090,12 @@ struct LLVM_ABI_FOR_TEST VPWidenStoreEVLRecipe final
 class VPExpandSCEVRecipe : public VPSingleDefRecipe {
   const SCEV *Expr;
 
-  /// Whether \p Expr must be expanded in IR, which hoists it out of the loops
-  /// it is invariant in, instead of as VPInstructions.
-  bool ExpandInIR;
-
 public:
-  VPExpandSCEVRecipe(const SCEV *Expr, bool ExpandInIR = false);
+  VPExpandSCEVRecipe(const SCEV *Expr);
 
   ~VPExpandSCEVRecipe() override = default;
 
-  VPExpandSCEVRecipe *clone() override {
-    return new VPExpandSCEVRecipe(Expr, ExpandInIR);
-  }
+  VPExpandSCEVRecipe *clone() override { return new VPExpandSCEVRecipe(Expr); }
 
   VP_CLASSOF_IMPL(VPRecipeBase::VPExpandSCEVSC)
 
@@ -4118,8 +4112,6 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe {
 
   const SCEV *getSCEV() const { return Expr; }
 
-  bool expandInIR() const { return ExpandInIR; }
-
 protected:
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.

diff  --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp 
b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 6daa320993d661..a62f1ddae9769b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1583,53 +1583,39 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, 
Value *Cond,
   attachVPCheckBlock(Plan, CondVPV, CheckBlockVPBB, AddBranchWeights);
 }
 
-/// Generate recipes for the memory runtime checks \p Checks of \p TheLoop
-/// using \p Builder and return the condition that is true if any check fails.
+/// Generate recipes for the memory runtime checks \p Checks using \p Builder
+/// and return the condition that is true if any check fails.
 static VPValue *createRuntimeChecks(VPlan &Plan, VPBuilder &Builder,
                                     VPSCEVExpander &Expander,
                                     ArrayRef<RuntimePointerCheck> Checks,
-                                    const Loop *TheLoop, ScalarEvolution &SE,
                                     DebugLoc DL) {
   // Expand each group's bounds once so all checks reuse the same frozen 
values.
   SmallDenseMap<const RuntimeCheckingPtrGroup *,
-                std::tuple<VPValue *, VPValue *, const SCEV *>, 8>
+                std::pair<VPValue *, VPValue *>, 8>
       GroupToBounds;
-  for (const auto &[A, B] : Checks) {
+  for (const auto &[A, B] : Checks)
     for (const RuntimeCheckingPtrGroup *CG : {A, B}) {
-      if (GroupToBounds.contains(CG))
+      auto &[Start, End] = GroupToBounds[CG];
+      if (Start)
         continue;
-      auto [Low, High, Stride] = getRuntimeCheckBounds(
-          CG, TheLoop, SE, VectorizerParams::HoistRuntimeChecks);
-      VPValue *Start = Expander.expand(Low);
-      VPValue *End = Expander.expand(High);
+      Start = Expander.expand(CG->Low);
+      End = Expander.expand(CG->High);
       if (CG->NeedsFreeze) {
         Start = Builder.createFreeze(Start, DL);
         End = Builder.createFreeze(End, DL);
       }
-      GroupToBounds[CG] = {Start, End, Stride};
     }
-  }
 
   VPValue *Cond = Plan.getFalse();
-  SmallPtrSet<const SCEV *, 4> CheckedStrides;
   for (const auto &[A, B] : Checks) {
-    auto [AStart, AEnd, AStride] = GroupToBounds[A];
-    auto [BStart, BEnd, BStride] = GroupToBounds[B];
+    auto [AStart, AEnd] = GroupToBounds[A];
+    auto [BStart, BEnd] = GroupToBounds[B];
     VPValue *Bound0 =
         Builder.createICmp(CmpInst::ICMP_ULT, AStart, BEnd, DL, "bound0");
     VPValue *Bound1 =
         Builder.createICmp(CmpInst::ICMP_ULT, BStart, AEnd, DL, "bound1");
     VPValue *IsConflict =
         Builder.createAnd(Bound0, Bound1, DL, "found.conflict");
-    for (const SCEV *Stride : {AStride, BStride}) {
-      if (!Stride || !CheckedStrides.insert(Stride).second)
-        continue;
-      VPValue *StrideV = Expander.expand(Stride);
-      VPValue *IsNegative = Builder.createICmp(
-          CmpInst::ICMP_SLT, StrideV, Plan.getZero(StrideV->getScalarType()),
-          DL, "stride.check");
-      IsConflict = Builder.createOr(IsConflict, IsNegative, DL);
-    }
     Cond = Builder.createOr(Cond, IsConflict, DL, "conflict.rdx");
   }
   return Cond;
@@ -1678,21 +1664,19 @@ static VPValue *createDiffRuntimeChecks(VPlan &Plan, 
VPBuilder &Builder,
 
 void VPlanTransforms::attachMemoryChecks(
     VPlan &Plan, const RuntimePointerChecking &RtPtrChecking,
-    const Loop *TheLoop, ScalarEvolution &SE, ElementCount VF, unsigned UF,
+    ScalarEvolution &SE, ElementCount VF, unsigned UF, DebugLoc DL,
     bool AddBranchWeights) {
   assert(!RtPtrChecking.getChecks().empty() && "No checks to generate");
 
-  DebugLoc DL = TheLoop->getStartLoc();
   auto *MemCheckVPBB = Plan.createVPBasicBlock("vector.memcheck");
   VPBuilder Builder(MemCheckVPBB);
-  VPSCEVExpander Expander(Builder, SE, DL, TheLoop->getParentLoop());
+  VPSCEVExpander Expander(Builder, SE, DL);
   auto DiffChecks = RtPtrChecking.getDiffChecks();
-  VPValue *Cond =
-      DiffChecks
-          ? createDiffRuntimeChecks(Plan, Builder, Expander, *DiffChecks, SE,
-                                    VF, UF, DL)
-          : createRuntimeChecks(Plan, Builder, Expander,
-                                RtPtrChecking.getChecks(), TheLoop, SE, DL);
+  VPValue *Cond = DiffChecks
+                      ? createDiffRuntimeChecks(Plan, Builder, Expander,
+                                                *DiffChecks, SE, VF, UF, DL)
+                      : createRuntimeChecks(Plan, Builder, Expander,
+                                            RtPtrChecking.getChecks(), DL);
   attachVPCheckBlock(Plan, Cond, MemCheckVPBB, AddBranchWeights);
 }
 

diff  --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp 
b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index e26f9b2ba0c27d..5335f00013869f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -1063,9 +1063,7 @@ void VPlanTransforms::expandSCEVsToVPInstructions(VPlan 
&Plan,
     if (ExpSCEV.user_empty())
       continue;
     Builder.setInsertPoint(&ExpSCEV);
-    VPValue *Expanded = Expander.expand(ExpSCEV);
-    if (Expanded == &ExpSCEV)
-      continue;
+    VPValue *Expanded = Expander.expand(ExpSCEV.getSCEV());
     ExpSCEV.replaceAllUsesWith(Expanded);
     // TripCount should not be used after expansion to VPInstructions. Reset to
     // poison to avoid dangling references.

diff  --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp 
b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 957990ceb90256..5d9fb8f21d3be1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -471,9 +471,9 @@ void VPRecipeBase::print(raw_ostream &O, const Twine 
&Indent,
 }
 #endif
 
-VPExpandSCEVRecipe::VPExpandSCEVRecipe(const SCEV *Expr, bool ExpandInIR)
+VPExpandSCEVRecipe::VPExpandSCEVRecipe(const SCEV *Expr)
     : VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, {}, Expr->getType()),
-      Expr(Expr), ExpandInIR(ExpandInIR) {}
+      Expr(Expr) {}
 
 static SmallVector<unsigned> getExtractValueIndices(ArrayRef<VPValue *> Ops) {
   return map_to_vector(drop_begin(Ops), [](VPValue *Op) {
@@ -5205,8 +5205,6 @@ void VPExpandSCEVRecipe::printRecipe(raw_ostream &O, 
const Twine &Indent,
   O << Indent << "EMIT ";
   printAsOperand(O, SlotTracker);
   O << " = EXPAND SCEV " << *Expr;
-  if (ExpandInIR)
-    O << " (in IR)";
 }
 #endif
 

diff  --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp 
b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3497d07d29ec9b..854d5c7dd4c249 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1274,11 +1274,6 @@ static VPValue *simplifyRecipe(VPlan &Plan, 
VPSingleDefRecipe *Def) {
 
   VPValue *A, *B;
 
-  // icmp Pred X, X -> true if Pred is true when equal, false otherwise.
-  CmpPredicate Pred;
-  if (match(Def, m_ICmp(Pred, m_VPValue(A), m_Deferred(A))))
-    return ICmpInst::isTrueWhenEqual(Pred) ? Plan.getTrue() : Plan.getFalse();
-
   if (match(Def, m_c_Add(m_VPValue(A), m_ZeroInt())))
     return A;
 
@@ -2399,13 +2394,9 @@ struct VPCSEDenseMapInfo : public 
DenseMapInfo<VPSingleDefRecipe *> {
     if (auto *LL = dyn_cast<VPWidenMemoryRecipe>(L))
       if (LL->isConsecutive() != cast<VPWidenMemoryRecipe>(R)->isConsecutive())
         return false;
-    if (auto *LExp = dyn_cast<VPExpandSCEVRecipe>(L)) {
-      auto *RExp = cast<VPExpandSCEVRecipe>(R);
-      if (LExp->getSCEV() != RExp->getSCEV())
+    if (auto *LExp = dyn_cast<VPExpandSCEVRecipe>(L))
+      if (LExp->getSCEV() != cast<VPExpandSCEVRecipe>(R)->getSCEV())
         return false;
-      assert(LExp->expandInIR() == RExp->expandInIR() &&
-             "cannot CSE recipes expanded in IR and as VPInstructions");
-    }
     // Phi recipes can only be equal if they are in the same VPBB, as they
     // implicitly depend on their predecessors.
     if (isa<VPWidenPHIRecipe>(L) && L->getParent() != R->getParent())

diff  --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h 
b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 5f7c26897bd076..3d6bda8641f6ea 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -214,13 +214,13 @@ struct VPlanTransforms {
   static void attachCheckBlock(VPlan &Plan, Value *Cond, BasicBlock 
*CheckBlock,
                                bool AddBranchWeights);
 
-  /// Generate the memory runtime checks of \p RtPtrChecking for \p TheLoop as
-  /// recipes and attach the check block to \p Plan. \p VF and \p UF are used
-  /// to compute the thresholds of 
diff erence checks.
+  /// Generate the memory runtime checks of \p RtPtrChecking as recipes and
+  /// attach the check block to \p Plan. \p VF and \p UF are used to compute
+  /// the thresholds of 
diff erence checks.
   static void attachMemoryChecks(VPlan &Plan,
                                  const RuntimePointerChecking &RtPtrChecking,
-                                 const Loop *TheLoop, ScalarEvolution &SE,
-                                 ElementCount VF, unsigned UF,
+                                 ScalarEvolution &SE, ElementCount VF,
+                                 unsigned UF, DebugLoc DL,
                                  bool AddBranchWeights);
 
   /// Model the blocks the executed \p MainPlan generated for the main vector

diff  --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp 
b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 5a6d0a3d0e2823..52c70621ae1db0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -80,8 +80,7 @@ bool vputils::onlyScalarValuesUsed(const VPValue *Def) {
                 [Def](const VPUser *U) { return U->usesScalars(Def); });
 }
 
-VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
-                                                bool ExpandInIR) {
+VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) 
{
   if (auto *E = dyn_cast<SCEVConstant>(Expr))
     return Plan.getOrAddLiveIn(E->getValue());
   // Skip SCEV expansion if Expr is a SCEVUnknown wrapping a non-instruction
@@ -91,7 +90,7 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, 
const SCEV *Expr,
   auto *U = dyn_cast<SCEVUnknown>(Expr);
   if (U && !isa<Instruction>(U->getValue()))
     return Plan.getOrAddLiveIn(U->getValue());
-  auto *Expanded = new VPExpandSCEVRecipe(Expr, ExpandInIR);
+  auto *Expanded = new VPExpandSCEVRecipe(Expr);
   VPBasicBlock *EntryVPBB = Plan.getEntry();
   auto Iter = EntryVPBB->getFirstNonPhi();
   while (Iter != EntryVPBB->end() && isa<VPIRInstruction>(*Iter))
@@ -987,11 +986,6 @@ VPValue *VPSCEVExpander::expandImpl(const SCEV *S) {
   if (VPValue *V = tryToReuseIRValue(S))
     return V;
 
-  if (HoistOutOf && !SafeUDivMode && SE.isLoopInvariant(S, HoistOutOf) &&
-      SE.isGuaranteedNotToCauseUB(S))
-    return vputils::getOrCreateVPValueForSCEVExpr(Builder.getPlan(), S,
-                                                  /*ExpandInIR=*/true);
-
   switch (S->getSCEVType()) {
   case scConstant:
     return Builder.getPlan().getOrAddLiveIn(cast<SCEVConstant>(S)->getValue());

diff  --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h 
b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 94676d87434010..b66dad723af5d0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -43,9 +43,8 @@ bool onlyScalarValuesUsed(const VPValue *Def);
 /// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the 
live-in
 /// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p 
Plan's
 /// pre-header already contains a recipe expanding \p Expr, return it. If not,
-/// create a new one. \p ExpandInIR is forwarded to the VPExpandSCEVRecipe.
-VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
-                                       bool ExpandInIR = false);
+/// create a new one.
+VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr);
 
 /// Return the SCEV expression for \p V. Returns SCEVCouldNotCompute if no
 /// SCEV expression could be constructed.
@@ -282,10 +281,6 @@ class VPSCEVExpander {
   /// zero, matching SCEVExpander's SafeUDivMode.
   bool SafeUDivMode = false;
 
-  /// If set, expressions invariant in this loop are expanded in IR via
-  /// VPExpandSCEVRecipes, which hoists them out of it.
-  const Loop *HoistOutOf;
-
   DenseMap<PointerIntPair<const SCEV *, 1, bool>, VPValue *>
       InsertedExpressions;
 
@@ -300,18 +295,11 @@ class VPSCEVExpander {
   VPValue *expandImpl(const SCEV *S);
 
 public:
-  VPSCEVExpander(VPBuilder &Builder, ScalarEvolution &SE, DebugLoc DL,
-                 const Loop *HoistOutOf = nullptr)
-      : Builder(Builder), SE(SE), DL(DL), HoistOutOf(HoistOutOf) {}
+  VPSCEVExpander(VPBuilder &Builder, ScalarEvolution &SE, DebugLoc DL)
+      : Builder(Builder), SE(SE), DL(DL) {}
 
   /// Expand \p S into recipes and live-ins using the builder.
   VPValue *expand(const SCEV *S);
-
-  /// Expand the expression of \p R, or return \p R if it must be expanded in
-  /// IR.
-  VPValue *expand(VPExpandSCEVRecipe &R) {
-    return R.expandInIR() ? &R : expand(R.getSCEV());
-  }
 };
 
//===----------------------------------------------------------------------===//
 // Utilities for modifying predecessors and successors of VPlan blocks.

diff  --git 
a/llvm/test/Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll
 
b/llvm/test/Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll
index 7998f85cd551d7..3f0a689a76dff7 100644
--- 
a/llvm/test/Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll
+++ 
b/llvm/test/Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll
@@ -37,7 +37,7 @@ while.body:
   %i = phi i64 [ 1023, %entry ], [ %i.next, %if.end ]
   %arrayidx = getelementptr inbounds i32, ptr %data1, i64 %i
   %ld = load i32, ptr %arrayidx, align 4
-  %cmp = icmp sgt i32 %ld, 0
+  %cmp = icmp sgt i32 %ld, %ld
   br i1 %cmp, label %if.then, label %if.end
 
 if.then:

diff  --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop-
diff -checks.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop-
diff -checks.ll
index 255126a88dacc5..050547a89f29c5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop-
diff -checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop-
diff -checks.ll
@@ -10,10 +10,10 @@ define void @foo(ptr %dst, ptr %src, i32 %m, i32 %n) 
vscale_range(1,16) {
 ; CHECK-NEXT:    [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
 ; CHECK-NEXT:    [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64
-; CHECK-NEXT:    [[TMP14:%.*]] = sub i64 [[DST1]], [[SRC2]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP0]], 5
 ; CHECK-NEXT:    [[TMP12:%.*]] = add nsw i64 [[TMP11]], -1
+; CHECK-NEXT:    [[TMP14:%.*]] = sub i64 [[DST1]], [[SRC2]]
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i32 [ [[OUTER_IV_NEXT:%.*]], 
%[[INNER_EXIT:.*]] ], [ 0, %[[ENTRY]] ]

diff  --git 
a/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll 
b/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll
index d00e93f3f34eee..2c2ef39597e648 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll
@@ -35,15 +35,15 @@ define void @
diff _checks(ptr nocapture noundef writeonly %dst, ptr nocapture nou
 ; CHECK-SAME: ptr noundef writeonly captures(none) [[DST:%.*]], ptr noundef 
readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef 
[[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[ADD5:%.*]] = add nsw i32 [[N]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[M]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[N]], 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[N]], [[M]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i32 [[TMP5]], 2
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i32 [[TMP6]]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[M]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[TMP3]], [[TMP1]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[N]], [[M]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl i32 [[TMP5]], 2
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i32 [[TMP6]]
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[I_023_US:%.*]] = phi i32 [ [[INC10_US:%.*]], 
%[[INNER_LOOP_EXIT:.*]] ], [ 0, %[[ENTRY]] ]

diff  --git 
a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll 
b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
index e3e55d10f8b437..1e65b4ff539662 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
@@ -351,6 +351,8 @@ define void @multiple_uniform_stores(ptr nocapture %var1, 
ptr nocapture readonly
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[VAR1:%.*]], i64 
[[TMP3]]
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[VAR2:%.*]], i64 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[ITR]], -1
 ; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[ITR]], -1
 ; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
 ; CHECK:       for.cond1.preheader:
@@ -369,11 +371,12 @@ define void @multiple_uniform_stores(ptr nocapture %var1, 
ptr nocapture readonly
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label 
[[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP4]], 2
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[VAR2:%.*]], i64 
[[TMP9]]
-; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw nsw i64 [[TMP7]], 2
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[VAR2]], i64 
[[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP5]], [[J_022]]
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 2
 ; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP9]], [[TMP13]]
-; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP14]], 4
-; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[VAR2]], i64 
[[TMP25]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SCEVGEP2]], i64 
[[TMP14]]
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[VAR1]], [[SCEVGEP3]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]

diff  --git 
a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll 
b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
index 3a898474d59637..7687357aa8199b 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
@@ -31,23 +31,23 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 define void @Test(ptr nocapture %obj, i64 %z) #0 {
 ; CHECK-LABEL: @Test(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[Z:%.*]], 2
-; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ:%.*]], i64 
[[TMP1]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 256
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ:%.*]], i64 
[[TMP1]]
 ; CHECK-NEXT:    br label [[DOTOUTER_PREHEADER:%.*]]
 ; CHECK:       .outer.preheader:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[I_NEXT:%.*]], 
[[DOTOUTER:%.*]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr 
[[OBJ]], i64 0, i32 1, i64 [[I]]
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[Z]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label 
[[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[I]], 7
-; CHECK-NEXT:    [[TMP4:%.*]] = add nuw i64 256, [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], 256
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[OBJ]], i64 
[[TMP4]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[I]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 132, [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP6]], 132
 ; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr 
[[OBJ]], i64 0, i32 1, i64 [[I]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[Z]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label 
[[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[OBJ]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -102,11 +102,12 @@ define void @Test(ptr nocapture %obj, i64 %z) #0 {
 ; CHECK-NEXT:    br i1 [[EXITCOND_INNER]], label [[DOTOUTER]], label 
[[DOTINNER]], !llvm.loop [[LOOP11:![0-9]+]]
 ;
 ; CHECK-HOIST-LABEL: @Test(
+; CHECK-HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr 
[[OBJ:%.*]], i64 256
 ; CHECK-HOIST-NEXT:    [[TMP1:%.*]] = shl i64 [[Z:%.*]], 2
-; CHECK-HOIST-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ:%.*]], 
i64 [[TMP1]]
 ; CHECK-HOIST-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 4224
 ; CHECK-HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[OBJ]], i64 
[[TMP2]]
-; CHECK-HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[OBJ]], 
i64 256
+; CHECK-HOIST-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ]], i64 
[[TMP1]]
+; CHECK-HOIST-NEXT:    [[SCEVGEP3:%.*]] = getelementptr nuw i8, ptr [[OBJ]], 
i64 128
 ; CHECK-HOIST-NEXT:    br label [[DOTOUTER_PREHEADER:%.*]]
 ; CHECK-HOIST:       .outer.preheader:
 ; CHECK-HOIST-NEXT:    [[I:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ 
[[I_NEXT:%.*]], [[DOTOUTER:%.*]] ]
@@ -114,10 +115,11 @@ define void @Test(ptr nocapture %obj, i64 %z) #0 {
 ; CHECK-HOIST-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[Z]], 4
 ; CHECK-HOIST-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], 
label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK-HOIST:       vector.memcheck:
-; CHECK-HOIST-NEXT:    [[BOUND08:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]]
-; CHECK-HOIST-NEXT:    [[BOUND19:%.*]] = icmp ult ptr [[OBJ]], [[SCEVGEP1]]
-; CHECK-HOIST-NEXT:    [[FOUND_CONFLICT10:%.*]] = and i1 [[BOUND08]], 
[[BOUND19]]
-; CHECK-HOIST-NEXT:    br i1 [[FOUND_CONFLICT10]], label [[SCALAR_PH]], label 
[[VECTOR_PH:%.*]]
+; CHECK-HOIST-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]]
+; CHECK-HOIST-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[OBJ]], [[SCEVGEP1]]
+; CHECK-HOIST-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-HOIST-NEXT:    [[BOUND14:%.*]] = icmp ult ptr [[SCEVGEP3]], 
[[SCEVGEP1]]
+; CHECK-HOIST-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label 
[[VECTOR_PH:%.*]]
 ; CHECK-HOIST:       vector.ph:
 ; CHECK-HOIST-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[Z]], 3
 ; CHECK-HOIST-NEXT:    [[N_VEC:%.*]] = sub i64 [[Z]], [[N_MOD_VF]]

diff  --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-
diff erence-rmw-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-
diff erence-rmw-hoist.ll
index 5ca0e8c289af10..0c829e26090be0 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-
diff erence-rmw-hoist.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-
diff erence-rmw-hoist.ll
@@ -7,9 +7,9 @@ define void @rmw_variant_distance(ptr %dst, ptr %src, i64 
%rows) {
 ; HOIST-LABEL: define void @rmw_variant_distance(
 ; HOIST-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[ROWS:%.*]]) {
 ; HOIST-NEXT:  [[ENTRY:.*]]:
-; HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 64
 ; HOIST-NEXT:    [[TMP0:%.*]] = shl i64 [[ROWS]], 6
 ; HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
+; HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 64
 ; HOIST-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; HOIST:       [[OUTER_HEADER]]:
 ; HOIST-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
@@ -30,13 +30,13 @@ define void @rmw_variant_distance(ptr %dst, ptr %src, i64 
%rows) {
 ; NO-HOIST-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; NO-HOIST:       [[OUTER_HEADER]]:
 ; NO-HOIST-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
+; NO-HOIST-NEXT:    [[TMP1:%.*]] = shl i64 [[OUTER_IV]], 6
+; NO-HOIST-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 64
+; NO-HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 
[[TMP2]]
 ; NO-HOIST-NEXT:    [[OFFSET:%.*]] = mul i64 [[OUTER_IV]], 16
 ; NO-HOIST-NEXT:    [[DST_BASE:%.*]] = getelementptr inbounds i32, ptr 
[[DST]], i64 [[OFFSET]]
 ; NO-HOIST-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
 ; NO-HOIST:       [[VECTOR_MEMCHECK]]:
-; NO-HOIST-NEXT:    [[TMP0:%.*]] = shl i64 [[OUTER_IV]], 6
-; NO-HOIST-NEXT:    [[TMP1:%.*]] = add i64 64, [[TMP0]]
-; NO-HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 
[[TMP1]]
 ; NO-HOIST-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST_BASE]], [[SCEVGEP1]]
 ; NO-HOIST-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
 ; NO-HOIST-NEXT:    [[DIFF_CHECK:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -79,8 +79,8 @@ define void @rmw_invariant_distance(ptr %dst, ptr %src, i64 
%rows) {
 ; HOIST-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[ROWS:%.*]]) {
 ; HOIST-NEXT:  [[ENTRY:.*]]:
 ; HOIST-NEXT:    [[TMP0:%.*]] = shl i64 [[ROWS]], 6
-; HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
 ; HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
+; HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
 ; HOIST-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; HOIST:       [[OUTER_HEADER]]:
 ; HOIST-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
@@ -101,15 +101,15 @@ define void @rmw_invariant_distance(ptr %dst, ptr %src, 
i64 %rows) {
 ; NO-HOIST-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; NO-HOIST:       [[OUTER_HEADER]]:
 ; NO-HOIST-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
+; NO-HOIST-NEXT:    [[TMP0:%.*]] = shl i64 [[OUTER_IV]], 6
+; NO-HOIST-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 64
+; NO-HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 
[[TMP1]]
+; NO-HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 
[[TMP1]]
 ; NO-HOIST-NEXT:    [[OFFSET:%.*]] = mul i64 [[OUTER_IV]], 16
 ; NO-HOIST-NEXT:    [[DST_BASE:%.*]] = getelementptr inbounds i32, ptr 
[[DST]], i64 [[OFFSET]]
 ; NO-HOIST-NEXT:    [[SRC_BASE:%.*]] = getelementptr inbounds i32, ptr 
[[SRC]], i64 [[OFFSET]]
 ; NO-HOIST-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
 ; NO-HOIST:       [[VECTOR_MEMCHECK]]:
-; NO-HOIST-NEXT:    [[TMP0:%.*]] = shl i64 [[OUTER_IV]], 6
-; NO-HOIST-NEXT:    [[TMP1:%.*]] = add i64 64, [[TMP0]]
-; NO-HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 
[[TMP1]]
-; NO-HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 
[[TMP1]]
 ; NO-HOIST-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST_BASE]], [[SCEVGEP1]]
 ; NO-HOIST-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC_BASE]], [[SCEVGEP]]
 ; NO-HOIST-NEXT:    [[DIFF_CHECK:%.*]] = and i1 [[BOUND0]], [[BOUND1]]

diff  --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-
diff erence.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-
diff erence.ll
index c8f81b7b7cc712..89c1585382a553 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-
diff erence.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-
diff erence.ll
@@ -326,13 +326,13 @@ define void 
@nested_loop_outer_iv_addrec_invariant_in_inner1(ptr %a, ptr %b, i64
 ; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; CHECK:       [[OUTER_HEADER]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[OUTER_IV_NEXT:%.*]], 
[[OUTER_LATCH:%.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[OUTER_IV]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 4
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 
[[OUTER_IV]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label 
%[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[OUTER_IV]], 2
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 4, [[TMP1]]
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[GEP_A]], [[SCEVGEP2]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -378,13 +378,13 @@ define void 
@nested_loop_outer_iv_addrec_invariant_in_inner2(ptr %a, ptr %b, i64
 ; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; CHECK:       [[OUTER_HEADER]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[OUTER_IV_NEXT:%.*]], 
[[OUTER_LATCH:%.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[OUTER_IV]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 4
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 
[[OUTER_IV]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label 
%[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[OUTER_IV]], 2
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 4, [[TMP1]]
-; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[GEP_A]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -433,8 +433,16 @@ define void @nested_loop_bounds_are_scaled_outer_iv(ptr 
%a, ptr %b, i32 %n, i32
 ; CHECK-NEXT:    [[N_EXT:%.*]] = zext nneg i32 [[N]] to i64
 ; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; CHECK:       [[OUTER_HEADER]]:
-; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], 
[[OUTER_LATCH:%.*]] ], [ 0, %[[OUTER_PH]] ]
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], 
[[OUTER_LATCH:%.*]] ], [ 0, %[[OUTER_PH]] ]
 ; CHECK-NEXT:    [[OUTER_IV1:%.*]] = phi i64 [ 1, %[[OUTER_PH]] ], [ 
[[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl nuw nsw i64 [[OUTER_IV]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 4
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[OUTER_IV]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP1]], 8
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[OUTER_OFF_IS:%.*]] = mul nsw i64 [[OUTER_IV1]], [[IS_EXT]]
 ; CHECK-NEXT:    [[OUTER_OFF_JS:%.*]] = mul nsw i64 [[OUTER_IV1]], [[JS_EXT]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[OUTER_IV1]], 4
@@ -445,14 +453,6 @@ define void @nested_loop_bounds_are_scaled_outer_iv(ptr 
%a, ptr %b, i32 %n, i32
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
 ; CHECK-NEXT:    br i1 [[TMP2]], [[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDVAR]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = add nuw i64 4, [[TMP1]]
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[INDVAR]], 3
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 8, [[TMP4]]
-; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
-; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
-; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP4]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP3]], [[SCEVGEP2]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -518,8 +518,16 @@ define void 
@nested_loop_only_one_bound_is_scaled_outer_iv(ptr %a, ptr %b, i32 %
 ; CHECK-NEXT:    [[N_EXT:%.*]] = zext nneg i32 [[N]] to i64
 ; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; CHECK:       [[OUTER_HEADER]]:
-; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], 
[[OUTER_LATCH:%.*]] ], [ 0, %[[OUTER_PH]] ]
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], 
[[OUTER_LATCH:%.*]] ], [ 0, %[[OUTER_PH]] ]
 ; CHECK-NEXT:    [[OUTER_IV1:%.*]] = phi i64 [ 1, %[[OUTER_PH]] ], [ 
[[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul nuw nsw i64 [[OUTER_IV]], 12
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 12
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw nsw i64 [[OUTER_IV]], 24
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 16
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[OUTER_OFF_IS:%.*]] = mul nsw i64 [[OUTER_IV1]], [[IS_EXT]]
 ; CHECK-NEXT:    [[OUTER_OFF_JS:%.*]] = mul nsw i64 [[OUTER_IV1]], [[JS_EXT]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[OUTER_IV1]], 4
@@ -530,14 +538,6 @@ define void 
@nested_loop_only_one_bound_is_scaled_outer_iv(ptr %a, ptr %b, i32 %
 ; CHECK-NEXT:    [[TMP3:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
 ; CHECK-NEXT:    br i1 [[TMP3]], [[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw nsw i64 [[INDVAR]], 12
-; CHECK-NEXT:    [[TMP2:%.*]] = add nuw i64 12, [[TMP1]]
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw nsw i64 [[INDVAR]], 24
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 16, [[TMP4]]
-; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
-; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
-; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP4]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP3]], [[SCEVGEP2]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -596,22 +596,22 @@ define void 
@nested_loop_bound_needs_constant_correction(ptr %a, ptr %b, i64 %n)
 ; CHECK-LABEL: define void @nested_loop_bound_needs_constant_correction(
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[N]], 3
 ; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[N]], 2
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
 ; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; CHECK:       [[OUTER_HEADER]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[N]], [[OUTER_IV]]
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label 
%[[VECTOR_MEMCHECK:.*]]
-; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[OUTER_IV]], 2
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[OUTER_IV]], 3
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 4, [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[N]], [[OUTER_IV]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label 
%[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]

diff  --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll 
b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
index e539bfb7f6c143..492c161868fbdc 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
@@ -37,16 +37,16 @@ define void @
diff _checks(ptr nocapture noundef writeonly %dst, ptr nocapture nou
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[ADD5]] to i64
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT:    [[TMP7:%.*]] = shl nsw i64 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]]
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP8]], 2
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i64 [[WIDE_M]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[TMP3]], 2
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP4]], [[TMP5]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = shl nsw i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]]
+; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP8]], 2
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP9]]
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[IV_OUTER_NEXT:%.*]], %[[INNER_EXIT:.*]] ]
@@ -166,8 +166,8 @@ define void @full_checks(ptr nocapture noundef %dst, ptr 
nocapture noundef reado
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 2
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_IV_NEXT:%.*]], %[[INNER_EXIT:.*]] ]
@@ -284,15 +284,15 @@ define void @full_checks_
diff _strides(ptr nocapture noundef %dst, ptr nocapture
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[WIDE_M]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], [[TMP1]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_IV_NEXT:%.*]], %[[INNER_EXIT:.*]] ]
@@ -409,17 +409,17 @@ define void @
diff _checks_src_start_invariant(ptr nocapture noundef writeonly %ds
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[WIDE_N]], 2
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[IV_OUTER_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], [[IV_OUTER]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nsw i64 [[IV_OUTER]], [[TMP0]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_N]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[IV_OUTER]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP1]], [[TMP13]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[TMP4]], 1
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], 15
 ; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label 
%[[VECTOR_PH:.*]]
@@ -516,11 +516,11 @@ define void @full_checks_src_start_invariant(ptr 
nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 2
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[IV_OUTER_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
@@ -640,22 +640,24 @@ define void @triple_nested_loop_mixed_access(ptr 
nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT68:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT60:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[O]] to i64
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[TMP3]], 2
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2
 ; CHECK-NEXT:    br label %[[OUTER_OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_OUTER_LOOP]]:
 ; CHECK-NEXT:    [[OUTER_OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_OUTER_IV_NEXT:%.*]], %[[OUTER_LOOP_END:.*]] ]
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP6]], [[OUTER_OUTER_IV]]
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP3]], [[TMP13]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP4]], [[OUTER_OUTER_IV]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 
[[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP8]], [[OUTER_OUTER_IV]]
 ; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 
[[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP9]], [[TMP12]]
 ; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 
[[TMP13]]
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP8]], [[OUTER_OUTER_IV]]
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP6]], [[TMP10]]
-; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[DST]], i64 
[[TMP11]]
-; CHECK-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[DST]], i64 
[[TMP10]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = mul nsw i64 [[OUTER_OUTER_IV]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul nsw i64 [[TMP14]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul nsw i64 [[TMP14]], [[TMP2]]
@@ -667,8 +669,8 @@ define void @triple_nested_loop_mixed_access(ptr nocapture 
noundef %dst, ptr noc
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP6]], [[SCEVGEP2]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP3]], [[SCEVGEP5]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label 
%[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -803,24 +805,25 @@ define void @uncomputable_outer_tc(ptr nocapture noundef 
%dst, ptr nocapture nou
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[ADD6]] to i64
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = shl nuw nsw i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[WIDE_TRIP_COUNT]], 2
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
-; CHECK-NEXT:    [[TMP3:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PREHEADER]] ], [ 
[[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw i64 [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nsw i64 [[TMP3]], [[TMP2]]
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_MEMCHECK:.*]]
-; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PREHEADER]] ], 
[ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP3]], [[OUTER_IV]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP4]], [[TMP6]]
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP5]], [[OUTER_IV]]
 ; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP4]], [[TMP8]]
 ; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP2]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -862,7 +865,7 @@ define void @uncomputable_outer_tc(ptr nocapture noundef 
%dst, ptr nocapture nou
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 
[[WIDE_TRIP_COUNT]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[INNER_LOOP_EXIT]], label 
%[[INNER_LOOP]], !llvm.loop [[LOOP45:![0-9]+]]
 ; CHECK:       [[INNER_LOOP_EXIT]]:
-; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add i64 [[TMP3]], 1
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i8, ptr [[STR]], 
i64 [[OUTER_IV_NEXT]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX_US]], align 1
 ; CHECK-NEXT:    [[CMP_NOT_US:%.*]] = icmp eq i8 [[TMP22]], 0
@@ -953,20 +956,20 @@ define void @decreasing_inner_iv(ptr nocapture noundef 
%dst, ptr nocapture nound
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[STRIDE2]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[STRIDE1]] to i64
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[M]] to i64
-; CHECK-NEXT:    [[TMP14:%.*]] = shl nsw i64 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP9:%.*]] = shl nsw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], 4
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nsw i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP3]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = shl i64 [[TMP10]], 2
 ; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 4
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 
[[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = shl nsw i64 [[TMP1]], 2
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PRE]] ], [ 
[[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
@@ -1115,21 +1118,21 @@ define void @decreasing_outer_iv(ptr nocapture noundef 
%dst, ptr nocapture nound
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[STRIDE1]] to i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[STRIDE2]] to i64
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[TMP14:%.*]] = mul nsw i64 [[TMP3]], -4
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[TMP6:%.*]] = shl nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nsw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2
 ; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nsw i64 [[TMP2]], -4
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP3]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = shl i64 [[TMP10]], 2
 ; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 
[[TMP11]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shl nsw i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP12:%.*]] = shl nsw i64 [[TMP3]], 2
 ; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], [[TMP7]]
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 
[[TMP13]]
-; CHECK-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP2]], [[TMP1]]
-; CHECK-NEXT:    [[TMP30:%.*]] = shl i64 [[TMP29]], 2
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP30]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 
[[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nsw i64 [[TMP3]], -4
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[TMP1]], %[[OUTER_LOOP_PRE]] ], 
[ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
@@ -1273,15 +1276,15 @@ define void @unknown_inner_stride(ptr nocapture noundef 
%dst, ptr nocapture noun
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[ADD6]] to i64
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT39:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[WIDE_TRIP_COUNT]], 
[[WIDE_TRIP_COUNT39]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shl i64 [[TMP9]], 2
-; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 
[[TMP10]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT39]], -1
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2
 ; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP6]], [[TMP7]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[WIDE_TRIP_COUNT]], 
[[WIDE_TRIP_COUNT39]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shl i64 [[TMP9]], 2
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 
[[TMP10]]
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PREHEADER]] ], 
[ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
@@ -1491,8 +1494,8 @@ define void @stride_check_known_via_loop_guard(ptr %C, 
ptr %A, i32 %Acols) {
 ; CHECK-NEXT:    [[PRE_C:%.*]] = icmp ugt i32 [[ACOLS]], 0
 ; CHECK-NEXT:    br i1 [[PRE_C]], label %[[EXIT:.*]], label 
%[[OUTER_HEADER_PREHEADER:.*]]
 ; CHECK:       [[OUTER_HEADER_PREHEADER]]:
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 8
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C]], i64 8000
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 8
 ; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; CHECK:       [[OUTER_HEADER]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i32 [ [[OUTER_IV_NEXT:%.*]], 
%[[OUTER_LATCH:.*]] ], [ 0, %[[OUTER_HEADER_PREHEADER]] ]
@@ -1607,13 +1610,13 @@ define void @shared_strides(ptr noundef %dst, ptr 
noundef readonly %src1, ptr no
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[STRIDE1]] to i64
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], [[TMP0]]
-; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC2]], i64 
[[TMP5]]
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC1]], i64 
[[TMP5]]
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP3]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], [[TMP0]]
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC1]], i64 
[[TMP7]]
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC2]], i64 
[[TMP7]]
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
 ; CHECK:       [[OUTER_LOOP]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PRE]] ], [ 
[[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
@@ -1726,256 +1729,3 @@ outer.loop.exit:
 exit:
   ret void
 }
-
-; Bounds with a udiv by a non-zero, non-poison divisor are hoisted.
-define void @udiv_known_nonzero(ptr %dst, ptr %src, i64 %n, i64 noundef %x) {
-; CHECK-LABEL: define void @udiv_known_nonzero(
-; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]], i64 noundef 
[[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[S:%.*]] = or i64 [[X]], 1
-; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[S]], i64 [[N]])
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[UMAX]], -1
-; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], [[S]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 101380
-; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
-; CHECK-NEXT:    br label %[[OUTER:.*]]
-; CHECK:       [[OUTER]]:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT:    [[DST_OFF:%.*]] = mul i64 [[OUTER_IV]], 1024
-; CHECK-NEXT:    [[DST_O:%.*]] = getelementptr i8, ptr [[DST]], i64 [[DST_OFF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umax.i64(i64 [[S]], i64 [[N]])
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], -1
-; CHECK-NEXT:    [[TMP8:%.*]] = udiv i64 [[TMP7]], [[S]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add nuw i64 [[TMP8]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP9]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_SCEVCHECK:.*]]
-; CHECK:       [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp sle i64 [[S]], 0
-; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label 
%[[VECTOR_MEMCHECK:.*]]
-; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP5]]
-; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label 
%[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = and i64 [[TMP9]], 3
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP9]], [[TMP10]]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[N_VEC]], [[S]]
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 
3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP12:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 
1)
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i64> [[TMP12]], i64 0
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 
[[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i64> [[TMP12]], i64 1
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 
[[TMP15]]
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i64> [[TMP12]], i64 2
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 
[[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <4 x i64> [[TMP12]], i64 3
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 
[[TMP19]]
-; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP14]], align 4, 
!alias.scope [[META86:![0-9]+]]
-; CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[TMP16]], align 4, 
!alias.scope [[META86]]
-; CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP18]], align 4, 
!alias.scope [[META86]]
-; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP20]], align 4, 
!alias.scope [[META86]]
-; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <4 x i32> poison, i32 
[[TMP21]], i64 0
-; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x i32> [[TMP25]], i32 
[[TMP22]], i64 1
-; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <4 x i32> [[TMP26]], i32 
[[TMP23]], i64 2
-; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x i32> [[TMP27]], i32 
[[TMP24]], i64 3
-; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[DST_O]], 
i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x i32> [[TMP28]], ptr [[TMP29]], align 4, 
!alias.scope [[META89:![0-9]+]], !noalias [[META86]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat 
(i64 4)
-; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP91:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP9]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], 
%[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, 
%[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[N_VEC]], 
%[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, 
%[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label %[[INNER:.*]]
-; CHECK:       [[INNER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT:    [[K:%.*]] = phi i64 [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ], [ 
[[K_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT:    [[K_2:%.*]] = shl nuw nsw i64 [[K]], 1
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], 
i64 [[K_2]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST_O]], 
i64 [[K]]
-; CHECK-NEXT:    store i32 [[L]], ptr [[GEP_DST]], align 4
-; CHECK-NEXT:    [[K_NEXT]] = add nuw nsw i64 [[K]], 1
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[S]]
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[C]], label %[[INNER]], label %[[OUTER_LATCH]], 
!llvm.loop [[LOOP92:![0-9]+]]
-; CHECK:       [[OUTER_LATCH]]:
-; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw i64 [[OUTER_IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 100
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[OUTER]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    ret void
-;
-entry:
-  %s = or i64 %x, 1
-  br label %outer
-
-outer:
-  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
-  %dst.off = mul i64 %outer.iv, 1024
-  %dst.o = getelementptr i8, ptr %dst, i64 %dst.off
-  br label %inner
-
-inner:
-  %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
-  %k = phi i64 [ 0, %outer ], [ %k.next, %inner ]
-  %k.2 = shl nuw nsw i64 %k, 1
-  %gep.src = getelementptr inbounds i32, ptr %src, i64 %k.2
-  %l = load i32, ptr %gep.src
-  %gep.dst = getelementptr inbounds i32, ptr %dst.o, i64 %k
-  store i32 %l, ptr %gep.dst
-  %k.next = add nuw nsw i64 %k, 1
-  %iv.next = add nuw i64 %iv, %s
-  %c = icmp ult i64 %iv.next, %n
-  br i1 %c, label %inner, label %outer.latch
-
-outer.latch:
-  %outer.iv.next = add nuw i64 %outer.iv, 1
-  %ec = icmp eq i64 %outer.iv.next, 100
-  br i1 %ec, label %exit, label %outer
-
-exit:
-  ret void
-}
-
-; Bounds with a udiv by a non-zero, maybe-poison divisor are not hoisted.
-define void @udiv_known_nonzero_maybe_poison(ptr %dst, ptr %src, i64 %n, i64 
%x, i1 %g) {
-; CHECK-LABEL: define void @udiv_known_nonzero_maybe_poison(
-; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]], i64 [[X:%.*]], 
i1 [[G:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[S:%.*]] = or i64 [[X]], 1
-; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[S]], i64 [[N]])
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[UMAX]], -1
-; CHECK-NEXT:    br label %[[OUTER:.*]]
-; CHECK:       [[OUTER]]:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT:    [[DST_OFF:%.*]] = mul i64 [[OUTER_IV]], 1024
-; CHECK-NEXT:    [[DST_O:%.*]] = getelementptr i8, ptr [[DST]], i64 [[DST_OFF]]
-; CHECK-NEXT:    br i1 [[G]], label %[[INNER_PH:.*]], label %[[OUTER_LATCH]]
-; CHECK:       [[INNER_PH]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[S]], i64 [[N]])
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], -1
-; CHECK-NEXT:    [[TMP3:%.*]] = udiv i64 [[TMP2]], [[S]]
-; CHECK-NEXT:    [[TMP4:%.*]] = add nuw i64 [[TMP3]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_SCEVCHECK:.*]]
-; CHECK:       [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp sle i64 [[S]], 0
-; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label 
%[[VECTOR_MEMCHECK:.*]]
-; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[TMP5:%.*]] = udiv i64 [[TMP0]], [[S]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 101380
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP5]], 3
-; CHECK-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[TMP9]], 4
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP10]]
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[TMP11]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[TMP8]]
-; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label 
%[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP4]], 3
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP4]], [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[N_VEC]], [[S]]
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 
3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP14:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 
1)
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i64> [[TMP14]], i64 0
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 
[[TMP15]]
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i64> [[TMP14]], i64 1
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 
[[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <4 x i64> [[TMP14]], i64 2
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 
[[TMP19]]
-; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i64> [[TMP14]], i64 3
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 
[[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP16]], align 4, 
!alias.scope [[META93:![0-9]+]]
-; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP18]], align 4, 
!alias.scope [[META93]]
-; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[TMP20]], align 4, 
!alias.scope [[META93]]
-; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[TMP22]], align 4, 
!alias.scope [[META93]]
-; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <4 x i32> poison, i32 
[[TMP23]], i64 0
-; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x i32> [[TMP27]], i32 
[[TMP24]], i64 1
-; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <4 x i32> [[TMP28]], i32 
[[TMP25]], i64 2
-; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <4 x i32> [[TMP29]], i32 
[[TMP26]], i64 3
-; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[DST_O]], 
i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP31]], align 4, 
!alias.scope [[META96:![0-9]+]], !noalias [[META93]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat 
(i64 4)
-; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP98:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LATCH_LOOPEXIT:.*]], label 
%[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], 
%[[MIDDLE_BLOCK]] ], [ 0, %[[INNER_PH]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, 
%[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i64 [ [[N_VEC]], 
%[[MIDDLE_BLOCK]] ], [ 0, %[[INNER_PH]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, 
%[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label %[[INNER:.*]]
-; CHECK:       [[INNER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT:    [[K:%.*]] = phi i64 [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ 
[[K_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT:    [[K_2:%.*]] = shl nuw nsw i64 [[K]], 1
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], 
i64 [[K_2]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST_O]], 
i64 [[K]]
-; CHECK-NEXT:    store i32 [[L]], ptr [[GEP_DST]], align 4
-; CHECK-NEXT:    [[K_NEXT]] = add nuw nsw i64 [[K]], 1
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[S]]
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[C]], label %[[INNER]], label 
%[[OUTER_LATCH_LOOPEXIT]], !llvm.loop [[LOOP99:![0-9]+]]
-; CHECK:       [[OUTER_LATCH_LOOPEXIT]]:
-; CHECK-NEXT:    br label %[[OUTER_LATCH]]
-; CHECK:       [[OUTER_LATCH]]:
-; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw i64 [[OUTER_IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 100
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[OUTER]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    ret void
-;
-entry:
-  %s = or i64 %x, 1
-  br label %outer
-
-outer:
-  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
-  %dst.off = mul i64 %outer.iv, 1024
-  %dst.o = getelementptr i8, ptr %dst, i64 %dst.off
-  br i1 %g, label %inner.ph, label %outer.latch
-
-inner.ph:
-  br label %inner
-
-inner:
-  %iv = phi i64 [ 0, %inner.ph ], [ %iv.next, %inner ]
-  %k = phi i64 [ 0, %inner.ph ], [ %k.next, %inner ]
-  %k.2 = shl nuw nsw i64 %k, 1
-  %gep.src = getelementptr inbounds i32, ptr %src, i64 %k.2
-  %l = load i32, ptr %gep.src
-  %gep.dst = getelementptr inbounds i32, ptr %dst.o, i64 %k
-  store i32 %l, ptr %gep.dst
-  %k.next = add nuw nsw i64 %k, 1
-  %iv.next = add nuw i64 %iv, %s
-  %c = icmp ult i64 %iv.next, %n
-  br i1 %c, label %inner, label %outer.latch
-
-outer.latch:
-  %outer.iv.next = add nuw i64 %outer.iv, 1
-  %ec = icmp eq i64 %outer.iv.next, 100
-  br i1 %ec, label %exit, label %outer
-
-exit:
-  ret void
-}

diff  --git 
a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
 
b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
index dde04130b24cb2..b17f36a7f5ae31 100644
--- 
a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
+++ 
b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
@@ -15,13 +15,11 @@ define void @expand(ptr %src, ptr %dst, i64 %0) {
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[SMAX6:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 
1000)
 ; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[SMAX6]], 4
 ; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP0]], 4
-; CHECK-NEXT:    [[SCEVGEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP9]]
-; CHECK-NEXT:    [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 8
 ; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; CHECK:       [[OUTER_HEADER]]:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 
[[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
@@ -44,7 +42,7 @@ define void @expand(ptr %src, ptr %dst, i64 %0) {
 ; CHECK-NEXT:    br i1 [[TMP17]], label %[[SCALAR_PH]], label 
%[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP7]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP9]], [[SCEVGEP8]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP5]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label 
%[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:

diff  --git 
a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll 
b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
index 4bd8876512ce46..b43d12ecf8b3b3 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
@@ -93,12 +93,12 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull 
align 8 dereferencea
 ; CHECK-NEXT:    [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 
[[TMP1]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[I]], 225
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[CONV6]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[CONV6]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i32 [[I]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label 
[[FOR_BODY4_US_PREHEADER:%.*]], label [[FOR_COND1_PREHEADER_US_PREHEADER:%.*]]
 ; CHECK:       vector.memcheck:
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[FOR_BODY4_US_PREHEADER]], 
label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -106,7 +106,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull 
align 8 dereferencea
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP17]], 
i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP17]], align 
8, !alias.scope [[META0:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD21:%.*]] = load <2 x double>, ptr [[TMP18]], align 
8, !alias.scope [[META0]]
@@ -115,7 +115,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull 
align 8 dereferencea
 ; CHECK-NEXT:    [[BROADCAST_SPLAT23:%.*]] = shufflevector <2 x double> 
[[BROADCAST_SPLATINSERT22]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP20:%.*]] = fmul <2 x double> [[WIDE_LOAD]], 
[[BROADCAST_SPLAT23]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = fmul <2 x double> [[WIDE_LOAD21]], 
[[BROADCAST_SPLAT23]]
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP22]], 
i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD24:%.*]] = load <2 x double>, ptr [[TMP22]], align 
8, !alias.scope [[META5:![0-9]+]], !noalias [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD25:%.*]] = load <2 x double>, ptr [[TMP23]], align 
8, !alias.scope [[META5]], !noalias [[META0]]
@@ -137,13 +137,13 @@ define void @matrix_extract_insert_loop(i32 %i, ptr 
nonnull align 8 dereferencea
 ; CHECK:       for.cond1.for.cond.cleanup3_crit_edge:
 ; CHECK-NEXT:    [[TMP31:%.*]] = icmp samesign ult i32 [[I]], 210
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP31]])
-; CHECK-NEXT:    [[TMP61:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[CONV6]]
+; CHECK-NEXT:    [[TMP61:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[CONV6]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP61]], 
i64 120
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK_1:%.*]] = icmp samesign ult i32 [[I]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_1]], label 
[[FOR_BODY4_US_PREHEADER_1:%.*]], label 
[[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US:%.*]]
 ; CHECK:       vector.memcheck.1:
-; CHECK-NEXT:    [[BOUND0_1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
-; CHECK-NEXT:    [[BOUND1_1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]]
+; CHECK-NEXT:    [[BOUND0_1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]]
+; CHECK-NEXT:    [[BOUND1_1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT_1:%.*]] = and i1 [[BOUND0_1]], [[BOUND1_1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT_1]], label 
[[FOR_BODY4_US_PREHEADER_1]], label [[VECTOR_PH_1:%.*]]
 ; CHECK:       vector.ph.1:
@@ -152,7 +152,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull 
align 8 dereferencea
 ; CHECK:       vector.body.1:
 ; CHECK-NEXT:    [[INDEX_1:%.*]] = phi i64 [ 0, [[VECTOR_PH_1]] ], [ 
[[INDEX_NEXT_1:%.*]], [[VECTOR_BODY_1]] ]
 ; CHECK-NEXT:    [[TMP33:%.*]] = add nuw nsw i64 [[INDEX_1]], 15
-; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP33]]
+; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP33]]
 ; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP47]], 
i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD_1:%.*]] = load <2 x double>, ptr [[TMP47]], align 
8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD21_1:%.*]] = load <2 x double>, ptr [[TMP48]], 
align 8, !alias.scope [[META0]]
@@ -161,7 +161,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull 
align 8 dereferencea
 ; CHECK-NEXT:    [[BROADCAST_SPLAT23_1:%.*]] = shufflevector <2 x double> 
[[BROADCAST_SPLATINSERT22_1]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP50:%.*]] = fmul <2 x double> [[WIDE_LOAD_1]], 
[[BROADCAST_SPLAT23_1]]
 ; CHECK-NEXT:    [[TMP51:%.*]] = fmul <2 x double> [[WIDE_LOAD21_1]], 
[[BROADCAST_SPLAT23_1]]
-; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP33]]
+; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP33]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP52]], 
i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD24_1:%.*]] = load <2 x double>, ptr [[TMP52]], 
align 8, !alias.scope [[META5]], !noalias [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD25_1:%.*]] = load <2 x double>, ptr [[TMP53]], 
align 8, !alias.scope [[META5]], !noalias [[META0]]
@@ -183,11 +183,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr 
nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP57:%.*]] = add nuw nsw i64 [[INDVARS_IV_1]], 15
 ; CHECK-NEXT:    [[TMP58:%.*]] = icmp samesign ult i64 [[INDVARS_IV_1]], 210
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP58]])
-; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP57]]
 ; CHECK-NEXT:    [[MATRIXEXT_US_1:%.*]] = load double, ptr [[TMP59]], align 8
 ; CHECK-NEXT:    [[MATRIXEXT8_US_1:%.*]] = load double, ptr [[TMP32]], align 8
 ; CHECK-NEXT:    [[MUL_US_1:%.*]] = fmul double [[MATRIXEXT_US_1]], 
[[MATRIXEXT8_US_1]]
-; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP57]]
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP57]]
 ; CHECK-NEXT:    [[MATRIXEXT11_US_1:%.*]] = load double, ptr [[TMP60]], align 8
 ; CHECK-NEXT:    [[SUB_US_1:%.*]] = fsub double [[MATRIXEXT11_US_1]], 
[[MUL_US_1]]
 ; CHECK-NEXT:    store double [[SUB_US_1]], ptr [[TMP60]], align 8
@@ -197,13 +197,13 @@ define void @matrix_extract_insert_loop(i32 %i, ptr 
nonnull align 8 dereferencea
 ; CHECK:       for.cond1.for.cond.cleanup3_crit_edge.1:
 ; CHECK-NEXT:    [[TMP62:%.*]] = icmp samesign ult i32 [[I]], 195
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP62]])
-; CHECK-NEXT:    [[TMP92:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[CONV6]]
+; CHECK-NEXT:    [[TMP92:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[CONV6]]
 ; CHECK-NEXT:    [[TMP63:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP92]], 
i64 240
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK_2:%.*]] = icmp samesign ult i32 [[I]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_2]], label 
[[FOR_BODY4_US_PREHEADER_2:%.*]], label 
[[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_1:%.*]]
 ; CHECK:       vector.memcheck.2:
-; CHECK-NEXT:    [[BOUND0_2:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
-; CHECK-NEXT:    [[BOUND1_2:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]]
+; CHECK-NEXT:    [[BOUND0_2:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]]
+; CHECK-NEXT:    [[BOUND1_2:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT_2:%.*]] = and i1 [[BOUND0_2]], [[BOUND1_2]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT_2]], label 
[[FOR_BODY4_US_PREHEADER_2]], label [[VECTOR_PH_2:%.*]]
 ; CHECK:       vector.ph.2:
@@ -212,7 +212,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull 
align 8 dereferencea
 ; CHECK:       vector.body.2:
 ; CHECK-NEXT:    [[INDEX_2:%.*]] = phi i64 [ 0, [[VECTOR_PH_2]] ], [ 
[[INDEX_NEXT_2:%.*]], [[VECTOR_BODY_2]] ]
 ; CHECK-NEXT:    [[TMP64:%.*]] = add nuw nsw i64 [[INDEX_2]], 30
-; CHECK-NEXT:    [[TMP78:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP64]]
+; CHECK-NEXT:    [[TMP78:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP64]]
 ; CHECK-NEXT:    [[TMP79:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP78]], 
i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD_2:%.*]] = load <2 x double>, ptr [[TMP78]], align 
8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD21_2:%.*]] = load <2 x double>, ptr [[TMP79]], 
align 8, !alias.scope [[META0]]
@@ -221,7 +221,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull 
align 8 dereferencea
 ; CHECK-NEXT:    [[BROADCAST_SPLAT23_2:%.*]] = shufflevector <2 x double> 
[[BROADCAST_SPLATINSERT22_2]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP81:%.*]] = fmul <2 x double> [[WIDE_LOAD_2]], 
[[BROADCAST_SPLAT23_2]]
 ; CHECK-NEXT:    [[TMP82:%.*]] = fmul <2 x double> [[WIDE_LOAD21_2]], 
[[BROADCAST_SPLAT23_2]]
-; CHECK-NEXT:    [[TMP83:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP64]]
+; CHECK-NEXT:    [[TMP83:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP64]]
 ; CHECK-NEXT:    [[TMP84:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP83]], 
i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD24_2:%.*]] = load <2 x double>, ptr [[TMP83]], 
align 8, !alias.scope [[META5]], !noalias [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD25_2:%.*]] = load <2 x double>, ptr [[TMP84]], 
align 8, !alias.scope [[META5]], !noalias [[META0]]
@@ -243,11 +243,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr 
nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP88:%.*]] = add nuw nsw i64 [[INDVARS_IV_2]], 30
 ; CHECK-NEXT:    [[TMP89:%.*]] = icmp samesign ult i64 [[INDVARS_IV_2]], 195
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP89]])
-; CHECK-NEXT:    [[TMP90:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP88]]
 ; CHECK-NEXT:    [[MATRIXEXT_US_2:%.*]] = load double, ptr [[TMP90]], align 8
 ; CHECK-NEXT:    [[MATRIXEXT8_US_2:%.*]] = load double, ptr [[TMP63]], align 8
 ; CHECK-NEXT:    [[MUL_US_2:%.*]] = fmul double [[MATRIXEXT_US_2]], 
[[MATRIXEXT8_US_2]]
-; CHECK-NEXT:    [[TMP91:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP88]]
+; CHECK-NEXT:    [[TMP91:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP88]]
 ; CHECK-NEXT:    [[MATRIXEXT11_US_2:%.*]] = load double, ptr [[TMP91]], align 8
 ; CHECK-NEXT:    [[SUB_US_2:%.*]] = fsub double [[MATRIXEXT11_US_2]], 
[[MUL_US_2]]
 ; CHECK-NEXT:    store double [[SUB_US_2]], ptr [[TMP91]], align 8
@@ -257,13 +257,13 @@ define void @matrix_extract_insert_loop(i32 %i, ptr 
nonnull align 8 dereferencea
 ; CHECK:       for.cond1.for.cond.cleanup3_crit_edge.2:
 ; CHECK-NEXT:    [[TMP93:%.*]] = icmp samesign ult i32 [[I]], 180
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP93]])
-; CHECK-NEXT:    [[TMP123:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[CONV6]]
+; CHECK-NEXT:    [[TMP123:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[CONV6]]
 ; CHECK-NEXT:    [[TMP94:%.*]] = getelementptr inbounds nuw i8, ptr 
[[TMP123]], i64 360
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK_3:%.*]] = icmp samesign ult i32 [[I]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_3]], label 
[[FOR_BODY4_US_PREHEADER_3:%.*]], label 
[[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_2:%.*]]
 ; CHECK:       vector.memcheck.3:
-; CHECK-NEXT:    [[BOUND0_3:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
-; CHECK-NEXT:    [[BOUND1_3:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]]
+; CHECK-NEXT:    [[BOUND0_3:%.*]] = icmp ult ptr [[B]], [[SCEVGEP20]]
+; CHECK-NEXT:    [[BOUND1_3:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT_3:%.*]] = and i1 [[BOUND0_3]], [[BOUND1_3]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT_3]], label 
[[FOR_BODY4_US_PREHEADER_3]], label [[VECTOR_PH_3:%.*]]
 ; CHECK:       vector.ph.3:
@@ -272,7 +272,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull 
align 8 dereferencea
 ; CHECK:       vector.body.3:
 ; CHECK-NEXT:    [[INDEX_3:%.*]] = phi i64 [ 0, [[VECTOR_PH_3]] ], [ 
[[INDEX_NEXT_3:%.*]], [[VECTOR_BODY_3]] ]
 ; CHECK-NEXT:    [[TMP95:%.*]] = add nuw nsw i64 [[INDEX_3]], 45
-; CHECK-NEXT:    [[TMP109:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP95]]
+; CHECK-NEXT:    [[TMP109:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP95]]
 ; CHECK-NEXT:    [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr 
[[TMP109]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD_3:%.*]] = load <2 x double>, ptr [[TMP109]], 
align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD21_3:%.*]] = load <2 x double>, ptr [[TMP110]], 
align 8, !alias.scope [[META0]]
@@ -281,7 +281,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull 
align 8 dereferencea
 ; CHECK-NEXT:    [[BROADCAST_SPLAT23_3:%.*]] = shufflevector <2 x double> 
[[BROADCAST_SPLATINSERT22_3]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP112:%.*]] = fmul <2 x double> [[WIDE_LOAD_3]], 
[[BROADCAST_SPLAT23_3]]
 ; CHECK-NEXT:    [[TMP113:%.*]] = fmul <2 x double> [[WIDE_LOAD21_3]], 
[[BROADCAST_SPLAT23_3]]
-; CHECK-NEXT:    [[TMP114:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP95]]
+; CHECK-NEXT:    [[TMP114:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP95]]
 ; CHECK-NEXT:    [[TMP115:%.*]] = getelementptr inbounds nuw i8, ptr 
[[TMP114]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD24_3:%.*]] = load <2 x double>, ptr [[TMP114]], 
align 8, !alias.scope [[META5]], !noalias [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD25_3:%.*]] = load <2 x double>, ptr [[TMP115]], 
align 8, !alias.scope [[META5]], !noalias [[META0]]
@@ -303,11 +303,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr 
nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP119:%.*]] = add nuw nsw i64 [[INDVARS_IV_3]], 45
 ; CHECK-NEXT:    [[TMP120:%.*]] = icmp samesign ult i64 [[INDVARS_IV_3]], 180
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP120]])
-; CHECK-NEXT:    [[TMP121:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP119]]
+; CHECK-NEXT:    [[TMP121:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP119]]
 ; CHECK-NEXT:    [[MATRIXEXT_US_3:%.*]] = load double, ptr [[TMP121]], align 8
 ; CHECK-NEXT:    [[MATRIXEXT8_US_3:%.*]] = load double, ptr [[TMP94]], align 8
 ; CHECK-NEXT:    [[MUL_US_3:%.*]] = fmul double [[MATRIXEXT_US_3]], 
[[MATRIXEXT8_US_3]]
-; CHECK-NEXT:    [[TMP122:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[TMP119]]
+; CHECK-NEXT:    [[TMP122:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[TMP119]]
 ; CHECK-NEXT:    [[MATRIXEXT11_US_3:%.*]] = load double, ptr [[TMP122]], align 
8
 ; CHECK-NEXT:    [[SUB_US_3:%.*]] = fsub double [[MATRIXEXT11_US_3]], 
[[MUL_US_3]]
 ; CHECK-NEXT:    store double [[SUB_US_3]], ptr [[TMP122]], align 8
@@ -318,11 +318,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr 
nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], 
[[FOR_BODY4_US]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY4_US_PREHEADER]] ]
 ; CHECK-NEXT:    [[TMP68:%.*]] = icmp samesign ult i64 [[INDVARS_IV]], 225
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP68]])
-; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[MATRIXEXT:%.*]] = load double, ptr [[TMP69]], align 8
 ; CHECK-NEXT:    [[MATRIXEXT8:%.*]] = load double, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[MATRIXEXT]], [[MATRIXEXT8]]
-; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds nuw [8 x i8], ptr 
[[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[MATRIXEXT11:%.*]] = load double, ptr [[TMP70]], align 8
 ; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[MATRIXEXT11]], [[MUL]]
 ; CHECK-NEXT:    store double [[SUB]], ptr [[TMP70]], align 8


        
_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits

Reply via email to