https://github.com/lukel97 created 
https://github.com/llvm/llvm-project/pull/237152

(Creating a new PR for https://github.com/llvm/llvm-project/pull/203164 so it 
can be included in the stack)

If we have a phi with an incoming value that is also a phi, and that inner phi 
postdominates the original phi, then we can "unfold" that inner phi and look 
through its incoming values.

This is needed to fold away blend masks when representing early exits via 
control flow.

The other changes in this PR:

- When peeking through phis we no longer have the invariant that each incoming 
edge to the original phi carries a unique incoming value, so we can't do the 
reuse common postdominator block in-mask optimisation. 
- When a nested phi is unfolded, we may end with an unconditional incoming 
value, i.e. all paths from the header lead to it. In this case there'll be no 
edge mask, so handle this in the `if (InValEdges.size() == 1)` check.
- Phis inserted for the tails of reduction chains during tail folding can be 
optimized with this, but this leads to a lot of test diffs and requires some 
transforms like `handleFindLastReductions` to be updated. For now just ignore 
them

---

<sub>Stack created with <a href="https://github.com/github/gh-stack";>GitHub 
Stacks CLI</a> • <a href="https://gh.io/stacks-feedback";>Give Feedback 
💬</a></sub>

>From 0c79ba3c1d120904bdbbd983bac446433f22954d Mon Sep 17 00:00:00 2001
From: Luke Lau <[email protected]>
Date: Mon, 26 Oct 2026 17:51:51 +0800
Subject: [PATCH] [VPlan] Peek through nested phi incoming values in
 computeBlendMasks

---
 .../Transforms/Vectorize/VPlanPredicator.cpp  |  91 +++-
 .../LoopVectorize/VPlan/predicator.ll         | 395 +++++++++++++-
 .../Transforms/LoopVectorize/find-last.ll     |  81 +++
 .../Transforms/LoopVectorize/predicator.ll    | 495 ++++++++++++++++++
 .../LoopVectorize/select-cmp-blend-chain.ll   |  16 +-
 .../Transforms/LoopVectorize/uniform-blend.ll |  15 +-
 6 files changed, 1061 insertions(+), 32 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp 
b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index e54920789dda60..a5fcaa8aa1630a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -83,15 +83,37 @@ class VPPredicator {
     return VPBB->getFirstNonPhi();
   }
 
+  /// Returns where to insert new blends in \p VPBB.
+  VPBasicBlock::iterator getBlendInsertPoint(VPBasicBlock *VPBB) {
+    VPBasicBlock::iterator It = getMaskInsertPoint(VPBB);
+    if (It == VPBB->end())
+      return It;
+    // Insert after any incoming edge masks if they were already inserted.
+    for (VPBlockBase *Pred : VPBB->predecessors())
+      if (VPValue *EdgeMask = getEdgeMask(cast<VPBasicBlock>(Pred), VPBB))
+        if (VPRecipeBase *EdgeR = EdgeMask->getDefiningRecipe())
+          if (EdgeR->getParent() == VPBB &&
+              !VPDT.properlyDominates(EdgeR, &*It)) {
+            It = std::next(EdgeR->getIterator());
+            if (It == VPBB->end())
+              return It;
+          }
+    return It;
+  }
+
   using EdgeTy = std::pair<const VPBasicBlock *, const VPBasicBlock *>;
 
   /// Compute the set of edges that are "furthest up" in the CFG for each
   /// incoming value of \p Phi.
   MapVector<EdgeTy, VPValue *> computeBlendEdges(VPPhi *Phi);
 
-  /// Given a set of \p Edges that each can reach \p VPBB, return the OR of all
-  /// edges, or an equivalent block in-mask.
-  VPValue *createBlendMaskForEdges(ArrayRef<EdgeTy> Edges, VPBasicBlock *VPBB);
+  /// Given a set of \p Edges that each can reach \p VPBB, try to see if there
+  /// is a block in-mask that can be used for its blend mask. Only valid when
+  /// there's exactly one unique incoming value on the paths from Edges to 
VPBB.
+  VPValue *tryFindBlockInMask(ArrayRef<EdgeTy> Edges, VPBasicBlock *VPBB);
+
+  /// Return the OR of the edge masks of \p Edges.
+  VPValue *createBlendMaskForEdges(ArrayRef<EdgeTy> Edges);
 
 public:
   VPPredicator(VPlan &Plan)
@@ -281,6 +303,15 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
   for (auto [InVal, InVPBB] : Phi->incoming_values_and_blocks())
     AddEdge(InVPBB, Phi->getParent(), InVal);
 
+  // Don't optimize any phis used to mask tail values of reductions during tail
+  // folding until handleFindLastReductions can match the new form.
+  VPlan *Plan = Phi->getParent()->getPlan();
+  if (Phi->getParent() == Plan->getVectorLoopRegion()->getExiting() &&
+      any_of(Phi->incoming_values(), IsaPred<VPReductionPHIRecipe>))
+    return Edges;
+
+  // Phis whose incoming values have been added to Edges.
+  SmallPtrSet<const VPUser *, 4> Expanded = {Phi};
   SetVector<const VPBlockBase *> Worklist(from_range, Phi->incoming_blocks());
   while (!Worklist.empty()) {
     auto *VPBB = cast<VPBasicBlock>(Worklist.pop_back_val());
@@ -299,6 +330,24 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
     for (EdgeTy Edge : OutEdges)
       Edges.erase(Edge);
 
+    // If the value is a phi postdominated by VPBB which shares an incoming
+    // value in Edges, then look through the inner incoming values instead of
+    // propagating the phi. This deduplicates a select.
+    if (auto *InnerPhi = dyn_cast<VPPhi>(Common))
+      if (all_of(InnerPhi->users(),
+                 [&Expanded](VPUser *U) { return Expanded.contains(U); }) &&
+          VPPDT.dominates(VPBB, InnerPhi->getParent()) &&
+          any_of(InnerPhi->incoming_values(), [&Edges](VPValue *InV) {
+            return is_contained(make_second_range(Edges), InV);
+          })) {
+        Expanded.insert(InnerPhi);
+        for (auto [InV, InVPBB] : InnerPhi->incoming_values_and_blocks()) {
+          AddEdge(InVPBB, InnerPhi->getParent(), InV);
+          Worklist.insert(InVPBB);
+        }
+        continue;
+      }
+
     // Iterate up through the post dominance frontier.
     assert(VPPDF.find(VPBB) != VPPDF.end() &&
            "VPBB must have a post-dominance frontier entry");
@@ -315,8 +364,8 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
   return Edges;
 }
 
-VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
-                                               VPBasicBlock *VPBB) {
+VPValue *VPPredicator::tryFindBlockInMask(ArrayRef<EdgeTy> Edges,
+                                          VPBasicBlock *VPBB) {
   // If the nearest common postdominator to all of Edges destinations isn't 
VPBB
   // then we can use its block in-mask. E.g:
   //
@@ -329,16 +378,19 @@ VPValue 
*VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
   //     VPBB
   //
   // If the edges are A->D and B->C, PostDom will be D. We can reuse Ds block
-  // in-mask.
+  // in-mask. Only valid when there is only one unique incoming value flowing
+  // from the edges to VPBB.
   const VPBasicBlock *PostDom = Edges[0].second;
   for (auto [_, DstVPBB] : drop_begin(Edges))
     PostDom =
         cast<VPBasicBlock>(VPPDT.findNearestCommonDominator(PostDom, DstVPBB));
   assert(VPPDT.dominates(VPBB, PostDom) && "VPBB doesn't postdominate edges");
-  if (PostDom != VPBB)
-    return getBlockInMask(PostDom);
+  if (PostDom == VPBB)
+    return nullptr;
+  return getBlockInMask(PostDom);
+}
 
-  // Otherwise, compute the disjunction of edges.
+VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges) {
   VPValue *Mask = nullptr;
   for (auto [Src, ConstDst] : Edges) {
     auto *Dst = const_cast<VPBasicBlock *>(ConstDst);
@@ -354,7 +406,7 @@ VPValue 
*VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
 }
 
 void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
-  Builder.setInsertPoint(VPBB, getMaskInsertPoint(VPBB));
+  Builder.setInsertPoint(VPBB, getBlendInsertPoint(VPBB));
 
   SmallVector<VPPhi *> Phis;
   for (VPRecipeBase &R : VPBB->phis())
@@ -379,8 +431,19 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) 
{
     MapVector<VPValue *, SmallVector<EdgeTy>> InValEdgesMap;
     for (auto [Edge, Val] : computeBlendEdges(PhiR))
       InValEdgesMap[Val].push_back(Edge);
+    bool IncomingEdgesUnique =
+        all_of(PhiR->incoming_values(), [&](VPValue *InV) {
+          return is_contained(InValEdgesMap.keys(), InV);
+        });
+
     auto InValEdges = InValEdgesMap.takeVector();
 
+    if (InValEdges.size() == 1) {
+      PhiR->replaceAllUsesWith(InValEdges[0].first);
+      PhiR->eraseFromParent();
+      continue;
+    }
+
     // Sort the incoming value order to match PhiR as much as possible.
     llvm::stable_sort(InValEdges, [&PhiR](auto &L, auto &R) {
       auto InVs = PhiR->incoming_values();
@@ -391,7 +454,13 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) 
{
     SmallVector<VPValue *, 2> OperandsWithMask;
     for (const auto &[InVPV, Edges] : InValEdges) {
       OperandsWithMask.push_back(InVPV);
-      OperandsWithMask.push_back(createBlendMaskForEdges(Edges, VPBB));
+      if (IncomingEdgesUnique) {
+        if (auto *Mask = tryFindBlockInMask(Edges, VPBB)) {
+          OperandsWithMask.push_back(Mask);
+          continue;
+        }
+      }
+      OperandsWithMask.push_back(createBlendMaskForEdges(Edges));
     }
     PHINode *IRPhi = cast_or_null<PHINode>(PhiR->getUnderlyingValue());
     auto *Blend =
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll 
b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 46887ec305304c..e8629499763db8 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -665,8 +665,6 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr 
noalias %b) {
 ; CHECK-NEXT:    Successor(s): merge.a
 ; CHECK-EMPTY:
 ; CHECK-NEXT:    merge.a:
-; CHECK-NEXT:      EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c0>
-; CHECK-NEXT:      BLEND ir<%blend.a> = ir<%v1>/ir<%c0> ir<%v1>/vp<[[VP5]]>
 ; CHECK-NEXT:      EMIT ir<%d0> = icmp sgt ir<%iv>, ir<0>
 ; CHECK-NEXT:    Successor(s): if.b
 ; CHECK-EMPTY:
@@ -675,16 +673,14 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr 
noalias %b) {
 ; CHECK-NEXT:    Successor(s): if.b.inner
 ; CHECK-EMPTY:
 ; CHECK-NEXT:    if.b.inner:
-; CHECK-NEXT:      EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
+; CHECK-NEXT:      EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
 ; CHECK-NEXT:    Successor(s): merge.b.inner
 ; CHECK-EMPTY:
 ; CHECK-NEXT:    merge.b.inner:
 ; CHECK-NEXT:    Successor(s): merge.b
 ; CHECK-EMPTY:
 ; CHECK-NEXT:    merge.b:
-; CHECK-NEXT:      EMIT vp<[[VP7:%[0-9]+]]> = not ir<%d0>
-; CHECK-NEXT:      BLEND ir<%blend.b> = ir<%v2>/ir<%d0> ir<%v2>/vp<[[VP7]]>
-; CHECK-NEXT:      EMIT ir<%sum> = add ir<%blend.a>, ir<%blend.b>
+; CHECK-NEXT:      EMIT ir<%sum> = add ir<%v1>, ir<%v2>
 ; CHECK-NEXT:      EMIT store ir<%sum>, ir<%gep>
 ; CHECK-NEXT:    Successor(s): loop.latch
 ; CHECK-EMPTY:
@@ -1042,3 +1038,390 @@ latch:
 exit:
   ret void
 }
+
+define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi'
+; CHECK-NEXT:  <x1> vector loop: {
+; CHECK-NEXT:  vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT:    vector.body:
+; CHECK-NEXT:      ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT:    Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb2:
+; CHECK-NEXT:      EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT:    Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb1:
+; CHECK-NEXT:    Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb4:
+; CHECK-NEXT:      EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT:      EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]>
+; CHECK-NEXT:      EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT:      BLEND ir<%phi4> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP6]]>
+; CHECK-NEXT:    Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb3:
+; CHECK-NEXT:      EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT:    Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb5:
+; CHECK-NEXT:      BLEND ir<%phi5> = ir<%x>/ir<%c0> ir<%y>/vp<[[VP4]]>
+; CHECK-NEXT:      EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT:      EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT:      EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT:      EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, 
vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT:    No successors
+; CHECK-NEXT:  }
+; CHECK-NEXT:  Successor(s): middle.block
+;
+;     bb0
+;     /  \
+;    bb1  \
+;   /  | bb2
+;   \  | /
+;  bb3 bb4
+;    \/
+;    bb5
+entry:
+  br label %bb0
+
+bb0:
+  %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+  br i1 %c0, label %bb1, label %bb2
+
+bb1:
+  br i1 %c1, label %bb3, label %bb4
+
+bb2:
+  br label %bb4
+
+bb3:
+  br label %bb5
+
+bb4:
+  %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+  br label %bb5
+
+bb5:
+  %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+  %gep = getelementptr i32, ptr %p, i32 %iv
+  store i32 %phi5, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 128
+  br i1 %ec, label %exit, label %bb0
+
+exit:
+  ret void
+}
+
+; Same as the test above, but shouldn't look thru the phi because it has
+; multiple uses.
+define void @look_thru_phi_multi_use(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_multi_use'
+; CHECK-NEXT:  <x1> vector loop: {
+; CHECK-NEXT:  vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT:    vector.body:
+; CHECK-NEXT:      ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT:      EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT:    Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb2:
+; CHECK-NEXT:      EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT:    Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb1:
+; CHECK-NEXT:    Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb4:
+; CHECK-NEXT:      EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT:      EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]>
+; CHECK-NEXT:      EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT:      BLEND ir<%phi4> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP6]]>
+; CHECK-NEXT:      EMIT store ir<%phi4>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT:    Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb3:
+; CHECK-NEXT:      EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT:    Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb5:
+; CHECK-NEXT:      BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<%x>/vp<[[VP8]]>
+; CHECK-NEXT:      EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT:      EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT:      EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, 
vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT:    No successors
+; CHECK-NEXT:  }
+; CHECK-NEXT:  Successor(s): middle.block
+;
+;     bb0
+;     /  \
+;    bb1  \
+;   /  | bb2
+;   \  | /
+;  bb3 bb4
+;    \/
+;    bb5
+entry:
+  br label %bb0
+
+bb0:
+  %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+  %gep = getelementptr i32, ptr %p, i32 %iv
+  br i1 %c0, label %bb1, label %bb2
+
+bb1:
+  br i1 %c1, label %bb3, label %bb4
+
+bb2:
+  br label %bb4
+
+bb3:
+  br label %bb5
+
+bb4:
+  %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+  store i32 %phi4, ptr %gep
+  br label %bb5
+
+bb5:
+  %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+  store i32 %phi5, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 128
+  br i1 %ec, label %exit, label %bb0
+
+exit:
+  ret void
+}
+
+; Don't look through %phi3 as it doesn't share any incoming values with %phi4.
+define void @look_thru_phi_no_shared_values(i1 %c0, i1 %c1, i32 %x, i32 %y, 
i32 %z, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_no_shared_values'
+; CHECK-NEXT:  <x1> vector loop: {
+; CHECK-NEXT:  vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT:    vector.body:
+; CHECK-NEXT:      ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT:    Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb1:
+; CHECK-NEXT:      EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT:    Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb2:
+; CHECK-NEXT:      EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
+; CHECK-NEXT:    Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb3:
+; CHECK-NEXT:      EMIT vp<[[VP6:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT:      EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, 
vp<[[VP6]]>
+; CHECK-NEXT:      BLEND ir<%phi3> = ir<%z>/vp<[[VP5]]> ir<%y>/vp<[[VP7]]>
+; CHECK-NEXT:    Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb4:
+; CHECK-NEXT:      BLEND ir<%phi4> = ir<%phi3>/vp<[[VP4]]> ir<%x>/ir<%c0>
+; CHECK-NEXT:      EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT:      EMIT store ir<%phi4>, ir<%gep>
+; CHECK-NEXT:      EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT:      EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, 
vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT:    No successors
+; CHECK-NEXT:  }
+; CHECK-NEXT:  Successor(s): middle.block
+;
+;     bb0
+;     |  \
+;     |  bb1
+;     |  |  \
+;     |  |  bb2
+;     |  |  /
+;     |  bb3
+;     |  /
+;     bb4
+entry:
+  br label %bb0
+
+bb0:
+  %iv = phi i32 [0, %entry], [%iv.next, %bb4]
+  br i1 %c0, label %bb4, label %bb1
+
+bb1:
+  br i1 %c1, label %bb2, label %bb3
+
+bb2:
+  br label %bb3
+
+bb3:
+  %phi3 = phi i32 [%y, %bb1], [%z, %bb2]
+  br label %bb4
+
+bb4:
+  %phi4 = phi i32 [%x, %bb0], [%phi3, %bb3]
+  %gep = getelementptr i32, ptr %p, i32 %iv
+  store i32 %phi4, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 128
+  br i1 %ec, label %exit, label %bb0
+
+exit:
+  ret void
+}
+
+; bb4 will create the edge mask bb1->bb3 before bb3. Make sure bb4
+; inserts its blend after the already created edge mask.
+define i32 @look_thru_phi_blend_insert_pos(i1 %c0, i1 %c1) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_blend_insert_pos'
+; CHECK-NEXT:  <x1> vector loop: {
+; CHECK-NEXT:  vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT:    vector.body:
+; CHECK-NEXT:      ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT:    Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb1:
+; CHECK-NEXT:      EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT:    Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb2:
+; CHECK-NEXT:      EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT:      EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, 
vp<[[VP5]]>
+; CHECK-NEXT:    Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb3:
+; CHECK-NEXT:      EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
+; CHECK-NEXT:      BLEND ir<%phi3> = ir<2>/vp<[[VP6]]> ir<1>/vp<[[VP7]]>
+; CHECK-NEXT:    Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb4:
+; CHECK-NEXT:      EMIT vp<[[VP8:%[0-9]+]]> = or ir<%c0>, vp<[[VP7]]>
+; CHECK-NEXT:      BLEND ir<%phi4> = ir<1>/vp<[[VP8]]> ir<2>/vp<[[VP6]]>
+; CHECK-NEXT:      EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT:      EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, 
vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT:    No successors
+; CHECK-NEXT:  }
+; CHECK-NEXT:  Successor(s): middle.block
+;
+;     bb0
+;    /  \
+;   bb1  \
+;  /  \   \
+; bb2 |   /
+;  \  /  /
+;  bb3  /
+;    \ /
+;    bb4
+entry:
+  br label %bb0
+
+bb0:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+  br i1 %c0, label %bb4, label %bb1
+
+bb1:
+  br i1 %c1, label %bb3, label %bb2
+
+bb2:
+  br label %bb3
+
+bb3:
+  %phi3 = phi i32 [ 1, %bb1 ], [ 2, %bb2 ]
+  br label %bb4
+
+bb4:
+  %phi4 = phi i32 [ 1, %bb0 ], [ %phi3, %bb3 ]
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 128
+  br i1 %ec, label %exit, label %bb0
+
+exit:
+  ret i32 %phi4
+}
+
+define void @diamond_phi2_look_thru_phi(ptr %a, i1 %c1, i1 %c2) {
+; CHECK-LABEL: VPlan for loop in 'diamond_phi2_look_thru_phi'
+; CHECK-NEXT:  <x1> vector loop: {
+; CHECK-NEXT:  vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT:    vector.body:
+; CHECK-NEXT:      ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, 
vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT:      EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT:    Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb2:
+; CHECK-NEXT:      EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT:    Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb1:
+; CHECK-NEXT:    Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb4:
+; CHECK-NEXT:      EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c2>
+; CHECK-NEXT:      EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT:      EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
+; CHECK-NEXT:      BLEND ir<%phi4> = ir<1>/vp<[[VP5]]> ir<0>/vp<[[VP6]]>
+; CHECK-NEXT:    Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT:    bb5:
+; CHECK-NEXT:      EMIT vp<[[VP8:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT:      EMIT vp<[[VP9:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP8]]>
+; CHECK-NEXT:      EMIT vp<[[VP10:%[0-9]+]]> = not ir<%c2>
+; CHECK-NEXT:      EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP4]]>, 
vp<[[VP10]]>
+; CHECK-NEXT:      BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<3>/vp<[[VP11]]> 
ir<2>/vp<[[VP9]]>
+; CHECK-NEXT:      EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT:      EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT:      EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT:      EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, 
vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT:    No successors
+; CHECK-NEXT:  }
+; CHECK-NEXT:  Successor(s): middle.block
+;
+;          bb0
+;         /  \
+;       bb1  bb2
+;       | \  / |
+;       |  bb4 |
+;        \  | /
+;          bb5
+entry:
+  br label %bb0
+
+bb0:
+  %iv = phi i64 [0, %entry], [%iv.next, %bb5]
+  %c0 = icmp sle i64 %iv, 0
+  br i1 %c0, label %bb1, label %bb2
+
+bb1:
+  %add1 = add i64 %iv, 1
+  br i1 %c1, label %bb4, label %bb5
+
+bb2:
+  %add2 = add i64 %iv, 2
+  br i1 %c2, label %bb4, label %bb5
+
+bb4:
+  %phi4 = phi i64 [0, %bb1], [1, %bb2]
+  br label %bb5
+
+bb5:
+  %phi5 = phi i64 [2, %bb1], [3, %bb2], [%phi4, %bb4]
+  %gep = getelementptr i64, ptr %a, i64 %iv
+  store i64 %phi5, ptr %gep
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 128
+  br i1 %ec, label %exit, label %bb0
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/find-last.ll 
b/llvm/test/Transforms/LoopVectorize/find-last.ll
index 9948f34e4571b6..35cf71099b101c 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last.ll
@@ -445,3 +445,84 @@ loop.latch:
 exit:
   ret i32 %rdx.next
 }
+
+define i32 @find_last_phi_of_two_updates(ptr noalias %a, ptr noalias %b, i64 
%n) {
+; CHECK-LABEL: define i32 @find_last_phi_of_two_updates(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), 
%[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] 
], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 2)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], 
zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP6]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP7]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> 
[[WIDE_LOAD1]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP11:%.*]] = freeze <4 x i1> [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP11]])
+; CHECK-NEXT:    [[TMP13]] = select i1 [[TMP12]], <4 x i1> [[TMP11]], <4 x i1> 
[[TMP1]]
+; CHECK-NEXT:    [[TMP14]] = select i1 [[TMP12]], <4 x i32> [[PREDPHI]], <4 x 
i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP16:%.*]] = call i32 
@llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP14]], <4 x 
i1> [[TMP13]], i32 -1)
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], 
%[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] 
], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %rdx = phi i32 [ -1, %entry ], [ %rdx.next, %loop.latch ]
+  %gep.a = getelementptr i32, ptr %a, i64 %iv
+  %ld.a = load i32, ptr %gep.a
+  %gep.b = getelementptr i32, ptr %b, i64 %iv
+  %ld.b = load i32, ptr %gep.b
+  %c.0 = icmp sgt i32 %ld.a, 5
+  br i1 %c.0, label %then, label %else
+
+then:
+  %c.1 = icmp ne i32 %ld.b, 0
+  br i1 %c.1, label %merge, label %update
+
+else:
+  %c.2 = icmp ne i32 %ld.a, 2
+  br i1 %c.2, label %merge, label %update
+
+update:
+  %val = phi i32 [ %ld.b, %then ], [ %ld.a, %else ]
+  br label %merge
+
+merge:
+  %rdx.next = phi i32 [ %rdx, %then ], [ %rdx, %else ], [ %val, %update ]
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret i32 %rdx.next
+}
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll 
b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 8a7b4fb2f71816..53d853dd86dff9 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -362,3 +362,498 @@ latch:
 exit:
   ret void
 }
+
+define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr 
[[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> 
poison, i32 [[Y]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> 
poison, i32 [[X]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI7:%.*]] = select i1 [[C0]], <4 x i32> 
[[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT:    store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+;     bb0
+;     /  \
+;    bb1  \
+;   /  | bb2
+;   \  | /
+;  bb3 bb4
+;    \/
+;    bb5
+entry:
+  br label %bb0
+
+bb0:
+  %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+  br i1 %c0, label %bb1, label %bb2
+
+bb1:
+  br i1 %c1, label %bb3, label %bb4
+
+bb2:
+  br label %bb4
+
+bb3:
+  br label %bb5
+
+bb4:
+  %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+  br label %bb5
+
+bb5:
+  %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+  %gep = getelementptr i32, ptr %p, i32 %iv
+  store i32 %phi5, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 128
+  br i1 %ec, label %exit, label %bb0
+
+exit:
+  ret void
+}
+
+; Same as the test above, but shouldn't look thru the phi because it has
+; multiple uses.
+define void @look_thru_phi_multi_use(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi_multi_use(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr 
[[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> 
poison, i32 [[Y]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> 
poison, i32 [[X]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i1> 
poison, i1 [[C1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i1> 
[[BROADCAST_SPLATINSERT5]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i1> 
poison, i1 [[C0]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i1> 
[[BROADCAST_SPLATINSERT6]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT7]], splat (i1 
true)
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT6]], splat (i1 
true)
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT7]], <4 x i1> 
[[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i1> [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> 
[[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> 
[[PREDPHI]], <4 x i32> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[PREDPHI]], 
ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT:    store <4 x i32> [[PREDPHI7]], ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+;     bb0
+;     /  \
+;    bb1  \
+;   /  | bb2
+;   \  | /
+;  bb3 bb4
+;    \/
+;    bb5
+entry:
+  br label %bb0
+
+bb0:
+  %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+  %gep = getelementptr i32, ptr %p, i32 %iv
+  br i1 %c0, label %bb1, label %bb2
+
+bb1:
+  br i1 %c1, label %bb3, label %bb4
+
+bb2:
+  br label %bb4
+
+bb3:
+  br label %bb5
+
+bb4:
+  %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+  store i32 %phi4, ptr %gep
+  br label %bb5
+
+bb5:
+  %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+  store i32 %phi5, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 128
+  br i1 %ec, label %exit, label %bb0
+
+exit:
+  ret void
+}
+
+; Don't look through %phi3 as it doesn't share any incoming values with %phi4.
+define void @look_thru_phi_no_shared_values(i1 %c0, i1 %c1, i32 %x, i32 %y, 
i32 %z, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi_no_shared_values(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], i32 
[[Z:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> 
poison, i32 [[X]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> 
poison, i32 [[Z]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> 
poison, i32 [[Y]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select i1 [[C1]], <4 x i32> 
[[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT4]]
+; CHECK-NEXT:    [[PREDPHI5:%.*]] = select i1 [[C0]], <4 x i32> 
[[BROADCAST_SPLAT]], <4 x i32> [[PREDPHI]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT:    store <4 x i32> [[PREDPHI5]], ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+;     bb0
+;     |  \
+;     |  bb1
+;     |  |  \
+;     |  |  bb2
+;     |  |  /
+;     |  bb3
+;     |  /
+;     bb4
+entry:
+  br label %bb0
+
+bb0:
+  %iv = phi i32 [0, %entry], [%iv.next, %bb4]
+  br i1 %c0, label %bb4, label %bb1
+
+bb1:
+  br i1 %c1, label %bb2, label %bb3
+
+bb2:
+  br label %bb3
+
+bb3:
+  %phi3 = phi i32 [%y, %bb1], [%z, %bb2]
+  br label %bb4
+
+bb4:
+  %phi4 = phi i32 [%x, %bb0], [%phi3, %bb3]
+  %gep = getelementptr i32, ptr %p, i32 %iv
+  store i32 %phi4, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 128
+  br i1 %ec, label %exit, label %bb0
+
+exit:
+  ret void
+}
+
+; bb4 will create the edge mask bb1->bb3 before bb3. Make sure bb4
+; inserts its blend after the already created edge mask.
+define i32 @look_thru_phi_blend_insert_pos(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @look_thru_phi_blend_insert_pos(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> 
poison, i1 [[C1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> 
[[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> 
poison, i1 [[C0]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> 
[[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 
true)
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 
true)
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP1]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> splat 
(i32 2), <4 x i32> splat (i32 1)
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+;     bb0
+;    /  \
+;   bb1  \
+;  /  \   \
+; bb2 |   /
+;  \  /  /
+;  bb3  /
+;    \ /
+;    bb4
+entry:
+  br label %bb0
+
+bb0:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+  br i1 %c0, label %bb4, label %bb1
+
+bb1:
+  br i1 %c1, label %bb3, label %bb2
+
+bb2:
+  br label %bb3
+
+bb3:
+  %phi3 = phi i32 [ 1, %bb1 ], [ 2, %bb2 ]
+  br label %bb4
+
+bb4:
+  %phi4 = phi i32 [ 1, %bb0 ], [ %phi3, %bb3 ]
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 128
+  br i1 %ec, label %exit, label %bb0
+
+exit:
+  ret i32 %phi4
+}
+
+define void @diamond_phi2_look_thru_phi(ptr %a, i1 %c1, i1 %c2) {
+; CHECK-LABEL: define void @diamond_phi2_look_thru_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> 
poison, i1 [[C1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> 
[[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> 
poison, i1 [[C2]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> 
[[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 
true)
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 
true)
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 
3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> 
[[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> 
zeroinitializer, <4 x i64> splat (i64 1)
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP0]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI3:%.*]] = select <4 x i1> [[TMP9]], <4 x i64> splat 
(i64 3), <4 x i64> [[PREDPHI]]
+; CHECK-NEXT:    [[PREDPHI4:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> splat 
(i64 2), <4 x i64> [[PREDPHI3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i64> [[PREDPHI4]], ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat 
(i64 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+;          bb0
+;         /  \
+;       bb1  bb2
+;       | \  / |
+;       |  bb4 |
+;        \  | /
+;          bb5
+entry:
+  br label %bb0
+
+bb0:
+  %iv = phi i64 [0, %entry], [%iv.next, %bb5]
+  %c0 = icmp sle i64 %iv, 0
+  br i1 %c0, label %bb1, label %bb2
+
+bb1:
+  %add1 = add i64 %iv, 1
+  br i1 %c1, label %bb4, label %bb5
+
+bb2:
+  %add2 = add i64 %iv, 2
+  br i1 %c2, label %bb4, label %bb5
+
+bb4:
+  %phi4 = phi i64 [0, %bb1], [1, %bb2]
+  br label %bb5
+
+bb5:
+  %phi5 = phi i64 [2, %bb1], [3, %bb2], [%phi4, %bb4]
+  %gep = getelementptr i64, ptr %a, i64 %iv
+  store i64 %phi5, ptr %gep
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 128
+  br i1 %ec, label %exit, label %bb0
+
+exit:
+  ret void
+}
+
+define void @blend_look_through_no_shared_values(ptr noalias %dst, ptr noalias 
%conds, i32 %x, i32 %y, i32 %u, i32 %w, i64 %n) {
+; CHECK-LABEL: define void @blend_look_through_no_shared_values(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[CONDS:%.*]], i32 
[[X:%.*]], i32 [[Y:%.*]], i32 [[U:%.*]], i32 [[W:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> 
poison, i32 [[U]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> 
poison, i32 [[W]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> 
poison, i32 [[X]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> 
poison, i32 [[Y]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> 
[[BROADCAST_SPLATINSERT5]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[CONDS]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <4 x i8> [[WIDE_LOAD]] to <4 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc <4 x i8> [[TMP3]] to <4 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 2)
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc <4 x i8> [[TMP5]] to <4 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 3)
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc <4 x i8> [[TMP7]] to <4 x i1>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> 
[[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP9]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP6]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> 
[[BROADCAST_SPLAT6]], <4 x i32> [[PREDPHI]]
+; CHECK-NEXT:    [[PREDPHI8:%.*]] = select <4 x i1> [[TMP13]], <4 x i32> 
[[BROADCAST_SPLAT4]], <4 x i32> [[PREDPHI7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i32> [[PREDPHI8]], ptr [[TMP14]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], 
%[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP_CONDS:%.*]] = getelementptr i8, ptr [[CONDS]], i64 
[[IV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i8, ptr [[GEP_CONDS]], align 1
+; CHECK-NEXT:    [[C_0:%.*]] = trunc i8 [[LD]] to i1
+; CHECK-NEXT:    [[SHR_1:%.*]] = lshr i8 [[LD]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = trunc i8 [[SHR_1]] to i1
+; CHECK-NEXT:    [[SHR_2:%.*]] = lshr i8 [[LD]], 2
+; CHECK-NEXT:    [[C_2:%.*]] = trunc i8 [[SHR_2]] to i1
+; CHECK-NEXT:    [[SHR_3:%.*]] = lshr i8 [[LD]], 3
+; CHECK-NEXT:    [[C_3:%.*]] = trunc i8 [[SHR_3]] to i1
+; CHECK-NEXT:    br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br i1 [[C_1]], label %[[THEN_THEN:.*]], label 
%[[THEN_ELSE:.*]]
+; CHECK:       [[THEN_THEN]]:
+; CHECK-NEXT:    br i1 [[C_2]], label %[[USE_X:.*]], label %[[USE_Y:.*]]
+; CHECK:       [[THEN_ELSE]]:
+; CHECK-NEXT:    br label %[[THEN_ELSE_CONT:.*]]
+; CHECK:       [[THEN_ELSE_CONT]]:
+; CHECK-NEXT:    br label %[[USE_X]]
+; CHECK:       [[USE_X]]:
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[USE_Y]]:
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br i1 [[C_3]], label %[[ELSE_THEN:.*]], label 
%[[ELSE_ELSE:.*]]
+; CHECK:       [[ELSE_THEN]]:
+; CHECK-NEXT:    br label %[[ELSE_MERGE:.*]]
+; CHECK:       [[ELSE_ELSE]]:
+; CHECK-NEXT:    br label %[[ELSE_MERGE]]
+; CHECK:       [[ELSE_MERGE]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[U]], %[[ELSE_THEN]] ], [ [[W]], 
%[[ELSE_ELSE]] ]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[X]], %[[USE_X]] ], [ [[Y]], 
%[[USE_Y]] ], [ [[P]], %[[ELSE_MERGE]] ]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 [[MERGE]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], 
!llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep.conds = getelementptr i8, ptr %conds, i64 %iv
+  %ld = load i8, ptr %gep.conds
+  %c.0 = trunc i8 %ld to i1
+  %shr.1 = lshr i8 %ld, 1
+  %c.1 = trunc i8 %shr.1 to i1
+  %shr.2 = lshr i8 %ld, 2
+  %c.2 = trunc i8 %shr.2 to i1
+  %shr.3 = lshr i8 %ld, 3
+  %c.3 = trunc i8 %shr.3 to i1
+  br i1 %c.0, label %then, label %else
+
+then:
+  br i1 %c.1, label %then.then, label %then.else
+
+then.then:
+  br i1 %c.2, label %use.x, label %use.y
+
+then.else:
+  br label %then.else.cont
+
+then.else.cont:
+  br label %use.x
+
+use.x:
+  br label %loop.latch
+
+use.y:
+  br label %loop.latch
+
+else:
+  br i1 %c.3, label %else.then, label %else.else
+
+else.then:
+  br label %else.merge
+
+else.else:
+  br label %else.merge
+
+else.merge:
+  %p = phi i32 [ %u, %else.then ], [ %w, %else.else ]
+  br label %loop.latch
+
+loop.latch:
+  %merge = phi i32 [ %x, %use.x ], [ %y, %use.y ], [ %p, %else.merge ]
+  %gep.dst = getelementptr i32, ptr %dst, i64 %iv
+  store i32 %merge, ptr %gep.dst
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll 
b/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
index 6cae52e2a8f283..e1fa8237cb9633 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
@@ -97,15 +97,25 @@ define i32 @anyof_three_blend_chain(i1 %c0, i1 %c1, i1 %c2, 
i32 %n) {
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[SMAX]], 3
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[SMAX]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> 
poison, i1 [[C2]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> 
[[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> 
poison, i1 [[C1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> 
[[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i1> 
poison, i1 [[C0]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i1> 
[[BROADCAST_SPLATINSERT3]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT4]], splat (i1 
true)
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 
true)
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP3]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 
true)
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP10]], 
<4 x i1> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 
3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, 
%[[VECTOR_PH]] ], [ [[PREDPHI2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i32> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i1> [[VEC_PHI]], [[TMP4]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select i1 [[C2]], <4 x i1> [[VEC_PHI]], <4 
x i1> [[TMP3]]
-; CHECK-NEXT:    [[PREDPHI1:%.*]] = select i1 [[C1]], <4 x i1> [[VEC_PHI]], <4 
x i1> [[PREDPHI]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP4]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI1:%.*]] = or <4 x i1> [[VEC_PHI]], [[TMP8]]
 ; CHECK-NEXT:    [[PREDPHI2]] = select i1 [[C0]], <4 x i1> [[VEC_PHI]], <4 x 
i1> [[PREDPHI1]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll 
b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index 8d86c8ca2bd9d7..a016a26d479ec7 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -96,19 +96,10 @@ define void @blend_chain_iv(i1 %c) {
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, 
i16 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, 
i16 0, i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, 
i16 0, i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, 
i16 0, i64 [[TMP7]]
-; CHECK-NEXT:    store i16 0, ptr [[TMP2]], align 2
-; CHECK-NEXT:    store i16 0, ptr [[TMP4]], align 2
-; CHECK-NEXT:    store i16 0, ptr [[TMP6]], align 2
-; CHECK-NEXT:    store i16 0, ptr [[TMP8]], align 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr [[TMP8]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP7]], 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
 ; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:

_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits

Reply via email to