https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/237152
>From 699a256c5c72301870d397800ef1288b5ff660ca Mon Sep 17 00:00:00 2001 From: Luke Lau <[email protected]> Date: Mon, 26 Oct 2026 17:51:51 +0800 Subject: [PATCH] [VPlan] Peek through nested phi incoming values in computeBlendMasks --- .../Transforms/Vectorize/VPlanPredicator.cpp | 91 +++- .../LoopVectorize/VPlan/predicator.ll | 395 +++++++++++++- .../Transforms/LoopVectorize/find-last.ll | 81 +++ .../Transforms/LoopVectorize/predicator.ll | 495 ++++++++++++++++++ .../LoopVectorize/select-cmp-blend-chain.ll | 16 +- .../Transforms/LoopVectorize/uniform-blend.ll | 15 +- 6 files changed, 1061 insertions(+), 32 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp index a12527775aa1a3..e0dbc41063921e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp @@ -83,15 +83,37 @@ class VPPredicator { return VPBB->getFirstNonPhi(); } + /// Returns where to insert new blends in \p VPBB. + VPBasicBlock::iterator getBlendInsertPoint(VPBasicBlock *VPBB) { + VPBasicBlock::iterator It = getMaskInsertPoint(VPBB); + if (It == VPBB->end()) + return It; + // Insert after any incoming edge masks if they were already inserted. + for (VPBlockBase *Pred : VPBB->predecessors()) + if (VPValue *EdgeMask = getEdgeMask(cast<VPBasicBlock>(Pred), VPBB)) + if (VPRecipeBase *EdgeR = EdgeMask->getDefiningRecipe()) + if (EdgeR->getParent() == VPBB && + !VPDT.properlyDominates(EdgeR, &*It)) { + It = std::next(EdgeR->getIterator()); + if (It == VPBB->end()) + return It; + } + return It; + } + using EdgeTy = std::pair<const VPBasicBlock *, const VPBasicBlock *>; /// Compute the set of edges that are "furthest up" in the CFG for each /// incoming value of \p Phi. MapVector<EdgeTy, VPValue *> computeBlendEdges(VPPhi *Phi); - /// Given a set of \p Edges that each can reach \p VPBB, return the OR of all - /// edges, or an equivalent block in-mask. - VPValue *createBlendMaskForEdges(ArrayRef<EdgeTy> Edges, VPBasicBlock *VPBB); + /// Given a set of \p Edges that each can reach \p VPBB, try to see if there + /// is a block in-mask that can be used for its blend mask. Only valid when + /// there's exactly one unique incoming value on the paths from Edges to VPBB. + VPValue *tryFindBlockInMask(ArrayRef<EdgeTy> Edges, VPBasicBlock *VPBB); + + /// Return the OR of the edge masks of \p Edges. + VPValue *createBlendMaskForEdges(ArrayRef<EdgeTy> Edges); public: VPPredicator(VPlan &Plan) @@ -281,6 +303,15 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) { for (auto [InVal, InVPBB] : Phi->incoming_values_and_blocks()) AddEdge(InVPBB, Phi->getParent(), InVal); + // Don't optimize any phis used to mask tail values of reductions during tail + // folding until handleFindLastReductions can match the new form. + VPlan *Plan = Phi->getParent()->getPlan(); + if (Phi->getParent() == Plan->getVectorLoopRegion()->getExiting() && + any_of(Phi->incoming_values(), IsaPred<VPReductionPHIRecipe>)) + return Edges; + + // Phis whose incoming values have been added to Edges. + SmallPtrSet<const VPUser *, 4> Expanded = {Phi}; SetVector<const VPBlockBase *> Worklist(from_range, Phi->incoming_blocks()); while (!Worklist.empty()) { auto *VPBB = cast<VPBasicBlock>(Worklist.pop_back_val()); @@ -299,6 +330,24 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) { for (EdgeTy Edge : OutEdges) Edges.erase(Edge); + // If the value is a phi postdominated by VPBB which shares an incoming + // value in Edges, then look through the inner incoming values instead of + // propagating the phi. This deduplicates a select. + if (auto *InnerPhi = dyn_cast<VPPhi>(Common)) + if (all_of(InnerPhi->users(), + [&Expanded](VPUser *U) { return Expanded.contains(U); }) && + VPPDT.dominates(VPBB, InnerPhi->getParent()) && + any_of(InnerPhi->incoming_values(), [&Edges](VPValue *InV) { + return is_contained(make_second_range(Edges), InV); + })) { + Expanded.insert(InnerPhi); + for (auto [InV, InVPBB] : InnerPhi->incoming_values_and_blocks()) { + AddEdge(InVPBB, InnerPhi->getParent(), InV); + Worklist.insert(InVPBB); + } + continue; + } + // Iterate up through the post dominance frontier. assert(VPPDF.find(VPBB) != VPPDF.end() && "VPBB must have a post-dominance frontier entry"); @@ -315,8 +364,8 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) { return Edges; } -VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges, - VPBasicBlock *VPBB) { +VPValue *VPPredicator::tryFindBlockInMask(ArrayRef<EdgeTy> Edges, + VPBasicBlock *VPBB) { // If the nearest common postdominator to all of Edges destinations isn't VPBB // then we can use its block in-mask. E.g: // @@ -329,16 +378,19 @@ VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges, // VPBB // // If the edges are A->D and B->C, PostDom will be D. We can reuse Ds block - // in-mask. + // in-mask. Only valid when there is only one unique incoming value flowing + // from the edges to VPBB. const VPBasicBlock *PostDom = Edges[0].second; for (auto [_, DstVPBB] : drop_begin(Edges)) PostDom = cast<VPBasicBlock>(VPPDT.findNearestCommonDominator(PostDom, DstVPBB)); assert(VPPDT.dominates(VPBB, PostDom) && "VPBB doesn't postdominate edges"); - if (PostDom != VPBB) - return getBlockInMask(PostDom); + if (PostDom == VPBB) + return nullptr; + return getBlockInMask(PostDom); +} - // Otherwise, compute the disjunction of edges. +VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges) { VPValue *Mask = nullptr; for (auto [Src, ConstDst] : Edges) { auto *Dst = const_cast<VPBasicBlock *>(ConstDst); @@ -354,7 +406,7 @@ VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges, } void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) { - Builder.setInsertPoint(getMaskInsertPoint(VPBB)); + Builder.setInsertPoint(getBlendInsertPoint(VPBB)); SmallVector<VPPhi *> Phis; for (VPRecipeBase &R : VPBB->phis()) @@ -379,8 +431,19 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) { MapVector<VPValue *, SmallVector<EdgeTy>> InValEdgesMap; for (auto [Edge, Val] : computeBlendEdges(PhiR)) InValEdgesMap[Val].push_back(Edge); + bool IncomingEdgesUnique = + all_of(PhiR->incoming_values(), [&](VPValue *InV) { + return is_contained(InValEdgesMap.keys(), InV); + }); + auto InValEdges = InValEdgesMap.takeVector(); + if (InValEdges.size() == 1) { + PhiR->replaceAllUsesWith(InValEdges[0].first); + PhiR->eraseFromParent(); + continue; + } + // Sort the incoming value order to match PhiR as much as possible. llvm::stable_sort(InValEdges, [&PhiR](auto &L, auto &R) { auto InVs = PhiR->incoming_values(); @@ -391,7 +454,13 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) { SmallVector<VPValue *, 2> OperandsWithMask; for (const auto &[InVPV, Edges] : InValEdges) { OperandsWithMask.push_back(InVPV); - OperandsWithMask.push_back(createBlendMaskForEdges(Edges, VPBB)); + if (IncomingEdgesUnique) { + if (auto *Mask = tryFindBlockInMask(Edges, VPBB)) { + OperandsWithMask.push_back(Mask); + continue; + } + } + OperandsWithMask.push_back(createBlendMaskForEdges(Edges)); } PHINode *IRPhi = cast_or_null<PHINode>(PhiR->getUnderlyingValue()); auto *Blend = diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll index 46887ec305304c..e8629499763db8 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll @@ -665,8 +665,6 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) { ; CHECK-NEXT: Successor(s): merge.a ; CHECK-EMPTY: ; CHECK-NEXT: merge.a: -; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c0> -; CHECK-NEXT: BLEND ir<%blend.a> = ir<%v1>/ir<%c0> ir<%v1>/vp<[[VP5]]> ; CHECK-NEXT: EMIT ir<%d0> = icmp sgt ir<%iv>, ir<0> ; CHECK-NEXT: Successor(s): if.b ; CHECK-EMPTY: @@ -675,16 +673,14 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) { ; CHECK-NEXT: Successor(s): if.b.inner ; CHECK-EMPTY: ; CHECK-NEXT: if.b.inner: -; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb> +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb> ; CHECK-NEXT: Successor(s): merge.b.inner ; CHECK-EMPTY: ; CHECK-NEXT: merge.b.inner: ; CHECK-NEXT: Successor(s): merge.b ; CHECK-EMPTY: ; CHECK-NEXT: merge.b: -; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%d0> -; CHECK-NEXT: BLEND ir<%blend.b> = ir<%v2>/ir<%d0> ir<%v2>/vp<[[VP7]]> -; CHECK-NEXT: EMIT ir<%sum> = add ir<%blend.a>, ir<%blend.b> +; CHECK-NEXT: EMIT ir<%sum> = add ir<%v1>, ir<%v2> ; CHECK-NEXT: EMIT store ir<%sum>, ir<%gep> ; CHECK-NEXT: Successor(s): loop.latch ; CHECK-EMPTY: @@ -1042,3 +1038,390 @@ latch: exit: ret void } + +define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) { +; CHECK-LABEL: VPlan for loop in 'look_thru_phi' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP6]]> +; CHECK-NEXT: BLEND ir<%phi4> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP6]]> +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1> +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%x>/ir<%c0> ir<%y>/vp<[[VP4]]> +; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv> +; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep> +; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; +; bb0 +; / \ +; bb1 \ +; / | bb2 +; \ | / +; bb3 bb4 +; \/ +; bb5 +entry: + br label %bb0 + +bb0: + %iv = phi i32 [0, %entry], [%iv.next, %bb5] + br i1 %c0, label %bb1, label %bb2 + +bb1: + br i1 %c1, label %bb3, label %bb4 + +bb2: + br label %bb4 + +bb3: + br label %bb5 + +bb4: + %phi4 = phi i32 [%x, %bb1], [%y, %bb2] + br label %bb5 + +bb5: + %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ] + %gep = getelementptr i32, ptr %p, i32 %iv + store i32 %phi5, ptr %gep + %iv.next = add i32 %iv, 1 + %ec = icmp eq i32 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +; Same as the test above, but shouldn't look thru the phi because it has +; multiple uses. +define void @look_thru_phi_multi_use(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) { +; CHECK-LABEL: VPlan for loop in 'look_thru_phi_multi_use' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv> +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP6]]> +; CHECK-NEXT: BLEND ir<%phi4> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP6]]> +; CHECK-NEXT: EMIT store ir<%phi4>, ir<%gep>, vp<[[VP7]]> +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1> +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<%x>/vp<[[VP8]]> +; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep> +; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; +; bb0 +; / \ +; bb1 \ +; / | bb2 +; \ | / +; bb3 bb4 +; \/ +; bb5 +entry: + br label %bb0 + +bb0: + %iv = phi i32 [0, %entry], [%iv.next, %bb5] + %gep = getelementptr i32, ptr %p, i32 %iv + br i1 %c0, label %bb1, label %bb2 + +bb1: + br i1 %c1, label %bb3, label %bb4 + +bb2: + br label %bb4 + +bb3: + br label %bb5 + +bb4: + %phi4 = phi i32 [%x, %bb1], [%y, %bb2] + store i32 %phi4, ptr %gep + br label %bb5 + +bb5: + %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ] + store i32 %phi5, ptr %gep + %iv.next = add i32 %iv, 1 + %ec = icmp eq i32 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +; Don't look through %phi3 as it doesn't share any incoming values with %phi4. +define void @look_thru_phi_no_shared_values(i1 %c0, i1 %c1, i32 %x, i32 %y, i32 %z, ptr %p) { +; CHECK-LABEL: VPlan for loop in 'look_thru_phi_no_shared_values' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0> +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1> +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%c1> +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP6]]> +; CHECK-NEXT: BLEND ir<%phi3> = ir<%z>/vp<[[VP5]]> ir<%y>/vp<[[VP7]]> +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: BLEND ir<%phi4> = ir<%phi3>/vp<[[VP4]]> ir<%x>/ir<%c0> +; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv> +; CHECK-NEXT: EMIT store ir<%phi4>, ir<%gep> +; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; +; bb0 +; | \ +; | bb1 +; | | \ +; | | bb2 +; | | / +; | bb3 +; | / +; bb4 +entry: + br label %bb0 + +bb0: + %iv = phi i32 [0, %entry], [%iv.next, %bb4] + br i1 %c0, label %bb4, label %bb1 + +bb1: + br i1 %c1, label %bb2, label %bb3 + +bb2: + br label %bb3 + +bb3: + %phi3 = phi i32 [%y, %bb1], [%z, %bb2] + br label %bb4 + +bb4: + %phi4 = phi i32 [%x, %bb0], [%phi3, %bb3] + %gep = getelementptr i32, ptr %p, i32 %iv + store i32 %phi4, ptr %gep + %iv.next = add i32 %iv, 1 + %ec = icmp eq i32 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +; bb4 will create the edge mask bb1->bb3 before bb3. Make sure bb4 +; inserts its blend after the already created edge mask. +define i32 @look_thru_phi_blend_insert_pos(i1 %c0, i1 %c1) { +; CHECK-LABEL: VPlan for loop in 'look_thru_phi_blend_insert_pos' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0> +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1> +; CHECK-NEXT: BLEND ir<%phi3> = ir<2>/vp<[[VP6]]> ir<1>/vp<[[VP7]]> +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or ir<%c0>, vp<[[VP7]]> +; CHECK-NEXT: BLEND ir<%phi4> = ir<1>/vp<[[VP8]]> ir<2>/vp<[[VP6]]> +; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; +; bb0 +; / \ +; bb1 \ +; / \ \ +; bb2 | / +; \ / / +; bb3 / +; \ / +; bb4 +entry: + br label %bb0 + +bb0: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ] + br i1 %c0, label %bb4, label %bb1 + +bb1: + br i1 %c1, label %bb3, label %bb2 + +bb2: + br label %bb3 + +bb3: + %phi3 = phi i32 [ 1, %bb1 ], [ 2, %bb2 ] + br label %bb4 + +bb4: + %phi4 = phi i32 [ 1, %bb0 ], [ %phi3, %bb3 ] + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret i32 %phi4 +} + +define void @diamond_phi2_look_thru_phi(ptr %a, i1 %c1, i1 %c2) { +; CHECK-LABEL: VPlan for loop in 'diamond_phi2_look_thru_phi' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0> +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c2> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1> +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]> +; CHECK-NEXT: BLEND ir<%phi4> = ir<1>/vp<[[VP5]]> ir<0>/vp<[[VP6]]> +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = not ir<%c1> +; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP8]]> +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not ir<%c2> +; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP10]]> +; CHECK-NEXT: BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<3>/vp<[[VP11]]> ir<2>/vp<[[VP9]]> +; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv> +; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; +; bb0 +; / \ +; bb1 bb2 +; | \ / | +; | bb4 | +; \ | / +; bb5 +entry: + br label %bb0 + +bb0: + %iv = phi i64 [0, %entry], [%iv.next, %bb5] + %c0 = icmp sle i64 %iv, 0 + br i1 %c0, label %bb1, label %bb2 + +bb1: + %add1 = add i64 %iv, 1 + br i1 %c1, label %bb4, label %bb5 + +bb2: + %add2 = add i64 %iv, 2 + br i1 %c2, label %bb4, label %bb5 + +bb4: + %phi4 = phi i64 [0, %bb1], [1, %bb2] + br label %bb5 + +bb5: + %phi5 = phi i64 [2, %bb1], [3, %bb2], [%phi4, %bb4] + %gep = getelementptr i64, ptr %a, i64 %iv + store i64 %phi5, ptr %gep + %iv.next = add nsw nuw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopVectorize/find-last.ll b/llvm/test/Transforms/LoopVectorize/find-last.ll index 9948f34e4571b6..35cf71099b101c 100644 --- a/llvm/test/Transforms/LoopVectorize/find-last.ll +++ b/llvm/test/Transforms/LoopVectorize/find-last.ll @@ -445,3 +445,84 @@ loop.latch: exit: ret i32 %rdx.next } + +define i32 @find_last_phi_of_two_updates(ptr noalias %a, ptr noalias %b, i64 %n) { +; CHECK-LABEL: define i32 @find_last_phi_of_two_updates( +; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 5) +; CHECK-NEXT: [[TMP5:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true) +; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 2) +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer +; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[WIDE_LOAD1]], <4 x i32> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]]) +; CHECK-NEXT: [[TMP13]] = select i1 [[TMP12]], <4 x i1> [[TMP11]], <4 x i1> [[TMP1]] +; CHECK-NEXT: [[TMP14]] = select i1 [[TMP12]], <4 x i32> [[PREDPHI]], <4 x i32> [[VEC_PHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP14]], <4 x i1> [[TMP13]], i32 -1) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ] +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; +entry: + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %rdx = phi i32 [ -1, %entry ], [ %rdx.next, %loop.latch ] + %gep.a = getelementptr i32, ptr %a, i64 %iv + %ld.a = load i32, ptr %gep.a + %gep.b = getelementptr i32, ptr %b, i64 %iv + %ld.b = load i32, ptr %gep.b + %c.0 = icmp sgt i32 %ld.a, 5 + br i1 %c.0, label %then, label %else + +then: + %c.1 = icmp ne i32 %ld.b, 0 + br i1 %c.1, label %merge, label %update + +else: + %c.2 = icmp ne i32 %ld.a, 2 + br i1 %c.2, label %merge, label %update + +update: + %val = phi i32 [ %ld.b, %then ], [ %ld.a, %else ] + br label %merge + +merge: + %rdx.next = phi i32 [ %rdx, %then ], [ %rdx, %else ], [ %val, %update ] + br label %loop.latch + +loop.latch: + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv.next, %n + br i1 %ec, label %exit, label %loop.header + +exit: + ret i32 %rdx.next +} diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll index 8a7b4fb2f71816..53d853dd86dff9 100644 --- a/llvm/test/Transforms/LoopVectorize/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/predicator.ll @@ -362,3 +362,498 @@ latch: exit: ret void } + +define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) { +; CHECK-LABEL: define void @look_thru_phi( +; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr [[P:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[C0]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]] +; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +; bb0 +; / \ +; bb1 \ +; / | bb2 +; \ | / +; bb3 bb4 +; \/ +; bb5 +entry: + br label %bb0 + +bb0: + %iv = phi i32 [0, %entry], [%iv.next, %bb5] + br i1 %c0, label %bb1, label %bb2 + +bb1: + br i1 %c1, label %bb3, label %bb4 + +bb2: + br label %bb4 + +bb3: + br label %bb5 + +bb4: + %phi4 = phi i32 [%x, %bb1], [%y, %bb2] + br label %bb5 + +bb5: + %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ] + %gep = getelementptr i32, ptr %p, i32 %iv + store i32 %phi5, ptr %gep + %iv.next = add i32 %iv, 1 + %ec = icmp eq i32 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +; Same as the test above, but shouldn't look thru the phi because it has +; multiple uses. +define void @look_thru_phi_multi_use(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) { +; CHECK-LABEL: define void @look_thru_phi_multi_use( +; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr [[P:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT5]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT6]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT7]], splat (i1 true) +; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT6]], splat (i1 true) +; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT7]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i1> [[TMP0]], [[TMP2]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[PREDPHI]], <4 x i32> [[BROADCAST_SPLAT2]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]] +; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[PREDPHI]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]]) +; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP4]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +; bb0 +; / \ +; bb1 \ +; / | bb2 +; \ | / +; bb3 bb4 +; \/ +; bb5 +entry: + br label %bb0 + +bb0: + %iv = phi i32 [0, %entry], [%iv.next, %bb5] + %gep = getelementptr i32, ptr %p, i32 %iv + br i1 %c0, label %bb1, label %bb2 + +bb1: + br i1 %c1, label %bb3, label %bb4 + +bb2: + br label %bb4 + +bb3: + br label %bb5 + +bb4: + %phi4 = phi i32 [%x, %bb1], [%y, %bb2] + store i32 %phi4, ptr %gep + br label %bb5 + +bb5: + %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ] + store i32 %phi5, ptr %gep + %iv.next = add i32 %iv, 1 + %ec = icmp eq i32 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +; Don't look through %phi3 as it doesn't share any incoming values with %phi4. +define void @look_thru_phi_no_shared_values(i1 %c0, i1 %c1, i32 %x, i32 %y, i32 %z, ptr %p) { +; CHECK-LABEL: define void @look_thru_phi_no_shared_values( +; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]], ptr [[P:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[Z]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT4]] +; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[C0]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[PREDPHI]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]] +; CHECK-NEXT: store <4 x i32> [[PREDPHI5]], ptr [[TMP0]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +; bb0 +; | \ +; | bb1 +; | | \ +; | | bb2 +; | | / +; | bb3 +; | / +; bb4 +entry: + br label %bb0 + +bb0: + %iv = phi i32 [0, %entry], [%iv.next, %bb4] + br i1 %c0, label %bb4, label %bb1 + +bb1: + br i1 %c1, label %bb2, label %bb3 + +bb2: + br label %bb3 + +bb3: + %phi3 = phi i32 [%y, %bb1], [%z, %bb2] + br label %bb4 + +bb4: + %phi4 = phi i32 [%x, %bb0], [%phi3, %bb3] + %gep = getelementptr i32, ptr %p, i32 %iv + store i32 %phi4, ptr %gep + %iv.next = add i32 %iv, 1 + %ec = icmp eq i32 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +; bb4 will create the edge mask bb1->bb3 before bb3. Make sure bb4 +; inserts its blend after the already created edge mask. +define i32 @look_thru_phi_blend_insert_pos(i1 %c0, i1 %c1) { +; CHECK-LABEL: define i32 @look_thru_phi_blend_insert_pos( +; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true) +; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) +; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> splat (i32 2), <4 x i32> splat (i32 1) +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret i32 [[TMP4]] +; +; bb0 +; / \ +; bb1 \ +; / \ \ +; bb2 | / +; \ / / +; bb3 / +; \ / +; bb4 +entry: + br label %bb0 + +bb0: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ] + br i1 %c0, label %bb4, label %bb1 + +bb1: + br i1 %c1, label %bb3, label %bb2 + +bb2: + br label %bb3 + +bb3: + %phi3 = phi i32 [ 1, %bb1 ], [ 2, %bb2 ] + br label %bb4 + +bb4: + %phi4 = phi i32 [ 1, %bb0 ], [ %phi3, %bb3 ] + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret i32 %phi4 +} + +define void @diamond_phi2_look_thru_phi(ptr %a, i1 %c1, i1 %c2) { +; CHECK-LABEL: define void @diamond_phi2_look_thru_phi( +; CHECK-SAME: ptr [[A:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C2]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) +; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true) +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true) +; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> zeroinitializer, <4 x i64> splat (i64 1) +; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP9]], <4 x i64> splat (i64 3), <4 x i64> [[PREDPHI]] +; CHECK-NEXT: [[PREDPHI4:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> splat (i64 2), <4 x i64> [[PREDPHI3]] +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: store <4 x i64> [[PREDPHI4]], ptr [[TMP7]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +; bb0 +; / \ +; bb1 bb2 +; | \ / | +; | bb4 | +; \ | / +; bb5 +entry: + br label %bb0 + +bb0: + %iv = phi i64 [0, %entry], [%iv.next, %bb5] + %c0 = icmp sle i64 %iv, 0 + br i1 %c0, label %bb1, label %bb2 + +bb1: + %add1 = add i64 %iv, 1 + br i1 %c1, label %bb4, label %bb5 + +bb2: + %add2 = add i64 %iv, 2 + br i1 %c2, label %bb4, label %bb5 + +bb4: + %phi4 = phi i64 [0, %bb1], [1, %bb2] + br label %bb5 + +bb5: + %phi5 = phi i64 [2, %bb1], [3, %bb2], [%phi4, %bb4] + %gep = getelementptr i64, ptr %a, i64 %iv + store i64 %phi5, ptr %gep + %iv.next = add nsw nuw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @blend_look_through_no_shared_values(ptr noalias %dst, ptr noalias %conds, i32 %x, i32 %y, i32 %u, i32 %w, i64 %n) { +; CHECK-LABEL: define void @blend_look_through_no_shared_values( +; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[CONDS:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[U:%.*]], i32 [[W:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[U]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[W]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[CONDS]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1 +; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i8> [[WIDE_LOAD]] to <4 x i1> +; CHECK-NEXT: [[TMP3:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 1) +; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i8> [[TMP3]] to <4 x i1> +; CHECK-NEXT: [[TMP5:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 2) +; CHECK-NEXT: [[TMP6:%.*]] = trunc <4 x i8> [[TMP5]] to <4 x i1> +; CHECK-NEXT: [[TMP7:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 3) +; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i8> [[TMP7]] to <4 x i1> +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]] +; CHECK-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true) +; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP10]], [[TMP12]] +; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32> [[PREDPHI]] +; CHECK-NEXT: [[PREDPHI8:%.*]] = select <4 x i1> [[TMP13]], <4 x i32> [[BROADCAST_SPLAT4]], <4 x i32> [[PREDPHI7]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]] +; CHECK-NEXT: store <4 x i32> [[PREDPHI8]], ptr [[TMP14]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[GEP_CONDS:%.*]] = getelementptr i8, ptr [[CONDS]], i64 [[IV]] +; CHECK-NEXT: [[LD:%.*]] = load i8, ptr [[GEP_CONDS]], align 1 +; CHECK-NEXT: [[C_0:%.*]] = trunc i8 [[LD]] to i1 +; CHECK-NEXT: [[SHR_1:%.*]] = lshr i8 [[LD]], 1 +; CHECK-NEXT: [[C_1:%.*]] = trunc i8 [[SHR_1]] to i1 +; CHECK-NEXT: [[SHR_2:%.*]] = lshr i8 [[LD]], 2 +; CHECK-NEXT: [[C_2:%.*]] = trunc i8 [[SHR_2]] to i1 +; CHECK-NEXT: [[SHR_3:%.*]] = lshr i8 [[LD]], 3 +; CHECK-NEXT: [[C_3:%.*]] = trunc i8 [[SHR_3]] to i1 +; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]] +; CHECK: [[THEN]]: +; CHECK-NEXT: br i1 [[C_1]], label %[[THEN_THEN:.*]], label %[[THEN_ELSE:.*]] +; CHECK: [[THEN_THEN]]: +; CHECK-NEXT: br i1 [[C_2]], label %[[USE_X:.*]], label %[[USE_Y:.*]] +; CHECK: [[THEN_ELSE]]: +; CHECK-NEXT: br label %[[THEN_ELSE_CONT:.*]] +; CHECK: [[THEN_ELSE_CONT]]: +; CHECK-NEXT: br label %[[USE_X]] +; CHECK: [[USE_X]]: +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[USE_Y]]: +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[ELSE]]: +; CHECK-NEXT: br i1 [[C_3]], label %[[ELSE_THEN:.*]], label %[[ELSE_ELSE:.*]] +; CHECK: [[ELSE_THEN]]: +; CHECK-NEXT: br label %[[ELSE_MERGE:.*]] +; CHECK: [[ELSE_ELSE]]: +; CHECK-NEXT: br label %[[ELSE_MERGE]] +; CHECK: [[ELSE_MERGE]]: +; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[U]], %[[ELSE_THEN]] ], [ [[W]], %[[ELSE_ELSE]] ] +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[X]], %[[USE_X]] ], [ [[Y]], %[[USE_Y]] ], [ [[P]], %[[ELSE_MERGE]] ] +; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]] +; CHECK-NEXT: store i32 [[MERGE]], ptr [[GEP_DST]], align 4 +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP12:![0-9]+]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %gep.conds = getelementptr i8, ptr %conds, i64 %iv + %ld = load i8, ptr %gep.conds + %c.0 = trunc i8 %ld to i1 + %shr.1 = lshr i8 %ld, 1 + %c.1 = trunc i8 %shr.1 to i1 + %shr.2 = lshr i8 %ld, 2 + %c.2 = trunc i8 %shr.2 to i1 + %shr.3 = lshr i8 %ld, 3 + %c.3 = trunc i8 %shr.3 to i1 + br i1 %c.0, label %then, label %else + +then: + br i1 %c.1, label %then.then, label %then.else + +then.then: + br i1 %c.2, label %use.x, label %use.y + +then.else: + br label %then.else.cont + +then.else.cont: + br label %use.x + +use.x: + br label %loop.latch + +use.y: + br label %loop.latch + +else: + br i1 %c.3, label %else.then, label %else.else + +else.then: + br label %else.merge + +else.else: + br label %else.merge + +else.merge: + %p = phi i32 [ %u, %else.then ], [ %w, %else.else ] + br label %loop.latch + +loop.latch: + %merge = phi i32 [ %x, %use.x ], [ %y, %use.y ], [ %p, %else.merge ] + %gep.dst = getelementptr i32, ptr %dst, i64 %iv + store i32 %merge, ptr %gep.dst + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv.next, %n + br i1 %ec, label %exit, label %loop.header + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll index 6cae52e2a8f283..e1fa8237cb9633 100644 --- a/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll +++ b/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll @@ -97,15 +97,25 @@ define i32 @anyof_three_blend_chain(i1 %c0, i1 %c1, i1 %c2, i32 %n) { ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[SMAX]], 3 ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[SMAX]], [[N_MOD_VF]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C2]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT3]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP7:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT4]], splat (i1 true) +; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true) +; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) +; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP10]], <4 x i1> zeroinitializer ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI2:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i32> [[VEC_IND]], zeroinitializer -; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i1> [[VEC_PHI]], [[TMP4]] -; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C2]], <4 x i1> [[VEC_PHI]], <4 x i1> [[TMP3]] -; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[C1]], <4 x i1> [[VEC_PHI]], <4 x i1> [[PREDPHI]] +; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[PREDPHI1:%.*]] = or <4 x i1> [[VEC_PHI]], [[TMP8]] ; CHECK-NEXT: [[PREDPHI2]] = select i1 [[C0]], <4 x i1> [[VEC_PHI]], <4 x i1> [[PREDPHI1]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4) diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll index 8d86c8ca2bd9d7..a016a26d479ec7 100644 --- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll +++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll @@ -96,19 +96,10 @@ define void @blend_chain_iv(i1 %c) { ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3 -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]] -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP3]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP5]] +; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP7]] -; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2 -; CHECK-NEXT: store i16 0, ptr [[TMP4]], align 2 -; CHECK-NEXT: store i16 0, ptr [[TMP6]], align 2 -; CHECK-NEXT: store i16 0, ptr [[TMP8]], align 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP8]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], 4 ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 ; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
