https://github.com/eas updated https://github.com/llvm/llvm-project/pull/218552
>From 15f4d3ebc6a0a9ebbf5dfda1c60ee6d0e2b517db Mon Sep 17 00:00:00 2001 From: Andrei Elovikov <[email protected]> Date: Wed, 26 Aug 2026 12:34:31 -0700 Subject: [PATCH 1/2] Outline `shouldPreserveTerminator` --- .../Transforms/Vectorize/VPlanPredicator.cpp | 112 +++++++++--------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp index 7f37bb2bcc562..f84a4a11f3283 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp @@ -110,6 +110,8 @@ class VPPredicator { /// Create a logical-or, factoring out a common header mask if present. VPValue *createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL); + bool shouldPreserveTerminator(VPBasicBlock *VPBB); + /// Record \p Mask as the *entry* mask of \p VPBB, which is expected to not /// already have a mask. void setBlockInMask(const VPBasicBlock *VPBB, VPValue *Mask) { @@ -207,6 +209,59 @@ VPValue *VPPredicator::createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL) { HeaderMask, Builder.createOr(LHSRemainder, RHSRemainder, DL), DL); } +bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) { + LLVM_DEBUG(dbgs() << "Checking if can preserve the branch at the end of " + << VPBB->getName() << "\n"); + auto False = []([[maybe_unused]] StringRef Reason = "") { + LLVM_DEBUG(dbgs() << " can't be preserved" + << (Reason.empty() ? Twine() : (": " + Reason)) << "\n"); + return false; + }; + if (VPBB->getNumSuccessors() != 2) + return False("#Successors != 2"); + auto *Term = dyn_cast<VPInstruction>(VPBB->getTerminator()); + if (!Term || Term->getOpcode() != VPInstruction::BranchOnCond) + return False("Not a branch"); + + bool IsUniformAndAvailable = [&](VPValue *V) { + auto *IRV = dyn_cast<VPIRValue>(V); + return IRV && isa<Argument, Constant>(IRV->getValue()); + }(Term->getOperand(0)); + + if (!IsUniformAndAvailable) + return False("non-uniform"); + + // Should not happen in the end-to-end pass pipeline (simplifycfg would + // have handled it), but possible when running `loop-vectorize` pass + // alone. Just don't preserve so that we won't have to handle that when + // executing VPlan. + if (all_equal(VPBB->successors())) + return False("All successors are the same"); + + auto *IPostDomNode = VPPDT.getNode(VPBB)->getIDom(); + if (!IPostDomNode) + return False("no post-dom"); + auto *IPostDom = dyn_cast<VPBasicBlock>(IPostDomNode->getBlock()); + if (!IPostDom || !VPDT.properlyDominates(VPBB, IPostDom)) + return False("doesn't dominate its post-dom"); + + LLVM_DEBUG(dbgs() << "IPostDom: " << IPostDom->getName() << "\n";); + + for (VPBlockBase *Pred : IPostDom->getPredecessors()) { + if (Pred == VPBB) + continue; + if (count_if(VPBB->successors(), + [&](auto *Succ) { return VPDT.dominates(Succ, Pred); }) != 1) + return False("Bailing out due to successors not being independent"); + } + + if (any_of(*IPostDom, IsaPred<VPBlendRecipe>)) + return False("Blends at reconvergence"); + + LLVM_DEBUG(dbgs() << "...yes, can be preserved.\n"); + return true; +} + VPValue *VPPredicator::createEdgeMask(const VPBasicBlock *Src, const VPBasicBlock *Dst) { assert(is_contained(Dst->getPredecessors(), Src) && "Invalid edge"); @@ -519,62 +574,7 @@ void VPPredicator::run() { LLVM_DEBUG(dbgs() << "Setting successors for " << VPBB->getName() << "\n"); auto Successors = to_vector(VPBB->getSuccessors()); - auto PreserveUniform = [&]() -> bool { - LLVM_DEBUG(dbgs() << "Checking if can preserve the branch at the end of " - << VPBB->getName() << "\n"); - auto False = []([[maybe_unused]] StringRef Reason = "") { - LLVM_DEBUG(dbgs() << " can't be preserved" - << (Reason.empty() ? Twine() : (": " + Reason)) - << "\n"); - return false; - }; - if (VPBB->getNumSuccessors() != 2) - return False("#Successors != 2"); - auto *Term = dyn_cast<VPInstruction>(VPBB->getTerminator()); - if (!Term || Term->getOpcode() != VPInstruction::BranchOnCond) - return False("Not a branch"); - - bool IsUniformAndAvailable = [&](VPValue *V) { - auto *IRV = dyn_cast<VPIRValue>(V); - return IRV && isa<Argument, Constant>(IRV->getValue()); - }(Term->getOperand(0)); - - if (!IsUniformAndAvailable) - return False("non-uniform"); - - // Should not happen in the end-to-end pass pipeline (simplifycfg would - // have handled it), but possible when running `loop-vectorize` pass - // alone. Just don't preserve so that we won't have to handle that when - // executing VPlan. - if (all_equal(VPBB->successors())) - return False("All successors are the same"); - - auto *IPostDomNode = VPPDT.getNode(VPBB)->getIDom(); - if (!IPostDomNode) - return False("no post-dom"); - auto *IPostDom = dyn_cast<VPBasicBlock>(IPostDomNode->getBlock()); - if (!IPostDom || !VPDT.properlyDominates(VPBB, IPostDom)) - return False("doesn't dominate its post-dom"); - - LLVM_DEBUG(dbgs() << "IPostDom: " << IPostDom->getName() << "\n";); - - for (VPBlockBase *Pred : IPostDom->getPredecessors()) { - if (Pred == VPBB) - continue; - if (count_if(VPBB->successors(), [&](auto *Succ) { - return VPDT.dominates(Succ, Pred); - }) != 1) - return False("Bailing out due to successors not being independent"); - } - - if (any_of(*IPostDom, IsaPred<VPBlendRecipe>)) - return False("Blends at reconvergence"); - - LLVM_DEBUG(dbgs() << "...yes, can be preserved.\n"); - return true; - }(); - - if (PreserveUniform) { + if (shouldPreserveTerminator(VPBB)) { for (auto *Succ : Successors) VPBlockUtils::disconnectBlocks(VPBB, Succ); >From b3475d42676f12dc4b76343231def4adf492a33d Mon Sep 17 00:00:00 2001 From: Andrei Elovikov <[email protected]> Date: Fri, 21 Aug 2026 11:08:53 -0700 Subject: [PATCH 2/2] WIP Don't adjust mask for preserved uniform edges --- .../Transforms/Vectorize/LoopVectorize.cpp | 6 +- .../Transforms/Vectorize/VPlanPredicator.cpp | 14 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 3 +- .../AArch64/predication_costs.ll | 38 +++- .../AArch64/sve-predicated-costs.ll | 5 +- ...-interleave-to-widen-memory-multi-block.ll | 36 +--- .../LoopVectorize/RISCV/dead-ops-cost.ll | 55 +----- .../LoopVectorize/RISCV/predicated-costs.ll | 10 +- .../RISCV/predicated-reverse-store.ll | 7 +- .../LoopVectorize/VPlan/predicator.ll | 81 ++++---- .../VPlan/vplan-sink-scalars-and-merge-vf1.ll | 18 +- .../VPlan/vplan-sink-scalars-and-merge.ll | 28 +-- .../LoopVectorize/X86/masked-store-cost.ll | 4 +- .../Transforms/LoopVectorize/div-exact.ll | 134 +------------ .../LoopVectorize/if-pred-stores.ll | 179 +++++------------- llvm/test/Transforms/LoopVectorize/pr37248.ll | 10 - .../Transforms/LoopVectorize/predicator.ll | 51 ++--- 17 files changed, 177 insertions(+), 502 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 28895d1d2b418..0eb7e59080780 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6240,8 +6240,10 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) { case Instruction::UDiv: case Instruction::SRem: case Instruction::URem: - // If not provably safe, use a masked intrinsic. - if (CM.isPredicatedInst(I)) + // If not provably safe, use a masked intrinsic. Predicator might have + // preserved uniform control flow making this unmasked even if CM expected + // it to be masked. + if (CM.isPredicatedInst(I) && VPI->isMasked()) return new VPWidenIntrinsicRecipe( getMaskedDivRemIntrinsic(VPI->getOpcode()), VPI->operands(), I->getType(), {}, {}, VPI->getDebugLoc()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp index f84a4a11f3283..58b1fd2ccc0ce 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp @@ -255,6 +255,10 @@ bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) { return False("Bailing out due to successors not being independent"); } + // FIXME: Only one of the two checks below should remain in the end. + if (!IPostDom->phis().empty()) + return False("Phis at reconvergence"); + if (any_of(*IPostDom, IsaPred<VPBlendRecipe>)) return False("Blends at reconvergence"); @@ -323,7 +327,10 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) { // This is the block mask. We OR all unique incoming edges. for (auto *Predecessor : SetVector<VPBlockBase *>( VPBB->getPredecessors().begin(), VPBB->getPredecessors().end())) { - VPValue *EdgeMask = createEdgeMask(cast<VPBasicBlock>(Predecessor), VPBB); + auto *Pred = cast<VPBasicBlock>(Predecessor); + VPValue *EdgeMask = shouldPreserveTerminator(Pred) + ? getBlockInMask(Pred) + : createEdgeMask(Pred, VPBB); if (!EdgeMask) { // Mask of predecessor is all-one so mask of block is // too. setBlockInMask(VPBB, EdgeMask); @@ -481,6 +488,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) { for (VPRecipeBase &R : VPBB->phis()) Phis.push_back(cast<VPPhi>(&R)); for (VPPhi *PhiR : Phis) { + LLVM_DEBUG(dbgs() << "Converting " << *PhiR << " to blend\n"); // The non-header Phi is converted into a Blend recipe below, // so we don't have to worry about the insertion order and we can just use // the builder. At this point we generate the predication tree. There may @@ -491,6 +499,9 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) { return !match(V, m_Poison()); }); if (all_equal(NotPoison)) { + LLVM_DEBUG(dbgs() << " all incoming values are the same or poison, " + "replacing with single value " + << PhiR->getIncomingValue(0) << "\n"); PhiR->replaceAllUsesWith(NotPoison.empty() ? PhiR->getIncomingValue(0) : *NotPoison.begin()); PhiR->eraseFromParent(); @@ -525,6 +536,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) { auto *Blend = new VPBlendRecipe(IRPhi, OperandsWithMask, *PhiR, PhiR->getDebugLoc()); Builder.insert(Blend); + LLVM_DEBUG(dbgs() << " blend: " << *Blend << "\n"); PhiR->replaceAllUsesWith(Blend); PhiR->eraseFromParent(); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index a0775ca6e4345..049814c054713 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -5401,7 +5401,8 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range, // A predicated access can only be widened (rather than scalarized) if // the target supports a masked load/store for it. // TODO: Determine if a load/store needs predication directly in VPlan. - bool IsPredicated = RecipeBuilder.isPredicatedInst(I); + bool IsPredicated = + RecipeBuilder.isPredicatedInst(I) && VPI->isMasked(); if (IsPredicated && !CostCtx.Config.isLegalMaskedLoadOrStore( IsLoad, ScalarTy, getLoadStoreAlignment(I), getLoadStoreAddressSpace(I))) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll index fdb22beb19695..e7b718dc0d483 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll @@ -62,7 +62,34 @@ for.end: ; CHECK: Scalarizing and predicating: store i32 %tmp2, ptr %tmp0, align 4 ; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4 ; -define void @predicated_store(ptr %a, i1 %c, i32 %x, i64 %n) { +define void @predicated_store(ptr %a, i32 %x, i64 %n) { +entry: + br label %for.body + +for.body: + %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ] + %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i + %tmp1 = load i32, ptr %tmp0, align 4 + %tmp2 = add nsw i32 %tmp1, %x + %c = icmp eq i32 %tmp1, 42 + br i1 %c, label %if.then, label %for.inc + +if.then: + store i32 %tmp2, ptr %tmp0, align 4 + br label %for.inc + +for.inc: + %i.next = add nuw nsw i64 %i, 1 + %cond = icmp slt i64 %i.next, %n + br i1 %cond, label %for.body, label %for.end + +for.end: + ret void +} + +; CHECK-LABEL: store_in_preserved_uniform_control_flow +; CHECK: Cost of 1 for VF 2: WIDEN store {{.*}}, ir<%tmp2> +define void @store_in_preserved_uniform_control_flow(ptr %a, i1 %c, i32 %x, i64 %n) { entry: br label %for.body @@ -96,7 +123,7 @@ for.end: ; CHECK: Cost of 0 for VF 2: induction instruction %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ] ; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %addr, align 4 ; -define void @predicated_store_phi(ptr %a, i1 %c, i32 %x, i64 %n) { +define void @predicated_store_phi(ptr %a, i32 %x, i64 %n) { entry: br label %for.body @@ -105,6 +132,7 @@ for.body: %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ] %tmp1 = load i32, ptr %addr, align 4 %tmp2 = add nsw i32 %tmp1, %x + %c = icmp eq i32 %tmp1, 42 br i1 %c, label %if.then, label %for.inc if.then: @@ -184,7 +212,7 @@ for.end: ; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4 ; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp2 = add nsw i32 %tmp1, %x ; -define void @predicated_store_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) { +define void @predicated_store_scalarized_operand(ptr %a, i32 %x, i64 %n) { entry: br label %for.body @@ -192,6 +220,7 @@ for.body: %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ] %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i %tmp1 = load i32, ptr %tmp0, align 4 + %c = icmp eq i32 %tmp1, 42 br i1 %c, label %if.then, label %for.inc if.then: @@ -240,7 +269,7 @@ for.end: ; CHECK: Cost of 7 for VF 2: REPLICATE ir<%tmp3> = sdiv ir<%tmp1>, ir<%tmp2> ; CHECK: Cost of 5 for VF 2: REPLICATE ir<%tmp4> = udiv ir<%tmp3>, ir<%tmp2> ; -define void @predication_multi_context(ptr %a, i1 %c, i32 %x, i64 %n) { +define void @predication_multi_context(ptr %a, i32 %x, i64 %n) { entry: br label %for.body @@ -248,6 +277,7 @@ for.body: %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ] %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i %tmp1 = load i32, ptr %tmp0, align 4 + %c = icmp eq i32 %tmp1, 42 br i1 %c, label %if.then, label %for.inc if.then: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll index 9360ce4aea08f..fd30156408e6a 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll @@ -81,10 +81,7 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1, i1 % ; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C2]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C1]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer -; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C0]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = select <vscale x 2 x i1> [[BROADCAST_SPLAT2]], <vscale x 2 x i1> [[BROADCAST_SPLAT]], <vscale x 2 x i1> zeroinitializer +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> [[BROADCAST_SPLAT1]], <vscale x 2 x i1> zeroinitializer ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll index 9efff93543685..3d69698888f16 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll @@ -13,7 +13,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr ; VF2IC1: [[VECTOR_PH]]: ; VF2IC1-NEXT: br label %[[VECTOR_BODY:.*]] ; VF2IC1: [[VECTOR_BODY]]: -; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] +; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ] ; VF2IC1-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1 ; VF2IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]] ; VF2IC1-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8 @@ -22,20 +22,11 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr ; VF2IC1-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> ; VF2IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> ; VF2IC1-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8 -; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; VF2IC1: [[PRED_STORE_IF]]: -; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]] -; VF2IC1: [[PRED_STORE_IF1]]: -; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1 -; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE1]] -; VF2IC1: [[PRED_STORE_CONTINUE1]]: -; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]] +; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]] ; VF2IC1: [[PRED_STORE_IF2]]: ; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1 ; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE3]] ; VF2IC1: [[PRED_STORE_CONTINUE3]]: -; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE]] -; VF2IC1: [[PRED_STORE_CONTINUE]]: ; VF2IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]] ; VF2IC1-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP2]], align 1 ; VF2IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 @@ -53,7 +44,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr ; VF2IC2: [[VECTOR_PH]]: ; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]] ; VF2IC2: [[VECTOR_BODY]]: -; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] +; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE11:.*]] ] ; VF2IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2 ; VF2IC2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1 ; VF2IC2-NEXT: [[TMP2:%.*]] = shl nsw i64 [[TMP0]], 1 @@ -71,30 +62,11 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr ; VF2IC2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC3]], <2 x i64> [[STRIDED_VEC4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> ; VF2IC2-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> ; VF2IC2-NEXT: store <4 x i64> [[INTERLEAVED_VEC5]], ptr [[TMP4]], align 8 -; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; VF2IC2: [[PRED_STORE_IF]]: -; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]] -; VF2IC2: [[PRED_STORE_IF1]]: -; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1 -; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE1]] -; VF2IC2: [[PRED_STORE_CONTINUE1]]: -; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]] -; VF2IC2: [[PRED_STORE_IF6]]: -; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1 -; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE7]] -; VF2IC2: [[PRED_STORE_CONTINUE7]]: -; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]] -; VF2IC2: [[PRED_STORE_IF8]]: -; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1 -; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE9]] -; VF2IC2: [[PRED_STORE_CONTINUE9]]: -; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]] +; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11]] ; VF2IC2: [[PRED_STORE_IF10]]: ; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1 ; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE11]] ; VF2IC2: [[PRED_STORE_CONTINUE11]]: -; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]] -; VF2IC2: [[PRED_STORE_CONTINUE]]: ; VF2IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]] ; VF2IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 2 ; VF2IC2-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP5]], align 1 diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll index c4ca7d1d47673..913eba75abf17 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll @@ -158,60 +158,19 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0 ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i32 8, i32 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[TMP9]] -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_EXITING18:.*]] ] -; CHECK-NEXT: br i1 [[C]], label %[[THEN3:.*]], label %[[LOOP_EXITING18]] -; CHECK: [[THEN3]]: -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[B]], i32 [[INDEX]] -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] -; CHECK: [[PRED_STORE_IF]]: -; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]] -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]] -; CHECK: [[PRED_STORE_CONTINUE]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]] -; CHECK: [[PRED_STORE_IF5]]: -; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]] -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]] -; CHECK: [[PRED_STORE_CONTINUE6]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]] -; CHECK: [[PRED_STORE_IF7]]: -; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]] -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]] -; CHECK: [[PRED_STORE_CONTINUE8]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]] -; CHECK: [[PRED_STORE_IF9]]: -; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]] -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]] -; CHECK: [[PRED_STORE_CONTINUE10]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]] -; CHECK: [[PRED_STORE_IF11]]: -; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]] -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]] -; CHECK: [[PRED_STORE_CONTINUE12]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]] -; CHECK: [[PRED_STORE_IF13]]: -; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]] -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]] -; CHECK: [[PRED_STORE_CONTINUE14]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]] -; CHECK: [[PRED_STORE_IF15]]: -; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]] -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16]] +; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16:.*]] ; CHECK: [[PRED_STORE_CONTINUE16]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]] +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE18:.*]] ] +; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]] ; CHECK: [[PRED_STORE_IF17]]: -; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]] +; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]] +; CHECK-NEXT: store <8 x i32> zeroinitializer, ptr [[TMP13]], align 4, !alias.scope [[META13]] ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]] ; CHECK: [[PRED_STORE_CONTINUE18]]: -; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[TMP11]], <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META13]] -; CHECK-NEXT: br label %[[LOOP_EXITING18]] -; CHECK: [[LOOP_EXITING18]]: ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 ; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE16]], !llvm.loop [[LOOP15:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: br label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll index e24e78fcf0602..532d9bc35e854 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll @@ -12,10 +12,7 @@ define void @nested(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer -; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: br label %[[THEN_1:.*]] ; CHECK: [[THEN_1]]: ; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH4:.*]] ] @@ -74,10 +71,7 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) { ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer -; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH4:.*]] ] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll index 29373388b3721..a5b3d96996c26 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll @@ -7,8 +7,6 @@ define void @reverse_predicated_store(i1 %c, ptr %dst, i64 %n) #0 { ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1 ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C:%.*]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[FOR_BODY]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ] @@ -16,16 +14,15 @@ define void @reverse_predicated_store(i1 %c, ptr %dst, i64 %n) #0 { ; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) ; CHECK-NEXT: [[IV:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] ; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], -1 -; CHECK-NEXT: br i1 [[C]], label [[IF_THEN1:%.*]], label [[LOOP_LATCH2]] +; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_THEN1:%.*]], label [[LOOP_LATCH2]] ; CHECK: if.then1: ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr float, ptr [[DST:%.*]], i64 [[IV_NEXT]] -; CHECK-NEXT: [[VP_REVERSE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]]) ; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64 ; CHECK-NEXT: [[TMP6:%.*]] = sub nuw nsw i64 [[TMP4]], 1 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 0, [[TMP6]] ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[ARRAYIDX]], i64 [[TMP7]] ; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]]) -; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP12]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[VP_REVERSE_MASK]], i32 [[TMP1]]) +; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP12]], ptr align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]]) ; CHECK-NEXT: br label [[LOOP_LATCH2]] ; CHECK: loop.latch2: ; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP1]] to i64 diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll index 239bd1b324257..a8e69d1ab4f4d 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll @@ -762,12 +762,12 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p) ; CHECK-NEXT: Successor(s): D ; CHECK-EMPTY: ; CHECK-NEXT: D: -; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%c2> -; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP7]]> +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c2> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP5]]> ; CHECK-NEXT: Successor(s): C ; CHECK-EMPTY: ; CHECK-NEXT: C: -; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and ir<%c1>, ir<%c2> +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%c1>, ir<%c2> ; CHECK-NEXT: Successor(s): latch ; CHECK-EMPTY: ; CHECK-NEXT: latch: @@ -781,8 +781,6 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p) ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: E: -; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c3> -; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> ; CHECK-NEXT: Successor(s): F ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): middle.block @@ -1043,15 +1041,14 @@ define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: Successor(s): bb1, bb4 ; CHECK-EMPTY: ; CHECK-NEXT: bb1: -; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0> -; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, ir<%u0> -; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, ir<%u0> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1> +; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep> +; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1> ; CHECK-NEXT: Successor(s): bb2 ; CHECK-EMPTY: ; CHECK-NEXT: bb2: -; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%u0>, ir<%v1> -; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]> -; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%v1> +; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%v1> ; CHECK-NEXT: Successor(s): bb3 ; CHECK-EMPTY: ; CHECK-NEXT: bb3: @@ -1216,20 +1213,18 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: Successor(s): bb2, bb1 ; CHECK-EMPTY: ; CHECK-NEXT: bb2: -; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0> +; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2> ; CHECK-NEXT: Successor(s): bb3 ; CHECK-EMPTY: ; CHECK-NEXT: bb3: -; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2> -; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]> -; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]> -; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP6]]> +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v2> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP4]]> +; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP4]]> ; CHECK-NEXT: Successor(s): bb4 ; CHECK-EMPTY: ; CHECK-NEXT: bb4: -; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2> -; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]> -; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP7]]> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v2> +; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, ir<%v2> ; CHECK-NEXT: Successor(s): bb5 ; CHECK-EMPTY: ; CHECK-NEXT: bb5: @@ -1243,9 +1238,8 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: bb1: -; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0> -; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> -; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1> +; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep> ; CHECK-NEXT: Successor(s): bb6 ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): middle.block @@ -1404,9 +1398,8 @@ define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) { ; CHECK-NEXT: Successor(s): bb2, bb3 ; CHECK-EMPTY: ; CHECK-NEXT: bb2: -; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1> -; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP5]]> -; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP5]]> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]> +; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]> ; CHECK-NEXT: Successor(s): bb3 ; CHECK-EMPTY: ; CHECK-NEXT: bb3: @@ -1583,9 +1576,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 % ; CHECK-NEXT: Successor(s): bb3, bb2 ; CHECK-EMPTY: ; CHECK-NEXT: bb3: -; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1> -; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]> -; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP7]]> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP4]]> +; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP4]]> ; CHECK-NEXT: Successor(s): bb4 ; CHECK-EMPTY: ; CHECK-NEXT: bb4: @@ -1604,10 +1596,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 % ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: bb2: -; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1> -; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> -; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]> -; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP6]]> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]> +; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]> ; CHECK-NEXT: Successor(s): bb4 ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): middle.block @@ -1779,9 +1769,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr ; CHECK-NEXT: Successor(s): bb4, bb3 ; CHECK-EMPTY: ; CHECK-NEXT: bb4: -; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%v0>, ir<%u2> -; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]> -; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP7]]> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v0> +; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, ir<%v0> ; CHECK-NEXT: Successor(s): bb5 ; CHECK-EMPTY: ; CHECK-NEXT: bb5: @@ -1795,10 +1784,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: bb3: -; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u2> -; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%v0>, vp<[[VP5]]> -; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]> -; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP6]]> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%v0> +; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, ir<%v0> ; CHECK-NEXT: Successor(s): bb5 ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): middle.block @@ -2823,15 +2810,14 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: Successor(s): bb2, bb1 ; CHECK-EMPTY: ; CHECK-NEXT: bb2: -; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0> -; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%u0> -; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2> +; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep> +; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2> ; CHECK-NEXT: Successor(s): bb3 ; CHECK-EMPTY: ; CHECK-NEXT: bb3: -; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2> -; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP5]]> -; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP5]]> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%v2> +; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, ir<%v2> ; CHECK-NEXT: Successor(s): bb4 ; CHECK-EMPTY: ; CHECK-NEXT: bb4: @@ -2842,9 +2828,8 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: bb1: -; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0> -; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> -; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1> +; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep> ; CHECK-NEXT: Successor(s): bb4 ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): middle.block diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll index c846b26096772..6595787aefd9d 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll @@ -32,23 +32,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: Successor(s): if.then, for.inc ; CHECK-EMPTY: ; CHECK-NEXT: if.then: -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: <xVFxUF> pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK ir<%c> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): if.then.0 -; CHECK-EMPTY: -; CHECK-NEXT: if.then.0: +; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2> ; CHECK-NEXT: Successor(s): for.inc ; CHECK-EMPTY: ; CHECK-NEXT: for.inc: diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll index c8503ebab5e45..43acb39578a7d 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll @@ -1164,27 +1164,13 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) { ; CHECK-NEXT: Successor(s): loop.then, loop.latch ; CHECK-EMPTY: ; CHECK-NEXT: loop.then: -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: <xVFxUF> pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK ir<%c> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src> -; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1> -; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0> -; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2> -; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.then.1 -; CHECK-EMPTY: -; CHECK-NEXT: loop.then.1: +; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src> +; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1> +; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0> +; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2> +; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = extract-last-part ir<%sel> +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = extract-last-lane vp<[[VP3]]> +; CHECK-NEXT: CLONE store vp<[[VP4]]>, ir<%dst> ; CHECK-NEXT: Successor(s): loop.latch ; CHECK-EMPTY: ; CHECK-NEXT: loop.latch: diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll index 4faabbc7b8f35..ba84fbbd5eeda 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll @@ -125,8 +125,6 @@ define void @test_scalar_cost_single_store_loop_invariant_cond(ptr %dst, i1 %c) ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 96 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ] @@ -134,7 +132,7 @@ define void @test_scalar_cost_single_store_loop_invariant_cond(ptr %dst, i1 %c) ; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]] ; CHECK-NEXT: br i1 [[C]], label %[[IF_THEN1:.*]], label %[[LOOP_LATCH2]] ; CHECK: [[IF_THEN1]]: -; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[NEXT_GEP]], <8 x i1> [[BROADCAST_SPLAT]]) +; CHECK-NEXT: store <8 x i32> zeroinitializer, ptr [[NEXT_GEP]], align 4 ; CHECK-NEXT: br label %[[LOOP_LATCH2]] ; CHECK: [[LOOP_LATCH2]]: ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 diff --git a/llvm/test/Transforms/LoopVectorize/div-exact.ll b/llvm/test/Transforms/LoopVectorize/div-exact.ll index 7bec969285194..b302e88a99656 100644 --- a/llvm/test/Transforms/LoopVectorize/div-exact.ll +++ b/llvm/test/Transforms/LoopVectorize/div-exact.ll @@ -14,76 +14,18 @@ define void @unknown_divisor(ptr noalias %p, i32 %n, i1 %c) { ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 3 ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ] -; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1 -; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2 -; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3 +; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ] ; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH14]] ; CHECK: [[IF1]]: -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]] -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP2]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP3]] ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP4]] -; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] -; CHECK: [[PRED_LOAD_IF]]: -; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP7]], align 4 -; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0 -; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] -; CHECK: [[PRED_LOAD_CONTINUE]]: -; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[IF1]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ] -; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]] -; CHECK: [[PRED_LOAD_IF3]]: -; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4 -; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i64 1 -; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]] -; CHECK: [[PRED_LOAD_CONTINUE4]]: -; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], %[[PRED_LOAD_IF3]] ] -; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]] -; CHECK: [[PRED_LOAD_IF5]]: -; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP9]], align 4 -; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i64 2 -; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]] -; CHECK: [[PRED_LOAD_CONTINUE6]]: -; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP18]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ] -; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]] -; CHECK: [[PRED_LOAD_IF7]]: -; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP24]], i64 3 -; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]] -; CHECK: [[PRED_LOAD_CONTINUE8]]: -; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP25]], %[[PRED_LOAD_IF7]] ] -; CHECK-NEXT: [[TMP28:%.*]] = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> splat (i32 100), <4 x i32> [[TMP23]], <4 x i1> [[BROADCAST_SPLAT]]) -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] -; CHECK: [[PRED_STORE_IF]]: -; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP28]], i64 0 -; CHECK-NEXT: store i32 [[TMP29]], ptr [[TMP7]], align 4 -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]] -; CHECK: [[PRED_STORE_CONTINUE]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]] -; CHECK: [[PRED_STORE_IF9]]: -; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP28]], i64 1 -; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP8]], align 4 -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]] -; CHECK: [[PRED_STORE_CONTINUE10]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]] -; CHECK: [[PRED_STORE_IF11]]: -; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP28]], i64 2 -; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP9]], align 4 -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]] -; CHECK: [[PRED_STORE_CONTINUE12]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]] -; CHECK: [[PRED_STORE_IF13]]: -; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i32> [[TMP28]], i64 3 -; CHECK-NEXT: store i32 [[TMP32]], ptr [[TMP10]], align 4 -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]] -; CHECK: [[PRED_STORE_CONTINUE14]]: +; CHECK-NEXT: [[TMP25:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4 +; CHECK-NEXT: [[TMP28:%.*]] = udiv exact <4 x i32> splat (i32 100), [[TMP25]] +; CHECK-NEXT: store <4 x i32> [[TMP28]], ptr [[TMP10]], align 4 ; CHECK-NEXT: br label %[[LATCH14]] ; CHECK: [[LATCH14]]: -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP4]], 4 ; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: @@ -146,72 +88,16 @@ define void @unknown_dividend(ptr noalias %p, i32 %n, i1 %c) { ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ] -; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1 -; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2 -; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3 +; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ] ; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH14]] ; CHECK: [[IF1]]: -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]] -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP2]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP3]] ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP4]] -; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] -; CHECK: [[PRED_LOAD_IF]]: -; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP7]], align 4 -; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0 -; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] -; CHECK: [[PRED_LOAD_CONTINUE]]: -; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[IF1]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ] -; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]] -; CHECK: [[PRED_LOAD_IF3]]: -; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4 -; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i64 1 -; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]] -; CHECK: [[PRED_LOAD_CONTINUE4]]: -; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], %[[PRED_LOAD_IF3]] ] -; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]] -; CHECK: [[PRED_LOAD_IF5]]: -; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP9]], align 4 -; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i64 2 -; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]] -; CHECK: [[PRED_LOAD_CONTINUE6]]: -; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP18]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ] -; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]] -; CHECK: [[PRED_LOAD_IF7]]: -; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP24]], i64 3 -; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]] -; CHECK: [[PRED_LOAD_CONTINUE8]]: -; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP25]], %[[PRED_LOAD_IF7]] ] -; CHECK-NEXT: [[TMP27:%.*]] = udiv exact <4 x i32> [[TMP23]], splat (i32 100) -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] -; CHECK: [[PRED_STORE_IF]]: -; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[TMP27]], i64 0 -; CHECK-NEXT: store i32 [[TMP28]], ptr [[TMP7]], align 4 -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]] -; CHECK: [[PRED_STORE_CONTINUE]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]] -; CHECK: [[PRED_STORE_IF9]]: -; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP27]], i64 1 -; CHECK-NEXT: store i32 [[TMP29]], ptr [[TMP8]], align 4 -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]] -; CHECK: [[PRED_STORE_CONTINUE10]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]] -; CHECK: [[PRED_STORE_IF11]]: -; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP27]], i64 2 -; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP9]], align 4 -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]] -; CHECK: [[PRED_STORE_CONTINUE12]]: -; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]] -; CHECK: [[PRED_STORE_IF13]]: -; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP27]], i64 3 -; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP10]], align 4 -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]] -; CHECK: [[PRED_STORE_CONTINUE14]]: +; CHECK-NEXT: [[TMP25:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4 +; CHECK-NEXT: [[TMP27:%.*]] = udiv exact <4 x i32> [[TMP25]], splat (i32 100) +; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP10]], align 4 ; CHECK-NEXT: br label %[[LATCH14]] ; CHECK: [[LATCH14]]: -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP4]], 4 ; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll index b57899e31811a..d89ad9ed28ea9 100644 --- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll +++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll @@ -418,26 +418,18 @@ define void @minimal_bit_widths(ptr %p, i1 %c) { ; UNROLL: [[VECTOR_PH]]: ; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]] ; UNROLL: [[VECTOR_BODY]]: -; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] +; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ] ; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1 ; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]] ; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]] ; UNROLL-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1 ; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1 -; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; UNROLL: [[PRED_STORE_IF]]: -; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]] +; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]] ; UNROLL: [[PRED_STORE_IF2]]: ; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1 +; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1 ; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]] ; UNROLL: [[PRED_STORE_CONTINUE1]]: -; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] -; UNROLL: [[PRED_STORE_IF1]]: -; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1 -; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]] -; UNROLL: [[PRED_STORE_CONTINUE2]]: -; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]] -; UNROLL: [[PRED_STORE_CONTINUE]]: ; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 ; UNROLL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -453,26 +445,18 @@ define void @minimal_bit_widths(ptr %p, i1 %c) { ; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]: ; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]] ; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]: -; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] +; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ] ; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1 ; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]] ; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]] ; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1 ; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1 -; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]: -; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]] +; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]] ; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]: ; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1 +; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1 ; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]] ; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]: -; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]: -; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1 -; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]: -; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]: ; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 ; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -488,28 +472,15 @@ define void @minimal_bit_widths(ptr %p, i1 %c) { ; VEC: [[VECTOR_PH]]: ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] ; VEC: [[VECTOR_BODY]]: -; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] -; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]] -; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1 -; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; VEC: [[PRED_STORE_IF]]: -; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]] +; VEC-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ] +; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]] +; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP3]], align 1 +; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]] ; VEC: [[PRED_STORE_IF2]]: -; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0 -; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1 +; VEC-NEXT: store <2 x i8> [[WIDE_LOAD]], ptr [[TMP3]], align 1 ; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]] ; VEC: [[PRED_STORE_CONTINUE1]]: -; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] -; VEC: [[PRED_STORE_IF1]]: -; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1 -; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]] -; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1 -; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1 -; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]] -; VEC: [[PRED_STORE_CONTINUE2]]: -; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]] -; VEC: [[PRED_STORE_CONTINUE]]: -; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2 ; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 ; VEC-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; VEC: [[MIDDLE_BLOCK]]: @@ -551,7 +522,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) { ; UNROLL: [[VECTOR_PH]]: ; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]] ; UNROLL: [[VECTOR_BODY]]: -; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] +; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ] ; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1 ; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]] ; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]] @@ -559,20 +530,12 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) { ; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1 ; UNROLL-NEXT: store i8 0, ptr [[TMP1]], align 1 ; UNROLL-NEXT: store i8 0, ptr [[TMP2]], align 1 -; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; UNROLL: [[PRED_STORE_IF]]: -; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]] +; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]] ; UNROLL: [[PRED_STORE_IF2]]: ; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1 +; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1 ; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]] ; UNROLL: [[PRED_STORE_CONTINUE1]]: -; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] -; UNROLL: [[PRED_STORE_IF1]]: -; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1 -; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]] -; UNROLL: [[PRED_STORE_CONTINUE2]]: -; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]] -; UNROLL: [[PRED_STORE_CONTINUE]]: ; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 ; UNROLL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -588,7 +551,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) { ; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]: ; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]] ; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]: -; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] +; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ] ; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1 ; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]] ; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]] @@ -596,20 +559,12 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) { ; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1 ; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP1]], align 1 ; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP2]], align 1 -; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]: -; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]] +; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]] ; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]: ; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1 +; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1 ; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]] ; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]: -; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]: -; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1 -; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]: -; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]: ; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 ; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -625,29 +580,16 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) { ; VEC: [[VECTOR_PH]]: ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] ; VEC: [[VECTOR_BODY]]: -; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] -; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]] -; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1 -; VEC-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP0]], align 1 -; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; VEC: [[PRED_STORE_IF]]: -; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]] +; VEC-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ] +; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP2]] +; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP3]], align 1 +; VEC-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP3]], align 1 +; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]] ; VEC: [[PRED_STORE_IF2]]: -; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0 -; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1 +; VEC-NEXT: store <2 x i8> [[WIDE_LOAD]], ptr [[TMP3]], align 1 ; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]] ; VEC: [[PRED_STORE_CONTINUE1]]: -; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] -; VEC: [[PRED_STORE_IF1]]: -; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1 -; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP2]] -; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1 -; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1 -; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]] -; VEC: [[PRED_STORE_CONTINUE2]]: -; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]] -; VEC: [[PRED_STORE_CONTINUE]]: -; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2 ; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 ; VEC-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; VEC: [[MIDDLE_BLOCK]]: @@ -685,31 +627,21 @@ for.end: define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) { ; UNROLL-LABEL: define void @sdiv_with_uniform_ops( ; UNROLL-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) { -; UNROLL-NEXT: [[ENTRY:.*:]] -; UNROLL-NEXT: br label %[[VECTOR_PH:.*]] -; UNROLL: [[VECTOR_PH]]: +; UNROLL-NEXT: [[VECTOR_PH:.*:]] ; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]] ; UNROLL: [[VECTOR_BODY]]: -; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] -; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; UNROLL: [[PRED_STORE_IF]]: -; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]] -; UNROLL: [[PRED_STORE_IF2]]: ; UNROLL-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]] -; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1 -; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]] +; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]] ; UNROLL: [[PRED_STORE_CONTINUE1]]: -; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] +; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ] +; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]] ; UNROLL: [[PRED_STORE_IF1]]: -; UNROLL-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]] -; UNROLL-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1 +; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1 ; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]] ; UNROLL: [[PRED_STORE_CONTINUE2]]: -; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]] -; UNROLL: [[PRED_STORE_CONTINUE]]: ; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 ; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98 -; UNROLL-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; UNROLL-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]] ; UNROLL: [[MIDDLE_BLOCK]]: ; UNROLL-NEXT: br label %[[SCALAR_PH:.*]] ; UNROLL: [[SCALAR_PH]]: @@ -730,31 +662,21 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) { ; ; UNROLL-NOSIMPLIFY-LABEL: define void @sdiv_with_uniform_ops( ; UNROLL-NOSIMPLIFY-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) { -; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]] -; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_PH:.*]] -; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]: +; UNROLL-NOSIMPLIFY-NEXT: [[VECTOR_PH:.*:]] ; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]] ; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]: -; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] -; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]: -; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]: ; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]] -; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1 -; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]] +; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]] ; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]: -; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] +; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ] +; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]] ; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]: -; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]] -; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1 +; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1 ; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]] ; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]: -; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]] -; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]: ; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 ; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98 -; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]] ; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]: ; UNROLL-NOSIMPLIFY-NEXT: br label %[[SCALAR_PH:.*]] ; UNROLL-NOSIMPLIFY: [[SCALAR_PH]]: @@ -775,36 +697,21 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) { ; ; VEC-LABEL: define void @sdiv_with_uniform_ops( ; VEC-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) { -; VEC-NEXT: [[ENTRY:.*:]] -; VEC-NEXT: br label %[[VECTOR_PH:.*]] -; VEC: [[VECTOR_PH]]: -; VEC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0 -; VEC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer -; VEC-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C]], i64 0 -; VEC-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer -; VEC-NEXT: [[TMP1:%.*]] = call <2 x i16> @llvm.masked.sdiv.v2i16(<2 x i16> splat (i16 10), <2 x i16> [[BROADCAST_SPLAT]], <2 x i1> [[BROADCAST_SPLAT2]]) +; VEC-NEXT: [[VECTOR_PH:.*:]] ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] ; VEC: [[VECTOR_BODY]]: -; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ] -; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]] -; VEC: [[PRED_STORE_IF]]: -; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]] -; VEC: [[PRED_STORE_IF1]]: -; VEC-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[TMP1]], i64 0 -; VEC-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1 -; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]] +; VEC-NEXT: [[TMP3:%.*]] = sdiv i16 10, [[TMP0]] +; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]] ; VEC: [[PRED_STORE_CONTINUE1]]: -; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]] +; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE4:.*]] ] +; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]] ; VEC: [[PRED_STORE_IF3]]: -; VEC-NEXT: [[TMP3:%.*]] = extractelement <2 x i16> [[TMP1]], i64 1 ; VEC-NEXT: store i16 [[TMP3]], ptr [[DST]], align 1 ; VEC-NEXT: br label %[[PRED_STORE_CONTINUE4]] ; VEC: [[PRED_STORE_CONTINUE4]]: -; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]] -; VEC: [[PRED_STORE_CONTINUE]]: ; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 ; VEC-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98 -; VEC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; VEC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]] ; VEC: [[MIDDLE_BLOCK]]: ; VEC-NEXT: br label %[[SCALAR_PH:.*]] ; VEC: [[SCALAR_PH]]: diff --git a/llvm/test/Transforms/LoopVectorize/pr37248.ll b/llvm/test/Transforms/LoopVectorize/pr37248.ll index 54109d418f5ba..9d86032bd27bc 100644 --- a/llvm/test/Transforms/LoopVectorize/pr37248.ll +++ b/llvm/test/Transforms/LoopVectorize/pr37248.ll @@ -35,23 +35,13 @@ define void @f1(ptr noalias %b, i1 %c, i32 %start) { ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP1]], 1 ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]] ; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[START]], [[N_VEC]] -; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[C]], true ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH4:.*]] ] ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[START]], [[INDEX]] ; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH4]], label %[[ELSE1:.*]] ; CHECK: [[ELSE1]]: -; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] -; CHECK: [[PRED_STORE_IF]]: ; CHECK-NEXT: store i32 10, ptr [[B]], align 1 -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]] -; CHECK: [[PRED_STORE_CONTINUE]]: -; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]] -; CHECK: [[PRED_STORE_IF2]]: -; CHECK-NEXT: store i32 10, ptr [[B]], align 1 -; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]] -; CHECK: [[PRED_STORE_CONTINUE3]]: ; CHECK-NEXT: br label %[[LOOP_LATCH4]] ; CHECK: [[LOOP_LATCH4]]: ; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[OFFSET_IDX]] to i16 diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll index 6623c01a01ab5..3e9b0dfbe2caa 100644 --- a/llvm/test/Transforms/LoopVectorize/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/predicator.ll @@ -377,8 +377,6 @@ define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB42:.*]] ] @@ -388,11 +386,10 @@ define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: br i1 [[U0]], label %[[BB11:.*]], label %[[BB42]] ; CHECK: [[BB11]]: ; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP1]], ptr align 4 [[TMP0]], <4 x i1> [[BROADCAST_SPLAT]]) +; CHECK-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP0]], align 4 ; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 1) -; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer ; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP0]], <4 x i1> [[TMP3]]) +; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP0]], <4 x i1> [[TMP2]]) ; CHECK-NEXT: br label %[[BB42]] ; CHECK: [[BB42]]: ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 @@ -531,9 +528,6 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB63:.*]] ] @@ -542,17 +536,15 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: br i1 [[U0]], label %[[BB22:.*]], label %[[BB11:.*]] ; CHECK: [[BB11]]: ; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]]) +; CHECK-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP1]], align 4 ; CHECK-NEXT: br label %[[BB63]] ; CHECK: [[BB22]]: ; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2) ; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true) -; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]]) -; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer +; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP7]]) +; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]]) ; CHECK-NEXT: br label %[[BB63]] ; CHECK: [[BB63]]: ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 @@ -693,8 +685,6 @@ define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB32:.*]] ] @@ -706,9 +696,8 @@ define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) { ; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP0]], <4 x i1> [[TMP2]]) ; CHECK-NEXT: br i1 [[U1]], label %[[BB21:.*]], label %[[BB32]] ; CHECK: [[BB21]]: -; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer ; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP0]], <4 x i1> [[TMP4]]) +; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP0]], <4 x i1> [[TMP2]]) ; CHECK-NEXT: br label %[[BB32]] ; CHECK: [[BB32]]: ; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) @@ -851,9 +840,6 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 % ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ] @@ -863,14 +849,12 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 % ; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true) ; CHECK-NEXT: br i1 [[U1]], label %[[BB32:.*]], label %[[BB21:.*]] ; CHECK: [[BB21]]: -; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer ; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]]) +; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]]) ; CHECK-NEXT: br label %[[BB43]] ; CHECK: [[BB32]]: -; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer ; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP7]], ptr align 4 [[TMP1]], <4 x i1> [[TMP6]]) +; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP7]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]]) ; CHECK-NEXT: br label %[[BB43]] ; CHECK: [[BB43]]: ; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5) @@ -1018,9 +1002,6 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U2]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB53:.*]] ] @@ -1032,14 +1013,12 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr ; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]]) ; CHECK-NEXT: br i1 [[U2]], label %[[BB42:.*]], label %[[BB31:.*]] ; CHECK: [[BB31]]: -; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]]) +; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP2]]) ; CHECK-NEXT: br label %[[BB53]] ; CHECK: [[BB42]]: -; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer ; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP7]]) +; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP2]]) ; CHECK-NEXT: br label %[[BB53]] ; CHECK: [[BB53]]: ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 @@ -2074,9 +2053,6 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ] @@ -2085,15 +2061,14 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) { ; CHECK-NEXT: br i1 [[U0]], label %[[BB22:.*]], label %[[BB11:.*]] ; CHECK: [[BB11]]: ; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]]) +; CHECK-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP1]], align 4 ; CHECK-NEXT: br label %[[BB43]] ; CHECK: [[BB22]]: ; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]]) +; CHECK-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP1]], align 4 ; CHECK-NEXT: [[TMP4:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2) -; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) -; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]]) +; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]]) ; CHECK-NEXT: br label %[[BB43]] ; CHECK: [[BB43]]: ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
