https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/237254
>From 7c7ccc3246a71cdfbc1b10d39c6a7a519dbea196 Mon Sep 17 00:00:00 2001 From: Luke Lau <[email protected]> Date: Wed, 28 Oct 2026 22:33:08 +0800 Subject: [PATCH 1/2] Precommit tests --- .../LoopVectorize/early_exit_with_stores.ll | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll index ad1269aa3c8523..139c9992cc38d7 100644 --- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll +++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll @@ -1555,3 +1555,224 @@ exit: } declare i64 @get_an_unknown_offset(); + +define void @store_in_latch_after_exit(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) { +; CHECK-LABEL: define void @store_in_latch_after_exit( +; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2 +; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) +; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[TMP3]], <4 x i16> poison) +; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1) +; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP5]], ptr align 2 [[TMP4]], <4 x i1> [[TMP3]]) +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 +; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] +; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) +; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20 +; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] +; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2 +; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500 +; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]] +; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2 +; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1 +; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20 +; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv + %ee.val = load i16, ptr %ee.addr + %ee.cond = icmp sgt i16 %ee.val, 500 + br i1 %ee.cond, label %exit, label %loop.latch + +loop.latch: + %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv + %data = load i16, ptr %st.addr + %inc = add nsw i16 %data, 1 + store i16 %inc, ptr %st.addr + %iv.next = add nuw nsw i64 %iv, 1 + %counted.cond = icmp eq i64 %iv.next, 20 + br i1 %counted.cond, label %exit, label %loop + +exit: + ret void +} + +define void @conditional_store_after_two_exits(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) { +; CHECK-LABEL: define void @conditional_store_after_two_exits( +; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED2:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] +; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2 +; CHECK-NEXT: [[EE2_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[IV]] +; CHECK-NEXT: [[EE2_VAL:%.*]] = load i16, ptr [[EE2_ADDR]], align 2 +; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500 +; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LOOP_2:.*]] +; CHECK: [[LOOP_2]]: +; CHECK-NEXT: [[EE2_COND:%.*]] = icmp eq i16 [[EE2_VAL]], 0 +; CHECK-NEXT: br i1 [[EE2_COND]], label %[[EXIT_2:.*]], label %[[LOOP_3:.*]] +; CHECK: [[LOOP_3]]: +; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]] +; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2 +; CHECK-NEXT: [[ST_COND:%.*]] = icmp slt i16 [[DATA]], 0 +; CHECK-NEXT: br i1 [[ST_COND]], label %[[STORE:.*]], label %[[LOOP_LATCH]] +; CHECK: [[STORE]]: +; CHECK-NEXT: [[INC:%.*]] = add i16 [[DATA]], 1 +; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2 +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20 +; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; CHECK: [[EXIT_2]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv + %ee.val = load i16, ptr %ee.addr + %ee2.addr = getelementptr inbounds nuw i16, ptr %pred2, i64 %iv + %ee2.val = load i16, ptr %ee2.addr + %ee.cond = icmp sgt i16 %ee.val, 500 + br i1 %ee.cond, label %exit, label %loop.2 + +loop.2: + %ee2.cond = icmp eq i16 %ee2.val, 0 + br i1 %ee2.cond, label %exit.2, label %loop.3 + +loop.3: + %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv + %data = load i16, ptr %st.addr + %st.cond = icmp slt i16 %data, 0 + br i1 %st.cond, label %store, label %loop.latch + +store: + %inc = add i16 %data, 1 + store i16 %inc, ptr %st.addr + br label %loop.latch + +loop.latch: + %iv.next = add nuw nsw i64 %iv, 1 + %counted.cond = icmp eq i64 %iv.next, 20 + br i1 %counted.cond, label %exit, label %loop + +exit: + ret void + +exit.2: + ret void +} + +define i16 @store_after_exit_live_out(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) { +; CHECK-LABEL: define i16 @store_after_exit_live_out( +; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2 +; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) +; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[TMP3]], <4 x i16> poison) +; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1) +; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP5]], ptr align 2 [[TMP4]], <4 x i1> [[TMP3]]) +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 +; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] +; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3 +; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20 +; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP11]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] +; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2 +; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500 +; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_STORE:.*]] +; CHECK: [[LOOP_STORE]]: +; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]] +; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2 +; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1 +; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2 +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20 +; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[LOOP]] ], [ [[INC]], %[[LOOP_LATCH]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i16 [[RES]] +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv + %ee.val = load i16, ptr %ee.addr + %ee.cond = icmp sgt i16 %ee.val, 500 + br i1 %ee.cond, label %exit, label %loop.store + +loop.store: + %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv + %data = load i16, ptr %st.addr + %inc = add nsw i16 %data, 1 + store i16 %inc, ptr %st.addr + br label %loop.latch + +loop.latch: + %iv.next = add nuw nsw i64 %iv, 1 + %counted.cond = icmp eq i64 %iv.next, 20 + br i1 %counted.cond, label %exit, label %loop + +exit: + %res = phi i16 [ -1, %loop ], [ %inc, %loop.latch ] + ret i16 %res +} >From a76b2c27da85f5cc5fca006d99d4665ca084a414 Mon Sep 17 00:00:00 2001 From: Luke Lau <[email protected]> Date: Fri, 30 Oct 2026 00:16:00 +0800 Subject: [PATCH 2/2] [VPlan] Allow early exit loops with side effects by masking in the vector loop. When all blocks with side effects are dominated by every early exit, we can vectorize the loop if we mask off any lanes past the exit. We can do this with control flow: ```mermaid flowchart TD header --> exiting exiting -->| exit mask | sideeffects --> latch --> latch.split exiting -->| !exit mask | latch.split ``` For every early exiting block we add a branch to the latch on `!mask.beforefirst(exitcond)`. This is implemented in UncountableExitStyle::ReadOnly which is now renamed to MaskedHandleExitInVectorLoop. This is a complementary exit style to MaskedHandleExitInScalarLoop. If any of the side effects aren't dominated by the early exit it still falls back to the scalar loop style, which is why none of the existing early exit side effect tests changed. Masking in loop is profitable for RISC-V and I'm assuming it's also profitable for AArch64 too, but if not then we can add a TTI hook for a preferred exit style. --- .../Transforms/Vectorize/LoopVectorize.cpp | 3 +- llvm/lib/Transforms/Vectorize/VPlan.h | 9 +- .../Vectorize/VPlanConstruction.cpp | 4 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 119 ++++++++++----- .../RISCV/early_exit_with_stores.ll | 15 +- .../X86/early_exit_with_stores.ll | 19 +-- .../LoopVectorize/early_exit_with_stores.ll | 135 ++++++++---------- .../Vectorize/VPlanVerifierTest.cpp | 5 +- 8 files changed, 176 insertions(+), 133 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index c2674a2d6ae801..679c36f768a5ff 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6267,7 +6267,8 @@ LoopVectorizationPlanner::tryToBuildVPlan1(LoopVectorizationCostModel &PlanCM) { return nullptr; // TODO: Check target preference for style. - UncountableExitStyle EEStyle = UncountableExitStyle::ReadOnly; + UncountableExitStyle EEStyle = + UncountableExitStyle::MaskedHandleExitInVectorLoop; if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0, ORE, OrigLoop, PSE, *DT, Legal->getAssumptionCache(), EEStyle)) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 544895c053b43a..0e322a13054370 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -79,10 +79,11 @@ using VPlanPtr = std::unique_ptr<VPlan>; /// Different methods of handling early exits. /// enum class UncountableExitStyle { - /// No side effects to worry about, so we can process any uncountable exits - /// in the loop and branch either to the middle block if the trip count was - /// reached, or an early exitblock to determine which exit was taken. - ReadOnly, + /// Any side effects come after all the uncountable exits and are masked off, + /// so we can process any uncountable exits in the loop and branch either to + /// the middle block if the trip count was reached, or an early exitblock to + /// determine which exit was taken. + MaskedHandleExitInVectorLoop, /// All memory operations other than the load(s) required to determine whether /// an uncountable exit occurre will be masked based on that condition. If an /// uncountable exit is taken, then all lanes before the exiting lane will diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 8a026aba613855..db47293ca8476d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -1272,7 +1272,9 @@ bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB, continue; auto *VPI = dyn_cast<VPInstruction>(&R); if (!VPI || VPI->getOpcode() != Instruction::Load) { - assert(!R.mayReadFromMemory() && "unexpected recipe reading memory"); + assert((!R.mayReadFromMemory() || + (VPI && VPI->getOpcode() == Instruction::Store)) && + "unexpected recipe reading memory"); continue; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 1c995a629f1468..a43360f33c8372 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3554,50 +3554,59 @@ static bool handleUncountableExitsWithSideEffects( // Returns true if the loop contains stores, the only writes legality accepts in // early-exit loops. -static bool loopHasStores(VPBasicBlock *HeaderVPBB) { - return any_of(vp_rpo_plain_cfg_loop_body(HeaderVPBB), [](VPBasicBlock *VPBB) { - return any_of(*VPBB, [VPBB](VPRecipeBase &R) { - (void)VPBB; - if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>())) - return false; - if (!R.mayWriteToMemory()) { - assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() || - R.isSafeToSpeculativelyExecute()) && - "legality only accepts recipes with side effects that are " - "safe to speculate"); - return false; - } - assert(match(&R, m_VPInstruction<Instruction::Store>()) && - "legality only accepts stores as writes in early-exit loops"); - return true; - }); - }); +static SmallVector<VPBasicBlock *> blocksWithStores(VPBasicBlock *HeaderVPBB) { + return to_vector(make_filter_range( + vp_rpo_plain_cfg_loop_body(HeaderVPBB), [](VPBasicBlock *VPBB) { + return any_of(*VPBB, [VPBB](VPRecipeBase &R) { + (void)VPBB; + if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>())) + return false; + if (!R.mayWriteToMemory()) { + assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() || + R.isSafeToSpeculativelyExecute()) && + "legality only accepts recipes with side effects that are " + "safe to speculate"); + return false; + } + assert(match(&R, m_VPInstruction<Instruction::Store>()) && + "legality only accepts stores as writes in early-exit loops"); + return true; + }); + })); } bool VPlanTransforms::handleUncountableEarlyExits( VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style) { -#ifndef NDEBUG VPDominatorTree VPDT(Plan); -#endif auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan); auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan); - if (loopHasStores(HeaderVPBB)) - Style = UncountableExitStyle::MaskedHandleExitInScalarLoop; + // If all side effects are dominated by the early exits, we can mask in the + // vector loop. Otherwise we need to handle side effects in the scalar loop. + // TODO: Legality only allows side effects that the scalar loop style can + // handle, relax it for side effects after all the early exits. + SmallVector<VPBasicBlock *> SideEffectVPBBs = blocksWithStores(HeaderVPBB); + if (!SideEffectVPBBs.empty()) { + auto EarlyExits = vputils::getEarlyExits(Plan, MiddleVPBB); + for (auto [EarlyExitingVPBB, _] : EarlyExits) + for (VPBasicBlock *VPBB : SideEffectVPBBs) + if (!VPDT.properlyDominates(EarlyExitingVPBB, VPBB)) + Style = UncountableExitStyle::MaskedHandleExitInScalarLoop; + } - // Dereferenceability is checked separately for uncountable exit loops with - // stores, as only the loads contributing to the exit condition need to - // be checked. - if (Style == UncountableExitStyle::ReadOnly && + if (Style == UncountableExitStyle::MaskedHandleExitInVectorLoop && !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC)) { - reportVectorizationFailure( - "Auto-vectorization of early exit loops with potentially " - "faulting loads is not supported", - "EarlyExitFaultingLoads", ORE, TheLoop); - return false; + if (SideEffectVPBBs.empty()) { + reportVectorizationFailure( + "Auto-vectorization of early exit loops with potentially " + "faulting loads is not supported", + "EarlyExitFaultingLoads", ORE, TheLoop); + return false; + } + Style = UncountableExitStyle::MaskedHandleExitInScalarLoop; } VPBuilder LatchBuilder(LatchVPBB->getTerminator()); @@ -3649,7 +3658,7 @@ bool VPlanTransforms::handleUncountableEarlyExits( }); } - if (Style == UncountableExitStyle::ReadOnly) { + if (Style == UncountableExitStyle::MaskedHandleExitInVectorLoop) { // Lanes past the exit can be poison that the scalar loop never branched on, // so freeze them to prevent branching on poison. for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) { @@ -3659,6 +3668,49 @@ bool VPlanTransforms::handleUncountableEarlyExits( Term->setOperand(0, VPBuilder(Term).createFreeze(Term->getOperand(0))); } + if (!SideEffectVPBBs.empty()) { + // Split the latch in case it has recipes with side effects. + LatchVPBB = LatchVPBB->splitAt(LatchVPBB->getTerminator()->getIterator()); + LatchBuilder.setInsertPoint(LatchVPBB->getTerminator()); + + // Branch to the latch to skip any side effects past the exiting lane. + for (EarlyExitInfo &Exit : Exits) { + VPBasicBlock *EarlyExitingVPBB = Exit.EarlyExitingVPBB; + VPBuilder ExitingBuilder(EarlyExitingVPBB->getTerminator()); + VPValue *ExitMask = ExitingBuilder.createWidenIntrinsic( + Intrinsic::mask_beforefirst, + ExitingBuilder.createFreeze(ExitingBuilder.createNaryOp( + VPInstruction::MaskedCond, Exit.CondToExit)), + Type::getInt1Ty(Plan.getContext())); + EarlyExitingVPBB->getTerminator()->setOperand(0, ExitMask); + VPBlockUtils::connectBlocks(EarlyExitingVPBB, LatchVPBB); + } + + // The latch's exit condition may be poison on lanes past the exit, freeze + // it. + VPRecipeBase *LatchTerm = LatchVPBB->getTerminator(); + LatchTerm->setOperand( + 0, VPBuilder(LatchTerm).createFreeze(LatchTerm->getOperand(0))); + + // If we took the exiting branch then any uses in the latch and beyond are + // poison. + VPDT.recalculate(Plan); + for (VPBasicBlock *VPBB : {LatchVPBB, MiddleVPBB}) { + for (VPRecipeBase &R : + make_range(VPBB->getFirstNonPhi(), VPBB->end())) { + for (auto [I, Op] : enumerate(R.operands())) { + VPRecipeBase *DefR = Op->getDefiningRecipe(); + if (!DefR || VPDT.dominates(DefR->getParent(), VPBB)) + continue; + DenseMap<VPBasicBlock *, VPValue *> Defs = { + {HeaderVPBB, Plan.getPoison(Op->getScalarType())}}; + Defs[DefR->getParent()] = Op; + R.setOperand(I, vputils::reconstructSSA(LatchVPBB, Defs)); + } + } + } + } + // An exit's condition is false on any path that didn't go through its // exiting block. for (EarlyExitInfo &Exit : Exits) { @@ -3816,7 +3868,8 @@ bool VPlanTransforms::handleUncountableEarlyExits( ExitIRI->addIncoming(NewIncoming); } - EarlyExitingVPBB->getTerminator()->eraseFromParent(); + if (EarlyExitingVPBB->getNumSuccessors() == 2) + EarlyExitingVPBB->getTerminator()->eraseFromParent(); VPBlockUtils::disconnectBlocks(EarlyExitingVPBB, EarlyExitVPBB); VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, EarlyExitVPBB); } diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll index 573ee73e76a439..8ba6d5f81679e5 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll @@ -1277,7 +1277,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]] ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ] ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 2 ; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500) @@ -1291,15 +1291,16 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d ; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP4]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] -; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false) -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]] -; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20 +; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK1]]: +; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 20, [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK1]] ], [ 0, %[[ENTRY]] ] ; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] ; CHECK: [[LOOP_HEADER]]: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] diff --git a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll index 6fcbe461084bfb..6f167b3faabdf6 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll @@ -12,7 +12,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d ; CHECK: [[LOOP_HEADER]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ] ; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[GEP_SRC]], align 2 ; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) @@ -54,7 +54,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]] ; CHECK: [[PRED_STORE_CONTINUE4]]: ; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], i64 3 -; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]] +; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]] ; CHECK: [[PRED_STORE_IF5]]: ; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[IV]], 3 ; CHECK-NEXT: [[TMP23:%.*]] = shl i64 [[TMP22]], 1 @@ -68,14 +68,15 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d ; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 -; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]] -; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP27]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) -; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]] -; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP30]], 20 -; CHECK-NEXT: br i1 [[TMP31]], [[EXIT:label %.*]], label %[[SCALAR_PH1:.*]] +; CHECK-NEXT: br i1 [[TMP28]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[SCALAR_PH1]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void ; entry: br label %loop.header @@ -168,7 +169,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 ; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]] -; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) ; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]] diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll index 139c9992cc38d7..d4f5ef95e5da73 100644 --- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll +++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll @@ -1564,7 +1564,7 @@ define void @store_in_latch_after_exit(ptr align 2 dereferenceable(40) noalias % ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2 ; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) @@ -1577,29 +1577,13 @@ define void @store_in_latch_after_exit(ptr align 2 dereferenceable(40) noalias % ; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 -; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] -; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]] +; CHECK: [[VECTOR_BODY_INTERIM]]: +; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) -; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20 -; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] -; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] -; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2 -; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500 -; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_LATCH]] -; CHECK: [[LOOP_LATCH]]: -; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]] -; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1 -; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 -; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20 -; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]] +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -1629,32 +1613,49 @@ exit: define void @conditional_store_after_two_exits(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) { ; CHECK-LABEL: define void @conditional_store_after_two_exits( ; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED2:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] -; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] -; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2 -; CHECK-NEXT: [[EE2_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[IV]] -; CHECK-NEXT: [[EE2_VAL:%.*]] = load i16, ptr [[EE2_ADDR]], align 2 -; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500 -; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LOOP_2:.*]] -; CHECK: [[LOOP_2]]: -; CHECK-NEXT: [[EE2_COND:%.*]] = icmp eq i16 [[EE2_VAL]], 0 -; CHECK-NEXT: br i1 [[EE2_COND]], label %[[EXIT_2:.*]], label %[[LOOP_3:.*]] -; CHECK: [[LOOP_3]]: -; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]] -; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: [[ST_COND:%.*]] = icmp slt i16 [[DATA]], 0 -; CHECK-NEXT: br i1 [[ST_COND]], label %[[STORE:.*]], label %[[LOOP_LATCH]] -; CHECK: [[STORE]]: -; CHECK-NEXT: [[INC:%.*]] = add i16 [[DATA]], 1 -; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: br label %[[LOOP_LATCH]] -; CHECK: [[LOOP_LATCH]]: -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 -; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20 -; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]] +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2 +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2 +; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD1]], zeroinitializer +; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP7]]) +; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP10]], <4 x i1> [[TMP9]], <4 x i16> poison) +; CHECK-NEXT: [[TMP11:%.*]] = icmp slt <4 x i16> [[WIDE_MASKED_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP11]] +; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP14:%.*]] = add <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1) +; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP14]], ptr align 2 [[TMP10]], <4 x i1> [[TMP13]]) +; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP15]] +; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 +; CHECK-NEXT: br i1 [[TMP17]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]] +; CHECK: [[VECTOR_BODY_INTERIM]]: +; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]: +; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP16]], i1 false) +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]] +; CHECK-NEXT: br i1 [[TMP19]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]] +; CHECK: [[VECTOR_EARLY_EXIT_1]]: +; CHECK-NEXT: br label %[[EXIT_2:.*]] +; CHECK: [[VECTOR_EARLY_EXIT_0]]: +; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; CHECK: [[EXIT_2]]: @@ -1707,7 +1708,7 @@ define i16 @store_after_exit_live_out(ptr align 2 dereferenceable(40) noalias %a ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2 ; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) @@ -1720,34 +1721,16 @@ define i16 @store_after_exit_live_out(ptr align 2 dereferenceable(40) noalias %a ; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 -; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] -; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]] +; CHECK: [[VECTOR_BODY_INTERIM]]: +; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] -; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20 -; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP11]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] -; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] -; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2 -; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500 -; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_STORE:.*]] -; CHECK: [[LOOP_STORE]]: -; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]] -; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1 -; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: br label %[[LOOP_LATCH]] -; CHECK: [[LOOP_LATCH]]: -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 -; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20 -; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[EXIT]]: -; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[LOOP]] ], [ [[INC]], %[[LOOP_LATCH]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[VECTOR_EARLY_EXIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret i16 [[RES]] ; entry: diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index 6eed6c30538198..1f4aeaeed0af0d 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -596,8 +596,9 @@ TEST_F(VPIRVerifierTest, BranchOnTwoCondsLatchHeaderVerification) { BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); // Build a plain CFG VPlan with BranchOnTwoConds as the latch terminator // (3 successors), without wrapping blocks in loop regions. - auto Plan = buildVPlan(LoopHeader, UncountableExitStyle::ReadOnly, - /*CreateLoopRegions=*/false); + auto Plan = + buildVPlan(LoopHeader, UncountableExitStyle::MaskedHandleExitInVectorLoop, + /*CreateLoopRegions=*/false); auto *MiddleVPBB = cast<VPBasicBlock>(Plan->getScalarPreheader()->getPredecessors()[0]); _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
