https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/235021
>From 2236f3860feb9e9869493f888fa1c7f3f3d4ea45 Mon Sep 17 00:00:00 2001 From: Luke Lau <[email protected]> Date: Fri, 23 Oct 2026 13:54:55 +0800 Subject: [PATCH] [VPlan] Support interleaving early exit loops with side effects We couldn't do this on main previously because the exit mask `active-lane-mask(0, FirstActiveLane(Cond))` doesn't get unrolled properly across parts. Now that we use mask.beforefirst we can unroll it properly in a chain. The unrolled form is: unroll(mask.beforefirst, P) = mask.beforefirst(cond[P] | broadcast(anyof(not(unroll(mask.beforefirst, P-1))))) On AArch64 this can be combined to brkpb in another patch. --- .../Transforms/Vectorize/LoopVectorize.cpp | 9 - .../Transforms/Vectorize/VPlanLowering.cpp | 10 + llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 10 + .../AArch64/early_exit_with_stores.ll | 231 ++++++++++++++++-- .../interleave_uncountable_exits.ll | 113 ++++++++- ...ountable-and-uncountable-exits-combined.ll | 39 ++- 6 files changed, 377 insertions(+), 35 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index b609265aeed570..128adbf60f1f22 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7785,15 +7785,6 @@ bool LoopVectorizePass::processLoop(Loop *L) { IC = 1; } - // FIXME: Enable interleaving for EE-with-side-effects. - if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) { - LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n"); - IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving", - "Unable to interleave due to early exit with side effects."}; - InterleaveLoop = false; - IC = 1; - } - if (InterleaveLoop && LVL.hasConditionalInductions()) { LLVM_DEBUG( dbgs() << "LV: Not interleaving due to conditional inductions.\n"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp index ce6d6d40d7ab48..93fb105d145839 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp @@ -675,6 +675,16 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) { continue; } + VPValue *Mask; + if (Plan.hasScalarVFOnly() && + match(&R, + m_Intrinsic<Intrinsic::mask_beforefirst>(m_VPValue(Mask)))) { + R.getVPSingleValue()->replaceAllUsesWith( + Builder.createNot(Mask, R.getDebugLoc())); + R.eraseFromParent(); + continue; + } + // Lower BranchOnCount to ICmp + BranchOnCond. VPValue *IV, *TC; if (match(&R, m_BranchOnCount(m_VPValue(IV), m_VPValue(TC)))) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index 53ae651de231fa..bcfdb3e1acc1b2 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -333,6 +333,16 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) { Copy->setOperand(1, getValueForPart(Op, Part)); continue; } + if (match(&R, m_Intrinsic<Intrinsic::mask_beforefirst>(m_VPValue(Op)))) { + VPBuilder Builder(Copy); + VPValue *Chain = Builder.createNaryOp( + VPInstruction::AnyOf, + Builder.createNot(getValueForPart(R.getVPSingleValue(), Part - 1))); + if (!Plan.hasScalarVFOnly()) + Chain = Builder.createNaryOp(VPInstruction::Broadcast, Chain); + Copy->setOperand(0, Builder.createOr(getValueForPart(Op, Part), Chain)); + continue; + } if (match(&R, m_VPInstruction<VPInstruction::ExtractVectorForPart>( m_VPValue(Op), m_VPValue()))) { Copy->setOperand(0, Op); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll index 616c2a95ff1fd8..aa4c9abbf0208b 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll @@ -54,32 +54,72 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea ; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[SCALAR_PH1:.*]]: ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]] ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64() ; CHECK-NEXT: br label %[[FOR_BODY1:.*]] ; CHECK: [[FOR_BODY1]]: ; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ] +; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]] +; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2 +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2 ; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500) +; CHECK-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500) ; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP19]] ; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP25:%.*]] = xor <vscale x 4 x i1> [[TMP6]], splat (i1 true) +; CHECK-NEXT: [[TMP30:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP25]]) +; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP30]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 4 x i1> [[TMP9]], [[BROADCAST_SPLAT3]] +; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP32]]) ; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]] +; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[ST_ADDR1]], i64 [[TMP2]] ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison) +; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison) ; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1) +; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1) ; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]]) -; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]]) +; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP9]] +; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]] ; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4 +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false) +; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1 +; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]] ; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false) -; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]] +; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0 +; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP7]] +; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP7]], [[TMP21]] +; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]] +; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4 +; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0 +; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0 +; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1 +; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]] +; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]] +; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]] +; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]] +; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]] ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20 ; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: @@ -130,32 +170,72 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias ; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]] ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP14]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64() ; CHECK-NEXT: br label %[[FOR_BODY:.*]] ; CHECK: [[FOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ] +; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw half, ptr [[TMP2]], i64 [[TMP14]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x half>, ptr [[TMP2]], align 2 +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x half>, ptr [[TMP18]], align 2 ; CHECK-NEXT: [[TMP3:%.*]] = fcmp ugt <vscale x 4 x half> [[WIDE_LOAD]], splat (half 5.000000e+02) +; CHECK-NEXT: [[TMP19:%.*]] = fcmp ugt <vscale x 4 x half> [[WIDE_LOAD1]], splat (half 5.000000e+02) ; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP25:%.*]] = freeze <vscale x 4 x i1> [[TMP19]] ; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP30:%.*]] = xor <vscale x 4 x i1> [[TMP13]], splat (i1 true) +; CHECK-NEXT: [[TMP31:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP30]]) +; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP31]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP33:%.*]] = or <vscale x 4 x i1> [[TMP25]], [[BROADCAST_SPLAT3]] +; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP33]]) ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i16, ptr [[TMP6]], i64 [[TMP14]] ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP6]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i16> poison) +; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP32]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison) ; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1) +; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1) ; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP7]], ptr align 2 [[TMP6]], <vscale x 4 x i1> [[TMP13]]) -; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP32]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]]) +; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP25]] +; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] ; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4 +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP25]], i1 false) +; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1 +; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]] ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false) -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]] +; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0 +; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP5]] +; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP5]], [[TMP21]] +; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]] +; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4 +; CHECK-NEXT: [[TMP42:%.*]] = mul i64 [[TMP41]], 0 +; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0 +; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1 +; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]] +; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]] +; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]] +; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]] +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP39]], [[TMP29]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20 ; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: @@ -363,33 +443,74 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref ; CHECK-SAME: ptr readonly align 2 dereferenceable(40) [[LOAD_ARRAY:%.*]], ptr noalias align 2 [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[SCALAR_PH:.*]]: ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]] ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64() ; CHECK-NEXT: br label %[[FOR_BODY:.*]] ; CHECK: [[FOR_BODY]]: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP16]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ] +; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2 +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP19]], align 2 ; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500) +; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500) ; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP21:%.*]] = freeze <vscale x 4 x i1> [[TMP20]] ; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP10:%.*]] = xor <vscale x 4 x i1> [[TMP6]], splat (i1 true) +; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP10]]) +; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP27]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP34:%.*]] = or <vscale x 4 x i1> [[TMP21]], [[BROADCAST_SPLAT3]] +; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP34]]) ; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]] +; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i16, ptr [[LD_ADDR]], i64 [[TMP2]] ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison) +; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP32]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison) ; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1) +; CHECK-NEXT: [[TMP33:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1) ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]] +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i16, ptr [[TMP9]], i64 [[TMP2]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 4 x i1> [[TMP6]]) -; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP33]], ptr align 2 [[TMP17]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]]) +; CHECK-NEXT: [[TMP18:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP21]] +; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP18]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]] ; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4 +; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP21]], i1 false) +; CHECK-NEXT: [[TMP25:%.*]] = mul i64 [[TMP23]], 1 +; CHECK-NEXT: [[TMP26:%.*]] = add i64 [[TMP25]], [[TMP24]] ; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false) -; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]] +; CHECK-NEXT: [[TMP28:%.*]] = mul i64 [[TMP23]], 0 +; CHECK-NEXT: [[TMP29:%.*]] = add i64 [[TMP28]], [[TMP7]] +; CHECK-NEXT: [[TMP30:%.*]] = icmp ne i64 [[TMP7]], [[TMP23]] +; CHECK-NEXT: [[TMP31:%.*]] = select i1 [[TMP30]], i64 [[TMP29]], i64 [[TMP26]] +; CHECK-NEXT: [[TMP42:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP43:%.*]] = mul nuw i64 [[TMP42]], 4 +; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP43]], 0 +; CHECK-NEXT: [[TMP36:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0 +; CHECK-NEXT: [[TMP37:%.*]] = mul i64 [[TMP43]], 1 +; CHECK-NEXT: [[TMP38:%.*]] = sub i64 0, [[TMP37]] +; CHECK-NEXT: [[TMP39:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP38]] +; CHECK-NEXT: [[TMP40:%.*]] = icmp uge i64 0, [[TMP37]] +; CHECK-NEXT: [[TMP41:%.*]] = select i1 [[TMP40]], i64 [[TMP39]], i64 [[TMP36]] +; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP41]], [[TMP31]] ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20 ; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH1]] ; CHECK: [[SCALAR_PH1]]: @@ -487,32 +608,72 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias ; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[SCALAR_PH1:.*]]: ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]] ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64() ; CHECK-NEXT: br label %[[FOR_BODY1:.*]] ; CHECK: [[FOR_BODY1]]: ; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ] +; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]] +; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[EE_ADDR1]], i64 [[TMP2]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[EE_ADDR1]], align 2 +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2 ; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500) +; CHECK-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500) ; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP19]] ; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP25:%.*]] = xor <vscale x 4 x i1> [[TMP5]], splat (i1 true) +; CHECK-NEXT: [[TMP30:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP25]]) +; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP30]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 4 x i1> [[TMP9]], [[BROADCAST_SPLAT3]] +; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP32]]) ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]] +; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[TMP7]], i64 [[TMP2]] ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP7]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i16> poison) +; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison) ; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1) +; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1) ; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 4 x i1> [[TMP5]]) -; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]]) +; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP9]] +; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]] ; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4 +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false) +; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1 +; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]] ; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false) -; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP6]] +; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0 +; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP6]] +; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP6]], [[TMP21]] +; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]] +; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4 +; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0 +; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0 +; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1 +; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]] +; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]] +; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]] +; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]] +; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]] ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20 ; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: @@ -1159,32 +1320,72 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no ; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[SCALAR_PH:.*]]: ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]] ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64() ; CHECK-NEXT: br label %[[FOR_BODY1:.*]] ; CHECK: [[FOR_BODY1]]: ; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ] +; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]] +; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2 +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2 ; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500) +; CHECK-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500) ; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP19]] ; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP25:%.*]] = xor <vscale x 4 x i1> [[TMP6]], splat (i1 true) +; CHECK-NEXT: [[TMP30:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP25]]) +; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP30]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 4 x i1> [[TMP9]], [[BROADCAST_SPLAT3]] +; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP32]]) ; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]] +; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[ST_ADDR1]], i64 [[TMP2]] ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison) +; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison) ; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1) +; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1) ; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]]) -; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]]) +; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP9]] +; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]] ; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4 +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false) +; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1 +; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]] ; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false) -; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]] +; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0 +; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP7]] +; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP7]], [[TMP21]] +; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]] +; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4 +; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0 +; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0 +; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1 +; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]] +; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]] +; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]] +; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]] +; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]] ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20 ; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH1]] ; CHECK: [[SCALAR_PH1]]: diff --git a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll index d3dae6a8c9651f..f81ad5ee2f6d3d 100644 --- a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll @@ -1,8 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6 ; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -force-target-max-vector-interleave=2 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects | FileCheck %s +; RUN: opt -S < %s -p loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -force-target-max-vector-interleave=2 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects | FileCheck %s --check-prefix=VF1IC2 -; We do not support interleaving loops with early exits yet, so make sure we don't -; even with a flag to force it. define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) { ; CHECK-LABEL: define void @loop_contains_store_condition_load_has_single_user( ; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) { @@ -12,23 +11,49 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4) ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP1]], i64 4 ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2 +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP13]], align 2 ; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) +; CHECK-NEXT: [[TMP14:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD1]], splat (i16 500) ; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP14]] ; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP21:%.*]] = xor <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], splat (i1 true) +; CHECK-NEXT: [[TMP22:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP21]]) +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP22]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP15]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP24]]) ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i16, ptr [[TMP0]], i64 4 ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison) +; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP23]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <4 x i16> poison) ; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1) +; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1) ; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 +; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP11]], ptr align 2 [[TMP23]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK2]]) +; CHECK-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP5]], [[TMP15]] +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 4) ; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] ; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false) -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]] +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP15]], i1 false) +; CHECK-NEXT: [[TMP17:%.*]] = add i64 4, [[TMP16]] +; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false) +; CHECK-NEXT: [[TMP19:%.*]] = add i64 0, [[TMP18]] +; CHECK-NEXT: [[TMP20:%.*]] = icmp ne i64 [[TMP18]], 4 +; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP20]], i64 [[TMP19]], i64 [[TMP17]] +; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 0 +; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[STEP_ADD]], i64 -4 +; CHECK-NEXT: [[TMP25:%.*]] = select i1 false, i64 [[TMP27]], i64 [[TMP26]] +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP25]], [[TMP3]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20 ; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] ; CHECK: [[SCALAR_PH]]: @@ -50,6 +75,80 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; +; VF1IC2-LABEL: define void @loop_contains_store_condition_load_has_single_user( +; VF1IC2-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) { +; VF1IC2-NEXT: [[ENTRY:.*:]] +; VF1IC2-NEXT: br label %[[VECTOR_PH:.*]] +; VF1IC2: [[VECTOR_PH]]: +; VF1IC2-NEXT: br label %[[VECTOR_BODY:.*]] +; VF1IC2: [[VECTOR_BODY]]: +; VF1IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ] +; VF1IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 +; VF1IC2-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 +; VF1IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP0]] +; VF1IC2-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP1]] +; VF1IC2-NEXT: [[TMP4:%.*]] = load i16, ptr [[TMP2]], align 2 +; VF1IC2-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP3]], align 2 +; VF1IC2-NEXT: [[TMP6:%.*]] = icmp sgt i16 [[TMP4]], 500 +; VF1IC2-NEXT: [[TMP7:%.*]] = icmp sgt i16 [[TMP5]], 500 +; VF1IC2-NEXT: [[TMP8:%.*]] = freeze i1 [[TMP6]] +; VF1IC2-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP7]] +; VF1IC2-NEXT: [[TMP10:%.*]] = xor i1 [[TMP8]], true +; VF1IC2-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP8]] +; VF1IC2-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true +; VF1IC2-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] +; VF1IC2: [[PRED_STORE_IF]]: +; VF1IC2-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]] +; VF1IC2-NEXT: [[TMP14:%.*]] = load i16, ptr [[TMP13]], align 2 +; VF1IC2-NEXT: [[TMP15:%.*]] = add nsw i16 [[TMP14]], 1 +; VF1IC2-NEXT: store i16 [[TMP15]], ptr [[TMP13]], align 2 +; VF1IC2-NEXT: br label %[[PRED_STORE_CONTINUE]] +; VF1IC2: [[PRED_STORE_CONTINUE]]: +; VF1IC2-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]] +; VF1IC2: [[PRED_STORE_IF1]]: +; VF1IC2-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP1]] +; VF1IC2-NEXT: [[TMP17:%.*]] = load i16, ptr [[TMP16]], align 2 +; VF1IC2-NEXT: [[TMP18:%.*]] = add nsw i16 [[TMP17]], 1 +; VF1IC2-NEXT: store i16 [[TMP18]], ptr [[TMP16]], align 2 +; VF1IC2-NEXT: br label %[[PRED_STORE_CONTINUE2]] +; VF1IC2: [[PRED_STORE_CONTINUE2]]: +; VF1IC2-NEXT: [[TMP19:%.*]] = or i1 [[TMP8]], [[TMP9]] +; VF1IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; VF1IC2-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 +; VF1IC2-NEXT: [[TMP21:%.*]] = or i1 [[TMP19]], [[TMP20]] +; VF1IC2-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; VF1IC2: [[MIDDLE_BLOCK]]: +; VF1IC2-NEXT: [[TMP22:%.*]] = icmp eq i1 [[TMP9]], false +; VF1IC2-NEXT: [[TMP23:%.*]] = zext i1 [[TMP22]] to i64 +; VF1IC2-NEXT: [[TMP24:%.*]] = add i64 1, [[TMP23]] +; VF1IC2-NEXT: [[TMP25:%.*]] = icmp eq i1 [[TMP8]], false +; VF1IC2-NEXT: [[TMP26:%.*]] = zext i1 [[TMP25]] to i64 +; VF1IC2-NEXT: [[TMP27:%.*]] = add i64 0, [[TMP26]] +; VF1IC2-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP26]], 1 +; VF1IC2-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]] +; VF1IC2-NEXT: [[TMP30:%.*]] = select i1 false, i64 [[TMP1]], i64 [[TMP0]] +; VF1IC2-NEXT: [[TMP31:%.*]] = add i64 [[TMP30]], [[TMP29]] +; VF1IC2-NEXT: [[TMP32:%.*]] = icmp eq i64 [[TMP31]], 20 +; VF1IC2-NEXT: br i1 [[TMP32]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] +; VF1IC2: [[SCALAR_PH]]: +; VF1IC2-NEXT: br label %[[FOR_BODY:.*]] +; VF1IC2: [[FOR_BODY]]: +; VF1IC2-NEXT: [[IV:%.*]] = phi i64 [ [[TMP31]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ] +; VF1IC2-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]] +; VF1IC2-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2 +; VF1IC2-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1 +; VF1IC2-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2 +; VF1IC2-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] +; VF1IC2-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2 +; VF1IC2-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500 +; VF1IC2-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]] +; VF1IC2: [[FOR_INC]]: +; VF1IC2-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; VF1IC2-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20 +; VF1IC2-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; VF1IC2: [[EXIT]]: +; VF1IC2-NEXT: ret void +; entry: br label %for.body diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll index 9d5804468f12b7..af05a83af92318 100644 --- a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll +++ b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll @@ -27,33 +27,64 @@ define void @foo() #0 { ; CHECK-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP0]], 2500 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP0]], 1250 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP0]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nuw nsw i64 10000, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP30:%.*]] = tail call <vscale x 4 x i64> @llvm.stepvector.nxv4i64() +; CHECK-NEXT: [[DOTSCALAR:%.*]] = shl nuw nsw i64 [[TMP0]], 3 +; CHECK-NEXT: [[TMP31:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[DOTSCALAR]], i64 0 +; CHECK-NEXT: [[REASS_ADD:%.*]] = shufflevector <vscale x 4 x i64> [[TMP31]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: br label %[[FOR_BODY:.*]] ; CHECK: [[FOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP30]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ] ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @c, i64 [[INDEX]] +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP2]], i64 [[TMP13]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP17]], align 4 ; CHECK-NEXT: [[WIDE_LOAD_FR:%.*]] = freeze <vscale x 4 x i32> [[WIDE_LOAD]] ; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD_FR]], zeroinitializer +; CHECK-NEXT: [[WIDE_LOAD3_FR:%.*]] = freeze <vscale x 4 x i32> [[WIDE_LOAD3]] +; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD3_FR]], zeroinitializer ; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = tail call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP3]]) +; CHECK-NEXT: [[TMP19:%.*]] = xor <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]], splat (i1 true) +; CHECK-NEXT: [[TMP24:%.*]] = tail call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP19]]) +; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP24]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP28:%.*]] = or <vscale x 4 x i1> [[TMP18]], [[BROADCAST_SPLAT5]] +; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK4:%.*]] = tail call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP28]]) ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i8], ptr @src, i64 [[INDEX]] +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr [4 x i8], ptr [[TMP5]], i64 [[TMP13]] ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 4 x i32> poison) +; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP26]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK4]], <vscale x 4 x i32> poison) ; CHECK-NEXT: [[TMP6:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42) +; CHECK-NEXT: [[TMP27:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD5]], splat (i32 42) ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [4 x i8], ptr @dst, i64 [[INDEX]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr [4 x i8], ptr [[TMP7]], i64 [[TMP13]] ; CHECK-NEXT: tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP6]], ptr align 4 [[TMP7]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]]) -; CHECK-NEXT: [[TMP8:%.*]] = tail call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP3]]) +; CHECK-NEXT: tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP27]], ptr align 4 [[TMP14]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK4]]) +; CHECK-NEXT: [[TMP15:%.*]] = or <vscale x 4 x i32> [[WIDE_LOAD_FR]], [[WIDE_LOAD3_FR]] +; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <vscale x 4 x i32> [[TMP15]], zeroinitializer +; CHECK-NEXT: [[TMP8:%.*]] = tail call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[REASS_ADD]] ; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] ; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP20:%.*]] = tail call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 2 +; CHECK-NEXT: [[TMP22:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP18]], i1 false) +; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP21]], [[TMP22]] ; CHECK-NEXT: [[TMP4:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP3]], i1 false) -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP4]] +; CHECK-NEXT: [[DOTNOT:%.*]] = icmp eq i64 [[TMP4]], [[TMP21]] +; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[DOTNOT]], i64 [[TMP23]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP29:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP29]], [[TMP25]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 10000 ; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[FOR_BODY_PREHEADER]] ; CHECK: [[FOR_BODY_PREHEADER]]: _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
