llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-llvm-transforms Author: Jeremy Morse (jmorse) <details> <summary>Changes</summary> See #<!-- -->230450 , this commit contains three commits: * Florians pre-landing of a test for a vectorized-alloca issue, * Florians fix for it, * A test-update commit that refreshes two tests which have different behaviour as a result of this change. The two commits from Florian are clean cherry-picks from the main branch, which IMHO don't warrant too much further review. However I'm completely unfamiliar with the test-changes and can't really vouch for them -- they definitely need review. It'd be nice to have this fixed in llvm23 given how it's a (rare) miscompile. --- Patch is 74.24 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/236602.diff 5 Files Affected: - (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+3-2) - (modified) llvm/lib/Transforms/Vectorize/VPlanUtils.cpp (+3-2) - (modified) llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll (+288-282) - (added) llvm/test/Transforms/LoopVectorize/alloca-in-loop.ll (+224) - (modified) llvm/test/Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll (+6-3) ``````````diff diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 48ba81dacefc7b..e94c80e98b273c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2537,9 +2537,10 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> { // The issue with (Insert|Extract)Value is that the index of the // insert/extract is not a proper operand in LLVM IR, and hence also not in - // VPlan. + // VPlan. Allocas must not be merged, as each creates a distinct allocation. if (!C || (!C->first && (C->second == Instruction::InsertValue || - C->second == Instruction::ExtractValue))) + C->second == Instruction::ExtractValue || + C->second == Instruction::Alloca))) return false; // During CSE, we can only handle non-memory recipes, as memory can alias. diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp index 7b4ffce87434c2..05bc010f6784cc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp @@ -454,8 +454,9 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) { .Case([](const VPReplicateRecipe *R) { // Be conservative about side-effects, except for the // known-side-effecting assumes and stores, which we know will be - // uniform. - return R->isSingleScalar() && + // uniform. Each alloca creates a distinct allocation, so allocas are + // never uniform. + return R->isSingleScalar() && R->getOpcode() != Instruction::Alloca && (!R->mayHaveSideEffects() || isa<AssumeInst, StoreInst>(R->getUnderlyingInstr())) && all_of(R->operands(), isUniformAcrossVFsAndUFs); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll index 07fc18e46c4fe4..3c8d118f539752 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll @@ -57,16 +57,16 @@ define void @test_iv_cost(ptr %ptr.start, i8 %a, i64 %b) { ; COST1: [[VECTOR_PH]]: ; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[START]], 32 ; COST1-NEXT: [[N_VEC:%.*]] = sub i64 [[START]], [[N_MOD_VF]] -; COST1-NEXT: [[IND_END:%.*]] = sub i64 [[START]], [[N_VEC]] +; COST1-NEXT: [[TMP6:%.*]] = sub i64 [[START]], [[N_VEC]] ; COST1-NEXT: [[IND_END9:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC]] ; COST1-NEXT: br label %[[VECTOR_BODY:.*]] ; COST1: [[VECTOR_BODY]]: -; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; COST1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[INDEX]] -; COST1-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 16 -; COST1-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1 +; COST1-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; COST1-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[TMP5]] +; COST1-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 16 +; COST1-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP2]], align 1 ; COST1-NEXT: store <16 x i8> zeroinitializer, ptr [[TMP0]], align 1 -; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32 +; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 32 ; COST1-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; COST1-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; COST1: [[MIDDLE_BLOCK]]: @@ -110,7 +110,7 @@ define void @test_iv_cost(ptr %ptr.start, i8 %a, i64 %b) { ; COST10: [[VECTOR_PH]]: ; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[START]], 16 ; COST10-NEXT: [[N_VEC:%.*]] = sub i64 [[START]], [[N_MOD_VF]] -; COST10-NEXT: [[IND_END:%.*]] = sub i64 [[START]], [[N_VEC]] +; COST10-NEXT: [[TMP4:%.*]] = sub i64 [[START]], [[N_VEC]] ; COST10-NEXT: [[IND_END9:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC]] ; COST10-NEXT: br label %[[VECTOR_BODY:.*]] ; COST10: [[VECTOR_BODY]]: @@ -185,73 +185,73 @@ define void @test_exit_branch_cost(ptr %dst, ptr noalias %x.ptr, ptr noalias %y. ; COMMON-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT2]], <2 x i1> poison, <2 x i32> zeroinitializer ; COMMON-NEXT: [[TMP0:%.*]] = select i1 [[C_4]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> zeroinitializer ; COMMON-NEXT: [[TMP1:%.*]] = xor <2 x i1> [[TMP0]], splat (i1 true) -; COMMON-NEXT: [[TMP3:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT3]], splat (i1 true) +; COMMON-NEXT: [[TMP2:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT3]], splat (i1 true) ; COMMON-NEXT: br label %[[VECTOR_BODY:.*]] ; COMMON: [[VECTOR_BODY]]: -; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE17:.*]] ] +; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE33:.*]] ] ; COMMON-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[X_PTR]], i64 [[INDEX]] ; COMMON-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8 -; COMMON-NEXT: [[TMP6:%.*]] = icmp eq <2 x i64> [[WIDE_LOAD]], zeroinitializer -; COMMON-NEXT: [[TMP7:%.*]] = xor <2 x i1> [[TMP6]], splat (i1 true) -; COMMON-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0 -; COMMON-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] -; COMMON: [[PRED_STORE_IF]]: -; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8 -; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE]] -; COMMON: [[PRED_STORE_CONTINUE]]: -; COMMON-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP7]], i64 1 -; COMMON-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]] -; COMMON: [[PRED_STORE_IF4]]: -; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8 -; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE5]] -; COMMON: [[PRED_STORE_CONTINUE5]]: -; COMMON-NEXT: [[TMP10:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[TMP1]], <2 x i1> zeroinitializer -; COMMON-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP10]], i64 0 +; COMMON-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[WIDE_LOAD]], zeroinitializer +; COMMON-NEXT: [[TMP6:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true) +; COMMON-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0 ; COMMON-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]] ; COMMON: [[PRED_STORE_IF6]]: -; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8 +; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8 ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE7]] ; COMMON: [[PRED_STORE_CONTINUE7]]: -; COMMON-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1 -; COMMON-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]] -; COMMON: [[PRED_STORE_IF8]]: -; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8 -; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE9]] -; COMMON: [[PRED_STORE_CONTINUE9]]: -; COMMON-NEXT: [[TMP22:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> zeroinitializer -; COMMON-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP22]], <2 x i1> [[BROADCAST_SPLAT3]], <2 x i1> zeroinitializer -; COMMON-NEXT: [[TMP14:%.*]] = or <2 x i1> [[TMP6]], [[TMP13]] -; COMMON-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP13]], <2 x i64> zeroinitializer, <2 x i64> splat (i64 1) -; COMMON-NEXT: [[TMP15:%.*]] = extractelement <2 x i1> [[TMP14]], i64 0 +; COMMON-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP6]], i64 1 +; COMMON-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]] +; COMMON: [[PRED_STORE_IF7]]: +; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8 +; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE8]] +; COMMON: [[PRED_STORE_CONTINUE8]]: +; COMMON-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP6]], <2 x i1> [[TMP1]], <2 x i1> zeroinitializer +; COMMON-NEXT: [[TMP15:%.*]] = extractelement <2 x i1> [[TMP8]], i64 0 ; COMMON-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]] ; COMMON: [[PRED_STORE_IF10]]: -; COMMON-NEXT: [[TMP16:%.*]] = extractelement <2 x i64> [[PREDPHI]], i64 0 -; COMMON-NEXT: store i64 [[TMP16]], ptr [[DST_2]], align 8 +; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8 ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE11]] ; COMMON: [[PRED_STORE_CONTINUE11]]: -; COMMON-NEXT: [[TMP17:%.*]] = extractelement <2 x i1> [[TMP14]], i64 1 -; COMMON-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]] -; COMMON: [[PRED_STORE_IF12]]: -; COMMON-NEXT: [[TMP18:%.*]] = extractelement <2 x i64> [[PREDPHI]], i64 1 -; COMMON-NEXT: store i64 [[TMP18]], ptr [[DST_2]], align 8 -; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE13]] -; COMMON: [[PRED_STORE_CONTINUE13]]: -; COMMON-NEXT: [[TMP19:%.*]] = select <2 x i1> [[TMP22]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer -; COMMON-NEXT: [[TMP20:%.*]] = or <2 x i1> [[TMP14]], [[TMP19]] -; COMMON-NEXT: [[TMP21:%.*]] = extractelement <2 x i1> [[TMP20]], i64 0 +; COMMON-NEXT: [[TMP27:%.*]] = extractelement <2 x i1> [[TMP8]], i64 1 +; COMMON-NEXT: br i1 [[TMP27]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]] +; COMMON: [[PRED_STORE_IF15]]: +; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8 +; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE16]] +; COMMON: [[PRED_STORE_CONTINUE16]]: +; COMMON-NEXT: [[TMP14:%.*]] = select <2 x i1> [[TMP6]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> zeroinitializer +; COMMON-NEXT: [[TMP12:%.*]] = select <2 x i1> [[TMP14]], <2 x i1> [[BROADCAST_SPLAT3]], <2 x i1> zeroinitializer +; COMMON-NEXT: [[TMP13:%.*]] = or <2 x i1> [[TMP4]], [[TMP12]] +; COMMON-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP12]], <2 x i64> zeroinitializer, <2 x i64> splat (i64 1) +; COMMON-NEXT: [[TMP21:%.*]] = extractelement <2 x i1> [[TMP13]], i64 0 ; COMMON-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]] ; COMMON: [[PRED_STORE_IF14]]: -; COMMON-NEXT: [[TMP24:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META6:![0-9]+]] -; COMMON-NEXT: store i64 [[TMP24]], ptr [[DST]], align 8, !alias.scope [[META9:![0-9]+]], !noalias [[META6]] +; COMMON-NEXT: [[TMP28:%.*]] = extractelement <2 x i64> [[PREDPHI]], i64 0 +; COMMON-NEXT: store i64 [[TMP28]], ptr [[DST_2]], align 8 ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE15]] ; COMMON: [[PRED_STORE_CONTINUE15]]: -; COMMON-NEXT: [[TMP23:%.*]] = extractelement <2 x i1> [[TMP20]], i64 1 -; COMMON-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17]] +; COMMON-NEXT: [[TMP23:%.*]] = extractelement <2 x i1> [[TMP13]], i64 1 +; COMMON-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17:.*]] ; COMMON: [[PRED_STORE_IF16]]: -; COMMON-NEXT: [[TMP26:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META6]] -; COMMON-NEXT: store i64 [[TMP26]], ptr [[DST]], align 8, !alias.scope [[META9]], !noalias [[META6]] +; COMMON-NEXT: [[TMP30:%.*]] = extractelement <2 x i64> [[PREDPHI]], i64 1 +; COMMON-NEXT: store i64 [[TMP30]], ptr [[DST_2]], align 8 ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE17]] ; COMMON: [[PRED_STORE_CONTINUE17]]: +; COMMON-NEXT: [[TMP18:%.*]] = select <2 x i1> [[TMP14]], <2 x i1> [[TMP2]], <2 x i1> zeroinitializer +; COMMON-NEXT: [[TMP19:%.*]] = or <2 x i1> [[TMP13]], [[TMP18]] +; COMMON-NEXT: [[TMP40:%.*]] = extractelement <2 x i1> [[TMP19]], i64 0 +; COMMON-NEXT: br i1 [[TMP40]], label %[[PRED_STORE_IF30:.*]], label %[[PRED_STORE_CONTINUE31:.*]] +; COMMON: [[PRED_STORE_IF30]]: +; COMMON-NEXT: [[TMP22:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META6:![0-9]+]] +; COMMON-NEXT: store i64 [[TMP22]], ptr [[DST]], align 8, !alias.scope [[META9:![0-9]+]], !noalias [[META6]] +; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE31]] +; COMMON: [[PRED_STORE_CONTINUE31]]: +; COMMON-NEXT: [[TMP43:%.*]] = extractelement <2 x i1> [[TMP19]], i64 1 +; COMMON-NEXT: br i1 [[TMP43]], label %[[PRED_STORE_IF32:.*]], label %[[PRED_STORE_CONTINUE33]] +; COMMON: [[PRED_STORE_IF32]]: +; COMMON-NEXT: [[TMP24:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META6]] +; COMMON-NEXT: store i64 [[TMP24]], ptr [[DST]], align 8, !alias.scope [[META9]], !noalias [[META6]] +; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE33]] +; COMMON: [[PRED_STORE_CONTINUE33]]: ; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; COMMON-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64 ; COMMON-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] @@ -323,15 +323,20 @@ define void @invalid_legacy_cost(i64 %N, ptr %x) #0 { ; COST1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] ; COST1-NEXT: br label %[[VECTOR_BODY:.*]] ; COST1: [[VECTOR_BODY]]: -; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; COST1-NEXT: [[TMP8:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; COST1-NEXT: [[TMP1:%.*]] = alloca i8, i64 0, align 16 -; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i64 0 -; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer -; COST1-NEXT: [[TMP2:%.*]] = getelementptr ptr, ptr [[X]], i64 [[INDEX]] -; COST1-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr [[TMP2]], i64 2 -; COST1-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8 -; COST1-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 8 -; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; COST1-NEXT: [[TMP10:%.*]] = alloca i8, i64 0, align 16 +; COST1-NEXT: [[TMP5:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i32 0 +; COST1-NEXT: [[TMP11:%.*]] = insertelement <2 x ptr> [[TMP5]], ptr [[TMP10]], i32 1 +; COST1-NEXT: [[TMP6:%.*]] = alloca i8, i64 0, align 16 +; COST1-NEXT: [[TMP7:%.*]] = alloca i8, i64 0, align 16 +; COST1-NEXT: [[TMP14:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP6]], i32 0 +; COST1-NEXT: [[TMP9:%.*]] = insertelement <2 x ptr> [[TMP14]], ptr [[TMP7]], i32 1 +; COST1-NEXT: [[TMP12:%.*]] = getelementptr ptr, ptr [[X]], i64 [[TMP8]] +; COST1-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 2 +; COST1-NEXT: store <2 x ptr> [[TMP11]], ptr [[TMP12]], align 8 +; COST1-NEXT: store <2 x ptr> [[TMP9]], ptr [[TMP3]], align 8 +; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], 4 ; COST1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; COST1-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] ; COST1: [[MIDDLE_BLOCK]]: @@ -352,10 +357,11 @@ define void @invalid_legacy_cost(i64 %N, ptr %x) #0 { ; COST10: [[VECTOR_BODY]]: ; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; COST10-NEXT: [[TMP1:%.*]] = alloca i8, i64 0, align 16 -; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i64 0 -; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer +; COST10-NEXT: [[TMP6:%.*]] = alloca i8, i64 0, align 16 +; COST10-NEXT: [[TMP4:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i32 0 +; COST10-NEXT: [[TMP5:%.*]] = insertelement <2 x ptr> [[TMP4]], ptr [[TMP6]], i32 1 ; COST10-NEXT: [[TMP2:%.*]] = getelementptr ptr, ptr [[X]], i64 [[INDEX]] -; COST10-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8 +; COST10-NEXT: store <2 x ptr> [[TMP5]], ptr [[TMP2]], align 8 ; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; COST10-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; COST10-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] @@ -383,26 +389,26 @@ for.end: define void @loop_with_freeze_and_conditional_srem(ptr %dst, ptr %keyinfo, ptr %invariant.ptr, i32 %divisor) { ; COMMON-LABEL: define void @loop_with_freeze_and_conditional_srem( ; COMMON-SAME: ptr [[DST:%.*]], ptr [[KEYINFO:%.*]], ptr [[INVARIANT_PTR:%.*]], i32 [[DIVISOR:%.*]]) { -; COMMON-NEXT: [[ENTRY:.*]]: -; COMMON-NEXT: br label %[[LOOP:.*]] -; COMMON: [[LOOP]]: -; COMMON-NEXT: [[INDEX_NEXT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; COMMON-NEXT: [[PRED_STORE_IF10:.*]]: +; COMMON-NEXT: br label %[[LOOP_LATCH:.*]] +; COMMON: [[LOOP_LATCH]]: +; COMMON-NEXT: [[INDEX_NEXT:%.*]] = phi i64 [ 0, %[[PRED_STORE_IF10]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE23:.*]] ] ; COMMON-NEXT: [[LOADED:%.*]] = load i32, ptr [[INVARIANT_PTR]], align 4 ; COMMON-NEXT: [[FROZEN:%.*]] = freeze i32 [[LOADED]] -; COMMON-NEXT: [[CMP:%.*]] = icmp eq i32 [[FROZEN]], 0 -; COMMON-NEXT: br i1 [[CMP]], label %[[IF_ZERO:.*]], label %[[IF_NONZERO:.*]] -; COMMON: [[IF_ZERO]]: +; COMMON-NEXT: [[TMP10:%.*]] = icmp eq i32 [[FROZEN]], 0 +; COMMON-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[PRED_STORE_CONTINUE13:.*]] +; COMMON: [[EXIT]]: ; COMMON-NEXT: store i32 0, ptr [[KEYINFO]], align 4 -; COMMON-NEXT: br label %[[LOOP_LATCH]] -; COMMON: [[IF_NONZERO]]: -; COMMON-NEXT: [[TMP11:%.*]] = srem i32 1, [[DIVISOR]] -; COMMON-NEXT: store i32 [[TMP11]], ptr [[DST]], align 4 -; COMMON-NEXT: br label %[[LOOP_LATCH]] -; COMMON: [[LOOP_LATCH]]: +; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE23]] +; COMMON: [[PRED_STORE_CONTINUE13]]: +; COMMON-NEXT: [[TMP14:%.*]] = srem i32 1, [[DIVISOR]] +; COMMON-NEXT: store i32 [[TMP14]], ptr [[DST]], align 4 +; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE23]] +; COMMON: [[PRED_STORE_CONTINUE23]]: ; COMMON-NEXT: [[IV_NEXT]] = add i64 [[INDEX_NEXT]], 1 -; COMMON-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 -; COMMON-NEXT: br i1 [[TMP16]], label %[[EXIT:.*]], label %[[LOOP]] -; COMMON: [[EXIT]]: +; COMMON-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 +; COMMON-NEXT: br i1 [[TMP20]], label %[[SCALAR_PH:.*]], label %[[LOOP_LATCH]] +; COMMON: [[SCALAR_PH]]: ; COMMON-NEXT: ret void ; entry: @@ -459,42 +465,42 @@ define void @interleave_group(ptr %dst) #1 { ; COST1: [[VEC_EPILOG_ITER_CHECK]]: ; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]] ; COST1: [[VEC_EPILOG_PH]]: -; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] -; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0 +; COST1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] +; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0 ; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer ; COST1-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3> ; COST1-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] ; COST1: [[VEC_EPILOG_VECTOR_BODY]]: -; COST1-NEXT: [[INDEX1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] ... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/236602 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
