https://github.com/dyung updated https://github.com/llvm/llvm-project/pull/234526
>From 5c71313dd978dfbf0427b647b16403b145ad7743 Mon Sep 17 00:00:00 2001 From: Florian Hahn <[email protected]> Date: Wed, 21 Oct 2026 15:09:47 +0100 Subject: [PATCH] [LoopUnroll] Invalidate epilogue exit phis with new predecessor. (#234202) Adding a new incoming value the to LCSSA phi in the exit block requires invalidating SCEV for the phi. Add missing forgetLcssaPhiWithNewPredecessor, like ConnectProlog already does. Fixes https://github.com/llvm/llvm-project/issues/231756 PR: https://github.com/llvm/llvm-project/pull/234202 (cherry picked from commit 1782f9b2cef910440a06240f3bcc638bef981584) --- .../Transforms/Utils/LoopUnrollRuntime.cpp | 1 + .../LoopUnroll/scev-invalidation-lcssa.ll | 107 ++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp index 4d7dee168f4311..d4251b4d45a81b 100644 --- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp @@ -306,6 +306,7 @@ static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, // For the instruction out of the loop, constant or undefined value // insert value itself. EpilogPN->addIncoming(V, EpilogLatch); + SE.forgetLcssaPhiWithNewPredecessor(L, EpilogPN); assert(EpilogPN->getBasicBlockIndex(EpilogPreHeader) >= 0 && "EpilogPN should have EpilogPreHeader incoming block"); diff --git a/llvm/test/Transforms/LoopUnroll/scev-invalidation-lcssa.ll b/llvm/test/Transforms/LoopUnroll/scev-invalidation-lcssa.ll index fd07238b16421f..4af5076e12d054 100644 --- a/llvm/test/Transforms/LoopUnroll/scev-invalidation-lcssa.ll +++ b/llvm/test/Transforms/LoopUnroll/scev-invalidation-lcssa.ll @@ -86,5 +86,112 @@ exit2: ret i32 0 } +; Make sure the SCEV for the LCSSA phi in the exit block is invalidated when +; the epilogue adds a new incoming value to it, so it is not used for loop2. +define void @epilog_lcssa_phi_used_by_later_loop(i64 %n, ptr %src) { +; CHECK-LABEL: define void @epilog_lcssa_phi_used_by_later_loop +; CHECK-SAME: (i64 [[N:%.*]], ptr [[SRC:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 +; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 7 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[N]], 7 +; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP1_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK: entry.new: +; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP0]], [[XTRAITER]] +; CHECK-NEXT: br label [[LOOP1:%.*]] +; CHECK: loop1: +; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[IV_1_NEXT_7:%.*]], [[LOOP1]] ] +; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[NITER_NEXT_7:%.*]], [[LOOP1]] ] +; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[SRC]], align 8 +; CHECK-NEXT: [[ADD_7:%.*]] = add i64 [[L]], 1 +; CHECK-NEXT: [[IV_1_NEXT_7]] = add i64 [[IV_1]], 8 +; CHECK-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 +; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] +; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[LOOP2_PH_UNR_LCSSA:%.*]], label [[LOOP1]] +; CHECK: loop2.ph.unr-lcssa: +; CHECK-NEXT: [[ADD_LCSSA_PH:%.*]] = phi i64 [ [[ADD_7]], [[LOOP1]] ] +; CHECK-NEXT: [[IV_1_UNR:%.*]] = phi i64 [ [[IV_1_NEXT_7]], [[LOOP1]] ] +; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP1_EPIL_PREHEADER]], label [[LOOP2_PH:%.*]] +; CHECK: loop1.epil.preheader: +; CHECK-NEXT: [[IV_1_EPIL_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_1_UNR]], [[LOOP2_PH_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) +; CHECK-NEXT: br label [[LOOP1_EPIL:%.*]] +; CHECK: loop1.epil: +; CHECK-NEXT: [[IV_1_EPIL:%.*]] = phi i64 [ [[IV_1_EPIL_INIT]], [[LOOP1_EPIL_PREHEADER]] ], [ [[IV_1_NEXT_EPIL:%.*]], [[LOOP1_EPIL]] ] +; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, [[LOOP1_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[LOOP1_EPIL]] ] +; CHECK-NEXT: [[L_EPIL:%.*]] = load i64, ptr [[SRC]], align 8 +; CHECK-NEXT: [[ADD_EPIL:%.*]] = add i64 [[L_EPIL]], 1 +; CHECK-NEXT: [[IV_1_NEXT_EPIL]] = add i64 [[IV_1_EPIL]], 1 +; CHECK-NEXT: [[EC_1_EPIL:%.*]] = icmp eq i64 [[IV_1_EPIL]], [[N]] +; CHECK-NEXT: [[EPIL_ITER_NEXT]] = add i64 [[EPIL_ITER]], 1 +; CHECK-NEXT: [[EPIL_ITER_CMP:%.*]] = icmp ne i64 [[EPIL_ITER_NEXT]], [[XTRAITER]] +; CHECK-NEXT: br i1 [[EPIL_ITER_CMP]], label [[LOOP1_EPIL]], label [[LOOP2_PH_EPILOG_LCSSA:%.*]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK: loop2.ph.epilog-lcssa: +; CHECK-NEXT: [[ADD_LCSSA_PH1:%.*]] = phi i64 [ [[ADD_EPIL]], [[LOOP1_EPIL]] ] +; CHECK-NEXT: br label [[LOOP2_PH]] +; CHECK: loop2.ph: +; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD_LCSSA_PH]], [[LOOP2_PH_UNR_LCSSA]] ], [ [[ADD_LCSSA_PH1]], [[LOOP2_PH_EPILOG_LCSSA]] ] +; CHECK-NEXT: [[TMP2:%.*]] = sub i64 1, [[ADD_LCSSA]] +; CHECK-NEXT: [[TMP3:%.*]] = sub i64 0, [[ADD_LCSSA]] +; CHECK-NEXT: [[XTRAITER3:%.*]] = and i64 [[TMP2]], 7 +; CHECK-NEXT: [[LCMP_MOD4:%.*]] = icmp ne i64 [[XTRAITER3]], 0 +; CHECK-NEXT: br i1 [[LCMP_MOD4]], label [[LOOP2_PROL_PREHEADER:%.*]], label [[LOOP2_PROL_LOOPEXIT:%.*]] +; CHECK: loop2.prol.preheader: +; CHECK-NEXT: br label [[LOOP2_PROL:%.*]] +; CHECK: loop2.prol: +; CHECK-NEXT: [[IV_2_PROL:%.*]] = phi i64 [ [[ADD_LCSSA]], [[LOOP2_PROL_PREHEADER]] ], [ [[IV_2_NEXT_PROL:%.*]], [[LOOP2_PROL]] ] +; CHECK-NEXT: [[PROL_ITER:%.*]] = phi i64 [ 0, [[LOOP2_PROL_PREHEADER]] ], [ [[PROL_ITER_NEXT:%.*]], [[LOOP2_PROL]] ] +; CHECK-NEXT: [[IV_2_NEXT_PROL]] = add i64 [[IV_2_PROL]], 1 +; CHECK-NEXT: [[EC_2_PROL:%.*]] = icmp eq i64 [[IV_2_PROL]], 0 +; CHECK-NEXT: [[PROL_ITER_NEXT]] = add i64 [[PROL_ITER]], 1 +; CHECK-NEXT: [[PROL_ITER_CMP:%.*]] = icmp ne i64 [[PROL_ITER_NEXT]], [[XTRAITER3]] +; CHECK-NEXT: br i1 [[PROL_ITER_CMP]], label [[LOOP2_PROL]], label [[LOOP2_PROL_LOOPEXIT_UNR_LCSSA:%.*]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK: loop2.prol.loopexit.unr-lcssa: +; CHECK-NEXT: [[IV_2_UNR_PH:%.*]] = phi i64 [ [[IV_2_NEXT_PROL]], [[LOOP2_PROL]] ] +; CHECK-NEXT: br label [[LOOP2_PROL_LOOPEXIT]] +; CHECK: loop2.prol.loopexit: +; CHECK-NEXT: [[IV_2_UNR:%.*]] = phi i64 [ [[ADD_LCSSA]], [[LOOP2_PH]] ], [ [[IV_2_UNR_PH]], [[LOOP2_PROL_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 7 +; CHECK-NEXT: br i1 [[TMP4]], label [[EXIT:%.*]], label [[LOOP2_PH_NEW:%.*]] +; CHECK: loop2.ph.new: +; CHECK-NEXT: br label [[LOOP2:%.*]] +; CHECK: loop2: +; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[IV_2_UNR]], [[LOOP2_PH_NEW]] ], [ [[IV_2_NEXT_7:%.*]], [[LOOP2]] ] +; CHECK-NEXT: [[IV_2_NEXT_6:%.*]] = add i64 [[IV_2]], 7 +; CHECK-NEXT: [[IV_2_NEXT_7]] = add i64 [[IV_2]], 8 +; CHECK-NEXT: [[EC_2_7:%.*]] = icmp eq i64 [[IV_2_NEXT_6]], 0 +; CHECK-NEXT: br i1 [[EC_2_7]], label [[EXIT_UNR_LCSSA:%.*]], label [[LOOP2]] +; CHECK: exit.unr-lcssa: +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + br label %loop1 + +loop1: + %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop1 ] + %l = load i64, ptr %src, align 8 + %add = add i64 %l, 1 + %iv.1.next = add i64 %iv.1, 1 + %ec.1 = icmp eq i64 %iv.1, %n + br i1 %ec.1, label %loop2.ph, label %loop1 + +loop2.ph: + %add.lcssa = phi i64 [ %add, %loop1 ] + br label %loop2 + +loop2: + %iv.2 = phi i64 [ %add.lcssa, %loop2.ph ], [ %iv.2.next, %loop2 ] + %iv.2.next = add i64 %iv.2, 1 + %ec.2 = icmp eq i64 %iv.2, 0 + br i1 %ec.2, label %exit, label %loop2 + +exit: + ret void +} + !0 = !{!"function_entry_count", i64 1} !1 = !{!"branch_weights", i32 123, i32 472} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
