https://github.com/ppenzin created https://github.com/llvm/llvm-project/pull/170611
Use register allocator to save callee-saved registers. This has been split out from https://github.com/mgudim/llvm-project/tree/save_csr_in_ra3, and is PR 5 out of 5. Co-authored-by: Mikhail Gudim <[email protected]> >From 2f93583a74dade13944af887178f83f857083496 Mon Sep 17 00:00:00 2001 From: Mikhail Gudim <[email protected]> Date: Wed, 5 Nov 2025 06:08:10 -0800 Subject: [PATCH 1/2] [CodeGen] Let RDA recompute live-ins. This is needed for the cases when RDA has to run on a function that does not have live-ins info. --- llvm/lib/CodeGen/ReachingDefAnalysis.cpp | 12 ++++++++++++ llvm/test/CodeGen/RISCV/pr53662.mir | 4 ++++ .../RISCV/rvv/fixed-vectors-emergency-slot.mir | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp index b12a5bc64ca0b..7014fd4bf890b 100644 --- a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp +++ b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp @@ -9,6 +9,8 @@ #include "llvm/CodeGen/ReachingDefAnalysis.h" #include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/TargetInstrInfo.h" @@ -288,6 +290,16 @@ void ReachingDefInfo::run(MachineFunction &mf) { TRI = STI.getRegisterInfo(); TII = STI.getInstrInfo(); LLVM_DEBUG(dbgs() << "********** REACHING DEFINITION ANALYSIS **********\n"); + + MachineFunctionProperties &Props = MF->getProperties(); + if (!Props.hasTracksLiveness()) { + Props.setTracksLiveness(); + + SmallVector<MachineBasicBlock *> AllMBBsInPostOrder; + for (MachineBasicBlock *MBB : post_order(MF)) + AllMBBsInPostOrder.push_back(MBB); + fullyRecomputeLiveIns(AllMBBsInPostOrder); + } init(); traverse(); } diff --git a/llvm/test/CodeGen/RISCV/pr53662.mir b/llvm/test/CodeGen/RISCV/pr53662.mir index dccad40368111..834bcbc1cf82c 100644 --- a/llvm/test/CodeGen/RISCV/pr53662.mir +++ b/llvm/test/CodeGen/RISCV/pr53662.mir @@ -18,15 +18,19 @@ body: | ; CHECK-LABEL: name: b ; CHECK: bb.0: ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: liveins: $x10 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: PseudoBR %bb.1 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: liveins: $x10 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: DBG_VALUE $noreg ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $x10 + ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: PseudoRET implicit killed $x10 bb.0 : PseudoBR %bb.1 diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-emergency-slot.mir b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-emergency-slot.mir index c728fcb8d8b0d..44f60a43a2790 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-emergency-slot.mir +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-emergency-slot.mir @@ -47,7 +47,7 @@ body: | SD $x10, %stack.0, 0 SD $x10, %stack.2, 0 - dead renamable $x15 = PseudoVSETIVLI 1, 72, implicit-def $vl, implicit-def $vtype + renamable $x15 = PseudoVSETIVLI 1, 72, implicit-def $vl, implicit-def $vtype VS1R_V killed renamable $v25, %stack.1 :: (store (<vscale x 1 x s64>) into %stack.1, align 8) ; This is here just to make all the eligible registers live at this point. ; This way when we replace the frame index %stack.1 with its actual address >From 5fd960191964a3b46311ccc8550b991b41aabcf5 Mon Sep 17 00:00:00 2001 From: Mikhail Gudim <[email protected]> Date: Fri, 2 May 2025 09:16:49 -0700 Subject: [PATCH 2/2] Make the RA to save CSRs. In `finalizeLowering` we copy all callee-saved registers from a physical register to a virtual one. In all return blocks we copy do the reverse. This has two effects: (1) It tells the optimizer that the value of callee-saved registers has to be preserved (2) Allows the optimizer to work on CSRs. In particular, we get shrink-wrapping "for free" - the register allocator will spill / restore CSRs if needed. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 92 ++++++++++++++++ llvm/lib/Target/RISCV/RISCVISelLowering.h | 2 + llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 8 ++ llvm/lib/Target/RISCV/RISCVInstrInfo.h | 2 + llvm/lib/Target/RISCV/RISCVInstrInfo.td | 3 + llvm/test/CodeGen/RISCV/save-csr-early.ll | 113 ++++++++++++++++++++ 6 files changed, 220 insertions(+) create mode 100644 llvm/test/CodeGen/RISCV/save-csr-early.ll diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index ab2652eac3823..7e7af8d48da0b 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -25827,3 +25827,95 @@ bool RISCVTargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0, return true; } + +static MachineInstr *findInstrWhichNeedAllCSRs(MachineBasicBlock &MBB) { + // Some instructions may require (implicitly) all CSRs to be saved. + // For example, call to __cxa_throw is noreturn, but expects that all CSRs are + // taken care of. + // TODO: try to speedup this? + for (MachineInstr &MI : MBB) { + unsigned Opc = MI.getOpcode(); + if (Opc != RISCV::PseudoCALL && Opc != RISCV::PseudoTAIL) + continue; + MachineOperand &MO = MI.getOperand(0); + StringRef Name = ""; + if (MO.isSymbol()) { + Name = MO.getSymbolName(); + } else if (MO.isGlobal()) { + Name = MO.getGlobal()->getName(); + } else { + llvm_unreachable("Unexpected operand type."); + } + if (Name == "__cxa_throw" || Name == "__cxa_rethrow" || + Name == "_Unwind_Resume") + return &MI; + } + return nullptr; +} + +void RISCVTargetLowering::finalizeLowering(MachineFunction &MF) const { + if (!Subtarget.savesCSRsEarly()) { + TargetLoweringBase::finalizeLowering(MF); + return; + } + + MachineRegisterInfo &MRI = MF.getRegInfo(); + const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); + const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo(); + const RISCVFrameLowering &TFI = *Subtarget.getFrameLowering(); + + SmallVector<MachineInstr *, 4> RestorePoints; + SmallVector<MachineBasicBlock *, 4> SaveMBBs; + SaveMBBs.push_back(&MF.front()); + for (MachineBasicBlock &MBB : MF) { + if (MBB.isReturnBlock()) + RestorePoints.push_back(&MBB.back()); + if (MachineInstr *CallToCxaThrow = findInstrWhichNeedAllCSRs(MBB)) { + MachineBasicBlock::iterator MII = MBB.getFirstTerminator(); + MachineInstr *NewRetMI = BuildMI(MBB, MII, CallToCxaThrow->getDebugLoc(), + TII.get(RISCV::UnreachableRET)); + RestorePoints.push_back(NewRetMI); + MII = ++NewRetMI->getIterator(); + MBB.erase(MII, MBB.end()); + } + } + + BitVector EarlyCSRs; + TFI.determineEarlyCalleeSaves(MF, EarlyCSRs); + + SmallVector<Register, 4> VRegs; + for (MachineBasicBlock *SaveMBB : SaveMBBs) { + for (unsigned Reg = 0; Reg < EarlyCSRs.size(); ++Reg) { + if (!EarlyCSRs[Reg]) + continue; + SaveMBB->addLiveIn(Reg); + Register VReg = MRI.createVirtualRegister( + TRI.getLargestLegalSuperClass(TRI.getMinimalPhysRegClass(Reg), MF)); + VRegs.push_back(VReg); + BuildMI(*SaveMBB, SaveMBB->begin(), + SaveMBB->findDebugLoc(SaveMBB->begin()), + TII.get(TargetOpcode::COPY), VReg) + .addReg(Reg); + MRI.setSimpleHint(VReg, Reg); + } + } + + for (MachineInstr *RestorePoint : RestorePoints) { + auto VRegI = VRegs.begin(); + for (unsigned Reg = 0; Reg < EarlyCSRs.size(); ++Reg) { + if (!EarlyCSRs[Reg]) + continue; + Register VReg = *VRegI; + BuildMI(*RestorePoint->getParent(), RestorePoint->getIterator(), + RestorePoint->getDebugLoc(), TII.get(TargetOpcode::COPY), Reg) + .addReg(VReg); + RestorePoint->addOperand(MF, + MachineOperand::CreateReg(Reg, + /*isDef=*/false, + /*isImplicit=*/true)); + VRegI++; + } + } + + TargetLoweringBase::finalizeLowering(MF); +} diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index 8a55a5634452c..bd21bb9e8593e 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -652,6 +652,8 @@ class RISCVTargetLowering : public TargetLowering { std::pair<const TargetRegisterClass *, uint8_t> findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override; + + void finalizeLowering(MachineFunction &MF) const override; }; namespace RISCVVIntrinsicsTable { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 89ec4a2a4a3e1..5bbf558644987 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -5127,3 +5127,11 @@ bool RISCVInstrInfo::isHighLatencyDef(int Opc) const { return true; } } + +bool RISCVInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { + if (MI.getOpcode() == RISCV::UnreachableRET) { + MI.eraseFromParent(); + return true; + } + return false; +} diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h index 0ffe015b9fac8..e98ec29d57446 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h @@ -343,6 +343,8 @@ class RISCVInstrInfo : public RISCVGenInstrInfo { static bool isFromLoadImm(const MachineRegisterInfo &MRI, const MachineOperand &Op, int64_t &Imm); + bool expandPostRAPseudo(MachineInstr &MI) const override; + protected: const RISCVSubtarget &STI; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td index 84b962b2a8607..3cc6cb6567cfe 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td @@ -1836,6 +1836,9 @@ let isBarrier = 1, isReturn = 1, isTerminator = 1 in def PseudoRET : Pseudo<(outs), (ins), [(riscv_ret_glue)]>, PseudoInstExpansion<(JALR X0, X1, 0)>; +let isBarrier = 1, isReturn = 1, isTerminator = 1, isMeta = 1, hasSideEffects = 1, mayLoad = 0, mayStore = 0 in +def UnreachableRET : Pseudo<(outs), (ins), []>; + // PseudoTAIL is a pseudo instruction similar to PseudoCALL and will eventually // expand to auipc and jalr while encoding. // Define AsmString to print "tail" when compile with -S flag. diff --git a/llvm/test/CodeGen/RISCV/save-csr-early.ll b/llvm/test/CodeGen/RISCV/save-csr-early.ll new file mode 100644 index 0000000000000..65feb5d867aab --- /dev/null +++ b/llvm/test/CodeGen/RISCV/save-csr-early.ll @@ -0,0 +1,113 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 +; RUN: llc %s -mtriple=riscv64 -riscv-save-csrs-early=true \ +; RUN: -stop-after=finalize-isel -o - | FileCheck %s + +define void @test0() { + ; CHECK-LABEL: name: test0 + ; CHECK: bb.0 (%ir-block.0): + ; CHECK-NEXT: liveins: $x1, $x8, $x9, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprjalrnonx7_and_gprnox31 = COPY $x27 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gprjalrnonx7_and_gprnox31 = COPY $x26 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gprjalrnonx7_and_gprnox31 = COPY $x25 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gprjalrnonx7_and_gprnox31 = COPY $x24 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sr07 = COPY $x23 + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:sr07 = COPY $x22 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:sr07 = COPY $x21 + ; CHECK-NEXT: [[COPY7:%[0-9]+]]:sr07 = COPY $x20 + ; CHECK-NEXT: [[COPY8:%[0-9]+]]:sr07 = COPY $x19 + ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sr07 = COPY $x18 + ; CHECK-NEXT: [[COPY10:%[0-9]+]]:gprc_and_sr07 = COPY $x9 + ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gprc_and_sr07 = COPY $x8 + ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gprx1 = COPY $x1 + ; CHECK-NEXT: $x1 = COPY [[COPY12]] + ; CHECK-NEXT: $x8 = COPY [[COPY11]] + ; CHECK-NEXT: $x9 = COPY [[COPY10]] + ; CHECK-NEXT: $x18 = COPY [[COPY9]] + ; CHECK-NEXT: $x19 = COPY [[COPY8]] + ; CHECK-NEXT: $x20 = COPY [[COPY7]] + ; CHECK-NEXT: $x21 = COPY [[COPY6]] + ; CHECK-NEXT: $x22 = COPY [[COPY5]] + ; CHECK-NEXT: $x23 = COPY [[COPY4]] + ; CHECK-NEXT: $x24 = COPY [[COPY3]] + ; CHECK-NEXT: $x25 = COPY [[COPY2]] + ; CHECK-NEXT: $x26 = COPY [[COPY1]] + ; CHECK-NEXT: $x27 = COPY [[COPY]] + ; CHECK-NEXT: PseudoRET implicit $x1, implicit $x8, implicit $x9, implicit $x18, implicit $x19, implicit $x20, implicit $x21, implicit $x22, implicit $x23, implicit $x24, implicit $x25, implicit $x26, implicit $x27 + ret void +} + +declare void @__cxa_throw(ptr, ptr, ptr) + +define void @test1(i1 %x, ptr %p0, ptr %p1, ptr %p2) { + ; CHECK-LABEL: name: test1 + ; CHECK: bb.0.entry: + ; CHECK-NEXT: successors: %bb.1(0x00000000), %bb.2(0x80000000) + ; CHECK-NEXT: liveins: $x10, $x11, $x12, $x13, $x1, $x8, $x9, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprjalrnonx7_and_gprnox31 = COPY $x27 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gprjalrnonx7_and_gprnox31 = COPY $x26 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gprjalrnonx7_and_gprnox31 = COPY $x25 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gprjalrnonx7_and_gprnox31 = COPY $x24 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sr07 = COPY $x23 + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:sr07 = COPY $x22 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:sr07 = COPY $x21 + ; CHECK-NEXT: [[COPY7:%[0-9]+]]:sr07 = COPY $x20 + ; CHECK-NEXT: [[COPY8:%[0-9]+]]:sr07 = COPY $x19 + ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sr07 = COPY $x18 + ; CHECK-NEXT: [[COPY10:%[0-9]+]]:gprc_and_sr07 = COPY $x9 + ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gprc_and_sr07 = COPY $x8 + ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gprx1 = COPY $x1 + ; CHECK-NEXT: [[COPY13:%[0-9]+]]:gpr = COPY $x13 + ; CHECK-NEXT: [[COPY14:%[0-9]+]]:gpr = COPY $x12 + ; CHECK-NEXT: [[COPY15:%[0-9]+]]:gpr = COPY $x11 + ; CHECK-NEXT: [[COPY16:%[0-9]+]]:gpr = COPY $x10 + ; CHECK-NEXT: [[ANDI:%[0-9]+]]:gpr = ANDI [[COPY16]], 1 + ; CHECK-NEXT: BEQ killed [[ANDI]], $x0, %bb.2 + ; CHECK-NEXT: PseudoBR %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.throw: + ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $x2, implicit $x2 + ; CHECK-NEXT: $x10 = COPY [[COPY15]] + ; CHECK-NEXT: $x11 = COPY [[COPY14]] + ; CHECK-NEXT: $x12 = COPY [[COPY13]] + ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) @__cxa_throw, csr_ilp32_lp64, implicit-def dead $x1, implicit $x10, implicit $x11, implicit $x12, implicit-def $x2 + ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $x2, implicit $x2 + ; CHECK-NEXT: $x1 = COPY [[COPY12]] + ; CHECK-NEXT: $x8 = COPY [[COPY11]] + ; CHECK-NEXT: $x9 = COPY [[COPY10]] + ; CHECK-NEXT: $x18 = COPY [[COPY9]] + ; CHECK-NEXT: $x19 = COPY [[COPY8]] + ; CHECK-NEXT: $x20 = COPY [[COPY7]] + ; CHECK-NEXT: $x21 = COPY [[COPY6]] + ; CHECK-NEXT: $x22 = COPY [[COPY5]] + ; CHECK-NEXT: $x23 = COPY [[COPY4]] + ; CHECK-NEXT: $x24 = COPY [[COPY3]] + ; CHECK-NEXT: $x25 = COPY [[COPY2]] + ; CHECK-NEXT: $x26 = COPY [[COPY1]] + ; CHECK-NEXT: $x27 = COPY [[COPY]] + ; CHECK-NEXT: UnreachableRET implicit $x1, implicit $x8, implicit $x9, implicit $x18, implicit $x19, implicit $x20, implicit $x21, implicit $x22, implicit $x23, implicit $x24, implicit $x25, implicit $x26, implicit $x27 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.return: + ; CHECK-NEXT: $x1 = COPY [[COPY12]] + ; CHECK-NEXT: $x8 = COPY [[COPY11]] + ; CHECK-NEXT: $x9 = COPY [[COPY10]] + ; CHECK-NEXT: $x18 = COPY [[COPY9]] + ; CHECK-NEXT: $x19 = COPY [[COPY8]] + ; CHECK-NEXT: $x20 = COPY [[COPY7]] + ; CHECK-NEXT: $x21 = COPY [[COPY6]] + ; CHECK-NEXT: $x22 = COPY [[COPY5]] + ; CHECK-NEXT: $x23 = COPY [[COPY4]] + ; CHECK-NEXT: $x24 = COPY [[COPY3]] + ; CHECK-NEXT: $x25 = COPY [[COPY2]] + ; CHECK-NEXT: $x26 = COPY [[COPY1]] + ; CHECK-NEXT: $x27 = COPY [[COPY]] + ; CHECK-NEXT: PseudoRET implicit $x1, implicit $x8, implicit $x9, implicit $x18, implicit $x19, implicit $x20, implicit $x21, implicit $x22, implicit $x23, implicit $x24, implicit $x25, implicit $x26, implicit $x27 +entry: + br i1 %x, label %throw, label %return +throw: + call void @__cxa_throw(ptr %p0, ptr %p1, ptr %p2) + unreachable +return: + ret void +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
