================
@@ -1979,106 +2139,182 @@ bool
PreRARematStage::canIncreaseOccupancyOrReduceSpill() {
*DAG.TII))
continue;
- REMAT_DEBUG(dbgs() << "Region " << I << ": remat instruction " << DefMI);
- RematInstruction &Remat =
- Rematerializations.try_emplace(&DefMI, UseMI).first->second;
-
- bool RematUseful = false;
- if (auto It = OptRegions.find(I); It != OptRegions.end()) {
- // Optimistically consider that moving the instruction out of its
- // defining region will reduce RP in the latter; this assumes that
- // maximum RP in the region is reached somewhere between the defining
- // instruction and the end of the region.
- REMAT_DEBUG(dbgs() << " Defining region is optimizable\n");
- LaneBitmask Mask = DAG.RegionLiveOuts.getLiveRegsForRegionIdx(I)[Reg];
- if (ReduceRPInRegion(It, Reg, Mask, RematUseful))
- return true;
- }
-
- for (unsigned LIRegion = 0; LIRegion != E; ++LIRegion) {
- // We are only collecting regions in which the register is a live-in
- // (and may be live-through).
- auto It = DAG.LiveIns[LIRegion].find(Reg);
- if (It == DAG.LiveIns[LIRegion].end() || It->second.none())
- continue;
- Remat.LiveInRegions.insert(LIRegion);
-
- // Account for the reduction in RP due to the rematerialization in an
- // optimizable region in which the defined register is a live-in. This
- // is exact for live-through region but optimistic in the using region,
- // where RP is actually reduced only if maximum RP is reached somewhere
- // between the beginning of the region and the rematerializable
- // instruction's use.
- if (auto It = OptRegions.find(LIRegion); It != OptRegions.end()) {
- REMAT_DEBUG(dbgs() << " Live-in in region " << LIRegion << '\n');
- if (ReduceRPInRegion(It, Reg, DAG.LiveIns[LIRegion][Reg],
- RematUseful))
- return true;
- }
- }
-
- // If the instruction is not a live-in or live-out in any optimizable
- // region then there is no point in rematerializing it.
- if (!RematUseful) {
- Rematerializations.pop_back();
- REMAT_DEBUG(dbgs() << " No impact, not rematerializing
instruction\n");
- } else {
- RematRegs.insert(Reg);
- }
+ // Add the instruction to the rematerializable list.
+ MarkedRegs.insert(Reg);
+ RematRegs.emplace_back(&DefMI, UseMI, DAG, MIRegion);
}
}
- if (TargetOcc) {
- // We were trying to increase occupancy but failed, abort the stage.
- REMAT_DEBUG(dbgs() << "Cannot increase occupancy\n");
- Rematerializations.clear();
- return false;
+ return !RematRegs.empty();
+}
+
+PreRARematStage::RematReg::RematReg(
+ MachineInstr *DefMI, MachineInstr *UseMI, GCNScheduleDAGMILive &DAG,
+ const DenseMap<MachineInstr *, unsigned> &MIRegion)
+ : DefMI(DefMI), UseMI(UseMI), LiveIn(DAG.Regions.size()),
+ LiveOut(DAG.Regions.size()), Live(DAG.Regions.size()),
+ DefRegion(MIRegion.at(DefMI)), UseRegion(MIRegion.at(UseMI)) {
+
+ // Mark regions in which the rematerializable register is live.
+ Register Reg = getReg();
+ for (unsigned I = 0, E = DAG.Regions.size(); I != E; ++I) {
+ auto LiveInIt = DAG.LiveIns[I].find(Reg);
+ if (LiveInIt != DAG.LiveIns[I].end())
+ LiveIn.set(I);
+ const auto &LiveOuts = DAG.RegionLiveOuts.getLiveRegsForRegionIdx(I);
+ if (auto LiveOutIt = LiveOuts.find(Reg); LiveOutIt != LiveOuts.end())
+ LiveOut.set(I);
+ }
+ Live |= LiveIn;
+ Live |= LiveOut;
+ Mask = DAG.RegionLiveOuts.getLiveRegsForRegionIdx(DefRegion).at(Reg);
+}
+
+bool PreRARematStage::RematReg::maybeBeneficial(
+ const BitVector &TargetRegions, ArrayRef<GCNRPTarget> RPTargets) const {
+ Register Reg = getReg();
+ for (unsigned I : TargetRegions.set_bits()) {
+ if (Live[I] && RPTargets[I].isSaveBeneficial(Reg))
+ return true;
}
- REMAT_DEBUG(dbgs() << "Can reduce but not eliminate spilling\n");
- return !Rematerializations.empty();
+ return false;
}
-void PreRARematStage::rematerialize() {
- const SIInstrInfo *TII = MF.getSubtarget<GCNSubtarget>().getInstrInfo();
+void PreRARematStage::RematReg::insertMI(unsigned RegionIdx,
+ MachineInstr *RematMI,
+ GCNScheduleDAGMILive &DAG) const {
+ RegionBoundaries &Bounds = DAG.Regions[RegionIdx];
+ if (Bounds.first == std::next(MachineBasicBlock::iterator(RematMI)))
+ Bounds.first = RematMI;
+ DAG.LIS->InsertMachineInstrInMaps(*RematMI);
+ DAG.LIS->createAndComputeVirtRegInterval(RematMI->getOperand(0).getReg());
+}
+
+PreRARematStage::ScoredRemat::FreqInfo::FreqInfo(
+ MachineFunction &MF, const GCNScheduleDAGMILive &DAG) {
+ assert(DAG.MLI && "MLI not defined in DAG");
+ MachineBranchProbabilityInfo MBPI;
+ MachineBlockFrequencyInfo MBFI(MF, MBPI, *DAG.MLI);
+
+ const unsigned NumRegions = DAG.Regions.size();
+ MinFreq = MBFI.getEntryFreq().getFrequency();
+ MaxFreq = 0;
+ Regions.reserve(NumRegions);
+ for (unsigned I = 0; I < NumRegions; ++I) {
+ MachineBasicBlock *MBB = DAG.Regions[I].first->getParent();
+ uint64_t BlockFreq = MBFI.getBlockFreq(MBB).getFrequency();
+ Regions.push_back(BlockFreq);
+ if (BlockFreq && BlockFreq < MinFreq)
+ MinFreq = BlockFreq;
+ else if (BlockFreq > MaxFreq)
+ MaxFreq = BlockFreq;
+ }
+ if (!MinFreq)
+ return;
- // Collect regions whose RP changes in unpredictable way; we will have to
- // fully recompute their RP after all rematerailizations.
- DenseSet<unsigned> RecomputeRP;
-
- // Rematerialize all instructions.
- for (auto &[DefMI, Remat] : Rematerializations) {
- MachineBasicBlock::iterator InsertPos(Remat.UseMI);
- Register Reg = DefMI->getOperand(0).getReg();
- unsigned DefRegion = MIRegion.at(DefMI);
-
- // Rematerialize DefMI to its use block.
- TII->reMaterialize(*InsertPos->getParent(), InsertPos, Reg,
- AMDGPU::NoSubRegister, *DefMI);
- Remat.RematMI = &*std::prev(InsertPos);
- DAG.LIS->InsertMachineInstrInMaps(*Remat.RematMI);
-
- // Update region boundaries in regions we sinked from (remove defining MI)
- // and to (insert MI rematerialized in use block). Only then we can erase
- // the original MI.
- DAG.updateRegionBoundaries(DAG.Regions[DefRegion], DefMI, nullptr);
- auto UseRegion = MIRegion.find(Remat.UseMI);
- if (UseRegion != MIRegion.end()) {
- DAG.updateRegionBoundaries(DAG.Regions[UseRegion->second], InsertPos,
- Remat.RematMI);
+ // Scale everything down if frequencies are high.
+ if (MinFreq >= ScaleFactor * ScaleFactor) {
+ for (uint64_t &Freq : Regions)
+ Freq /= ScaleFactor;
+ MinFreq /= ScaleFactor;
+ MaxFreq /= ScaleFactor;
+ }
+}
+
+PreRARematStage::ScoredRemat::ScoredRemat(const RematReg *Remat,
+ const FreqInfo &Freq,
+ const GCNScheduleDAGMILive &DAG)
+ : Remat(Remat), NumRegs(getNumRegs(DAG)), FreqDiff(getFreqDiff(Freq)) {}
+
+unsigned PreRARematStage::ScoredRemat::getNumRegs(
+ const GCNScheduleDAGMILive &DAG) const {
+ const TargetRegisterClass &RC = *DAG.MRI.getRegClass(Remat->getReg());
+ unsigned RegSize = DAG.TRI->getRegSizeInBits(RC);
+ if (unsigned SubIdx = Remat->DefMI->getOperand(0).getSubReg()) {
+ // The following may return -1 (i.e., a large unsigned number) on indices
+ // that may be used to access subregisters of multiple sizes; in such cases
+ // fallback on the size derived from the register class.
+ unsigned SubRegSize = DAG.TRI->getSubRegIdxSize(SubIdx);
+ if (SubRegSize < RegSize)
+ RegSize = SubRegSize;
+ }
+ return divideCeil(RegSize, 32);
+}
+
+int64_t PreRARematStage::ScoredRemat::getFreqDiff(const FreqInfo &Freq) const {
+ // Get frequencies of defining and using regions. A rematerialization from
the
+ // least frequent region to the most frequent region will yield the greatest
+ // latency penalty and therefore should get minimum score. Reciprocally, a
+ // rematerialization in the other direction should get maximum score. Default
+ // to values that will yield the worst possible score given known frequencies
+ // in order to penalize rematerializations from or into regions whose
+ // frequency is unknown.
+ int64_t DefOrMin = std::max(Freq.Regions[Remat->DefRegion], Freq.MinFreq);
+ int64_t UseOrMax = Freq.Regions[Remat->UseRegion];
+ if (!UseOrMax)
+ UseOrMax = Freq.MaxFreq;
+ return DefOrMin - UseOrMax;
+}
+
+void PreRARematStage::ScoredRemat::update(const BitVector &TargetRegions,
+ ArrayRef<GCNRPTarget> RPTargets,
+ const FreqInfo &FreqInfo,
+ bool ReduceSpill) {
+ MaxFreq = 0;
+ RegionImpact = 0;
+ for (unsigned I : TargetRegions.set_bits()) {
+ if (!Remat->Live[I] || !RPTargets[I].isSaveBeneficial(Remat->getReg()))
+ continue;
+ bool UnusedLT = Remat->isUnusedLiveThrough(I);
+
+ // Regions in which RP is guaranteed to decrease have more weight.
+ RegionImpact += UnusedLT ? 2 : 1;
+
+ if (ReduceSpill) {
+ uint64_t Freq = FreqInfo.Regions[I];
+ if (!UnusedLT) {
+ // Apply a frequency penalty in regions in which we are not sure that
RP
+ // will decrease.
+ Freq /= 2;
+ }
+ MaxFreq = std::max(MaxFreq, Freq);
}
- DAG.LIS->RemoveMachineInstrFromMaps(*DefMI);
- DefMI->eraseFromParent();
+ }
+ RegionImpact *= NumRegs;
+}
- // Collect all regions impacted by the rematerialization and update their
- // live-in/RP information.
- for (unsigned I : Remat.LiveInRegions) {
- ImpactedRegions.insert({I, DAG.Pressure[I]});
- GCNRPTracker::LiveRegSet &RegionLiveIns = DAG.LiveIns[I];
+void PreRARematStage::rematerialize(const RematReg &Remat,
+ BitVector &RecomputeRP,
+ RollbackInfo *Rollback) {
+ const SIInstrInfo *TII = MF.getSubtarget<GCNSubtarget>().getInstrInfo();
+ MachineInstr &DefMI = *Remat.DefMI;
+ Register Reg = DefMI.getOperand(0).getReg();
+ Register NewReg = DAG.MRI.cloneVirtualRegister(Reg);
+
+ // Rematerialize the register in the region where it is used.
+ MachineBasicBlock::iterator InsertPos = Remat.UseMI;
+ TII->reMaterialize(*InsertPos->getParent(), InsertPos, NewReg, 0, DefMI);
+ MachineInstr *RematMI = &*std::prev(InsertPos);
+ Remat.UseMI->substituteRegister(Reg, NewReg, 0, *DAG.TRI);
+ Remat.insertMI(Remat.UseRegion, RematMI, DAG);
+ if (Rollback) {
+ Rollback->RematMI = RematMI;
+ // Make the original MI a debug instruction so that it does not influence
+ // scheduling.
+ DefMI.setDesc(TII->get(TargetOpcode::DBG_VALUE));
----------------
lucas-rami wrote:
Yes the def/KILL interaction causes issues here. I was able to verify that KILL
instructions with defs still contribute to and increase RP locally. Even if the
defined register is marked dead RP increases over the instruction, which can
incorrectly push our max pressure estimate above what it really is.
https://github.com/llvm/llvm-project/pull/177206
_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
