https://github.com/heiher updated https://github.com/llvm/llvm-project/pull/218597
>From c1b6f774b5781ca2ced34430866f10a481c54c82 Mon Sep 17 00:00:00 2001 From: WANG Rui <[email protected]> Date: Mon, 17 Aug 2026 14:04:22 +0800 Subject: [PATCH 1/4] [LoongArch] Add memory barrier optimization pass Add a machine-level pass to remove redundant DBARs and merge them with atomic memory operations. The pass uses machine dominator and post-dominator trees to ensure the transformation is safe across basic blocks. --- llvm/lib/Target/LoongArch/CMakeLists.txt | 1 + llvm/lib/Target/LoongArch/LoongArch.h | 2 + .../LoongArch/LoongArchMemoryBarrierOpt.cpp | 645 ++++++++++++ .../LoongArch/LoongArchTargetMachine.cpp | 3 + .../CodeGen/LoongArch/memory-barrier-opt.ll | 987 ++++++++++++++---- llvm/test/CodeGen/LoongArch/opt-pipeline.ll | 3 + .../llvm/lib/Target/LoongArch/BUILD.gn | 1 + 7 files changed, 1437 insertions(+), 205 deletions(-) create mode 100644 llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp diff --git a/llvm/lib/Target/LoongArch/CMakeLists.txt b/llvm/lib/Target/LoongArch/CMakeLists.txt index c856173ca89de..e720afb7aab45 100644 --- a/llvm/lib/Target/LoongArch/CMakeLists.txt +++ b/llvm/lib/Target/LoongArch/CMakeLists.txt @@ -25,6 +25,7 @@ add_llvm_target(LoongArchCodeGen LoongArchISelDAGToDAG.cpp LoongArchISelLowering.cpp LoongArchMCInstLower.cpp + LoongArchMemoryBarrierOpt.cpp LoongArchMergeBaseOffset.cpp LoongArchOptWInstrs.cpp LoongArchRegisterInfo.cpp diff --git a/llvm/lib/Target/LoongArch/LoongArch.h b/llvm/lib/Target/LoongArch/LoongArch.h index e5b3083348792..b5534b535b54b 100644 --- a/llvm/lib/Target/LoongArch/LoongArch.h +++ b/llvm/lib/Target/LoongArch/LoongArch.h @@ -37,6 +37,7 @@ FunctionPass *createLoongArchDeadRegisterDefinitionsPass(); FunctionPass *createLoongArchExpandAtomicPseudoPass(); FunctionPass *createLoongArchISelDag(LoongArchTargetMachine &TM, CodeGenOptLevel OptLevel); +FunctionPass *createLoongArchMemoryBarrierOptPass(); FunctionPass *createLoongArchMergeBaseOffsetOptPass(); FunctionPass *createLoongArchOptWInstrsPass(); FunctionPass *createLoongArchPreRAExpandPseudoPass(); @@ -45,6 +46,7 @@ void initializeLoongArchAsmPrinterPass(PassRegistry &); void initializeLoongArchDAGToDAGISelLegacyPass(PassRegistry &); void initializeLoongArchDeadRegisterDefinitionsPass(PassRegistry &); void initializeLoongArchExpandAtomicPseudoPass(PassRegistry &); +void initializeLoongArchMemoryBarrierOptPass(PassRegistry &); void initializeLoongArchMergeBaseOffsetOptPass(PassRegistry &); void initializeLoongArchOptWInstrsPass(PassRegistry &); void initializeLoongArchPreRAExpandPseudoPass(PassRegistry &); diff --git a/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp b/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp new file mode 100644 index 0000000000000..1f29cdc8638c1 --- /dev/null +++ b/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp @@ -0,0 +1,645 @@ +//===---- LoongArchMemoryBarrierOpt.cpp - Memory barrier Optimization -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// This pass removes or merges redundant memory barrier instructions. +/// +/// - DBAR x + DBAR y -> DBAR (x & y) +/// - DBAR x + AMO_DB -> AMO_DB +/// - DBAR x + AMO -> AMO_DB +/// - DBAR x + LL -> LL +/// - AMO_DB + DBAR x -> AMO_DB +/// - AMO + DBAR x -> AMO_DB +/// - SC + DBAR x -> SC +/// +//===----------------------------------------------------------------------===// + +#include "LoongArch.h" +#include "LoongArchInstrInfo.h" +#include "LoongArchSubtarget.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachinePostDominators.h" +#include "llvm/InitializePasses.h" + +using namespace llvm; + +#define DEBUG_TYPE "loongarch-memory-barrier-opt" +#define LOONGARCH_MEMORY_BARRIER_OPT_NAME \ + "LoongArch Memory Barrier Optimisation pass" + +static cl::opt<bool> RequireNoPathBypass( + "loongarch-require-no-path-bypass", + cl::desc("Optimize only when no paths bypass either memory barrier"), + cl::init(true), cl::Hidden); + +static cl::opt<bool> DisableInlineAsm( + "loongarch-disable-inline-asm-barrier-opt", + cl::desc("Disable optimization of memory barriers in InlineAsm"), + cl::init(false), cl::Hidden); + +static cl::opt<bool> ReplaceEliminatedMBToNop( + "loongarch-replace-eliminated-dbar-to-nop", + cl::desc("Replace eliminated DBARs with NOPs to preserve code layout"), + cl::init(false), cl::Hidden); + +namespace { + +static std::optional<std::pair<StringRef, StringRef>> parseMB(StringRef Asm) { + auto T1 = llvm::getToken(Asm); + if (!T1.first.equals_insensitive("dbar")) + return std::nullopt; + auto T2 = llvm::getToken(T1.second); + if (T2.first.empty()) + return std::nullopt; + auto T3 = llvm::getToken(T2.second); + if (T3.first.trim().empty() || T3.first.starts_with('#')) + return std::pair(T1.first, T2.first); + return std::nullopt; +} + +static std::optional<std::pair<StringRef, StringRef>> +isAsmMB(const MachineInstr &MI) { + if (DisableInlineAsm) + return std::nullopt; + if (!MI.isInlineAsm()) + return std::nullopt; + auto Asm = MI.getOperand(InlineAsm::MIOp_AsmString).getSymbolName(); + return parseMB(Asm); +} + +static StringRef getAMDB(StringRef Name) { + return StringSwitch<StringRef>(Name.lower()) + .Case("amswap.b", "amswap_db.b") + .Case("amswap.h", "amswap_db.h") + .Case("amswap.w", "amswap_db.w") + .Case("amswap.d", "amswap_db.d") + .Case("amswap_db.b", "amswap_db.b") + .Case("amswap_db.h", "amswap_db.h") + .Case("amswap_db.w", "amswap_db.w") + .Case("amswap_db.d", "amswap_db.d") + .Case("amadd.b", "amadd_db.b") + .Case("amadd.h", "amadd_db.h") + .Case("amadd.w", "amadd_db.w") + .Case("amadd.d", "amadd_db.d") + .Case("amadd_db.b", "amadd_db.b") + .Case("amadd_db.h", "amadd_db.h") + .Case("amadd_db.w", "amadd_db.w") + .Case("amadd_db.d", "amadd_db.d") + .Case("amand.w", "amand_db.w") + .Case("amand.d", "amand_db.d") + .Case("amand_db.w", "amand_db.w") + .Case("amand_db.d", "amand_db.d") + .Case("amor.w", "amor_db.w") + .Case("amor.d", "amor_db.d") + .Case("amor_db.w", "amor_db.w") + .Case("amor_db.d", "amor_db.d") + .Case("amxor.w", "amxor_db.w") + .Case("amxor.d", "amxor_db.d") + .Case("amxor_db.w", "amxor_db.w") + .Case("amxor_db.d", "amxor_db.d") + .Case("ammax.w", "ammax_db.w") + .Case("ammax.d", "ammax_db.d") + .Case("ammax_db.w", "ammax_db.w") + .Case("ammax_db.d", "ammax_db.d") + .Case("ammin.w", "ammin_db.w") + .Case("ammin.d", "ammin_db.d") + .Case("ammin_db.w", "ammin_db.w") + .Case("ammin_db.d", "ammin_db.d") + .Case("ammax.wu", "ammax_db.wu") + .Case("ammax.du", "ammax_db.du") + .Case("ammax_db.wu", "ammax_db.wu") + .Case("ammax_db.du", "ammax_db.du") + .Case("ammin.wu", "ammin_db.wu") + .Case("ammin.du", "ammin_db.du") + .Case("ammin_db.wu", "ammin_db.wu") + .Case("ammin_db.du", "ammin_db.du") + .Case("amcas.b", "amcas_db.b") + .Case("amcas.h", "amcas_db.h") + .Case("amcas.w", "amcas_db.w") + .Case("amcas.d", "amcas_db.d") + .Case("amcas_db.b", "amcas_db.b") + .Case("amcas_db.h", "amcas_db.h") + .Case("amcas_db.w", "amcas_db.w") + .Case("amcas_db.d", "amcas_db.d") + .Default({}); +} + +static std::optional<std::pair<StringRef, StringRef>> parseAM(StringRef Asm) { + auto T1 = llvm::getToken(Asm); + auto OpName = getAMDB(T1.first); + if (OpName.empty()) + return std::nullopt; + auto T2 = llvm::getToken(T1.second, ","); + if (T2.first.empty()) + return std::nullopt; + auto T3 = llvm::getToken(T2.second, ","); + if (T3.first.empty()) + return std::nullopt; + auto T4 = llvm::getToken(T3.second, ","); + if (T4.first.empty()) + return std::nullopt; + auto T5 = llvm::getToken(T4.second); + if (T5.first.trim().empty() || T5.first.starts_with('#')) { + StringRef Operands(T2.first.data(), + T4.first.data() + T4.first.size() - T2.first.data()); + return std::pair(OpName, Operands); + } + return std::nullopt; +} + +static std::optional<std::pair<StringRef, StringRef>> +isAsmAM(const MachineInstr &MI) { + if (DisableInlineAsm) + return std::nullopt; + if (!MI.isInlineAsm()) + return std::nullopt; + auto Asm = MI.getOperand(InlineAsm::MIOp_AsmString).getSymbolName(); + return parseAM(Asm); +} + +static bool isMB(const MachineInstr &MI) { + return MI.getOpcode() == LoongArch::DBAR; +} + +static bool isLL(const MachineInstr &MI) { + switch (MI.getOpcode()) { + case LoongArch::LL_W: + case LoongArch::LL_D: + return true; + default: + return false; + } +} + +static bool isSC(const MachineInstr &MI) { + switch (MI.getOpcode()) { + case LoongArch::SC_W: + case LoongArch::SC_D: + case LoongArch::SC_Q: + return true; + default: + return false; + } +} + +static std::optional<unsigned> isAM(const MachineInstr &MI) { + switch (MI.getOpcode()) { + case LoongArch::AMSWAP_B: + case LoongArch::AMSWAP__DB_B: + return LoongArch::AMSWAP__DB_B; + case LoongArch::AMSWAP_H: + case LoongArch::AMSWAP__DB_H: + return LoongArch::AMSWAP__DB_H; + case LoongArch::AMSWAP_W: + case LoongArch::AMSWAP__DB_W: + return LoongArch::AMSWAP__DB_W; + case LoongArch::AMSWAP_D: + case LoongArch::AMSWAP__DB_D: + return LoongArch::AMSWAP__DB_D; + case LoongArch::AMADD_B: + case LoongArch::AMADD__DB_B: + return LoongArch::AMADD__DB_B; + case LoongArch::AMADD_H: + case LoongArch::AMADD__DB_H: + return LoongArch::AMADD__DB_H; + case LoongArch::AMADD_W: + case LoongArch::AMADD__DB_W: + return LoongArch::AMADD__DB_W; + case LoongArch::AMADD_D: + case LoongArch::AMADD__DB_D: + return LoongArch::AMADD__DB_D; + case LoongArch::AMAND_W: + case LoongArch::AMAND__DB_W: + return LoongArch::AMAND__DB_W; + case LoongArch::AMAND_D: + case LoongArch::AMAND__DB_D: + return LoongArch::AMAND__DB_D; + case LoongArch::AMOR_W: + case LoongArch::AMOR__DB_W: + return LoongArch::AMOR__DB_W; + case LoongArch::AMOR_D: + case LoongArch::AMOR__DB_D: + return LoongArch::AMOR__DB_D; + case LoongArch::AMXOR_W: + case LoongArch::AMXOR__DB_W: + return LoongArch::AMXOR__DB_W; + case LoongArch::AMXOR_D: + case LoongArch::AMXOR__DB_D: + return LoongArch::AMXOR__DB_D; + case LoongArch::AMMAX_W: + case LoongArch::AMMAX__DB_W: + return LoongArch::AMMAX__DB_W; + case LoongArch::AMMAX_D: + case LoongArch::AMMAX__DB_D: + return LoongArch::AMMAX__DB_D; + case LoongArch::AMMIN_W: + case LoongArch::AMMIN__DB_W: + return LoongArch::AMMIN__DB_W; + case LoongArch::AMMIN_D: + case LoongArch::AMMIN__DB_D: + return LoongArch::AMMIN__DB_D; + case LoongArch::AMMAX_WU: + case LoongArch::AMMAX__DB_WU: + return LoongArch::AMMAX__DB_WU; + case LoongArch::AMMAX_DU: + case LoongArch::AMMAX__DB_DU: + return LoongArch::AMMAX__DB_DU; + case LoongArch::AMMIN_WU: + case LoongArch::AMMIN__DB_WU: + return LoongArch::AMMIN__DB_WU; + case LoongArch::AMMIN_DU: + case LoongArch::AMMIN__DB_DU: + return LoongArch::AMMIN__DB_DU; + case LoongArch::AMCAS_B: + case LoongArch::AMCAS__DB_B: + return LoongArch::AMCAS__DB_B; + case LoongArch::AMCAS_H: + case LoongArch::AMCAS__DB_H: + return LoongArch::AMCAS__DB_H; + case LoongArch::AMCAS_W: + case LoongArch::AMCAS__DB_W: + return LoongArch::AMCAS__DB_W; + case LoongArch::AMCAS_D: + case LoongArch::AMCAS__DB_D: + return LoongArch::AMCAS__DB_D; + default: + return std::nullopt; + } +} + +static bool isSafeToSkip(const MachineInstr &MI) { + if (MI.mayLoadOrStore()) + return false; + if (MI.isCall() || MI.isReturn()) + return false; + if (MI.isInlineAsm()) + return isAsmMB(MI) != std::nullopt; + if (MI.hasUnmodeledSideEffects()) + return isMB(MI); + return true; +} + +struct BarrierHint { + BarrierHint(unsigned Hint) : Hint(Hint) {} + + bool subsumes(const BarrierHint &O) const { return (Hint & O.Hint) == Hint; } + + BarrierHint merge(const BarrierHint &O) const { + return BarrierHint(Hint & O.Hint); + } + + static inline bool isValid(unsigned Hint) { return (Hint & ~0x1f) == 0; } + + unsigned Hint; +}; + +struct InstBarrier { + InstBarrier(MachineInstr &MI) + : MI(&MI), Pre(0), Post(0), Data(0), IsMB(false), IsAM(false), + IsAsm(false) { + if (isMB(MI)) { + unsigned Hint = MI.getOperand(0).getImm(); + if (!BarrierHint::isValid(Hint)) + return; + IsMB = true; + Pre = Post = BarrierHint(Hint); + } else if (isLL(MI)) { + IsAM = true; + Pre = BarrierHint(0b10000); + Post = BarrierHint(0b11111); + } else if (isSC(MI)) { + IsAM = true; + Pre = BarrierHint(0b11111); + Post = BarrierHint(0b10000); + } else if (auto R = isAM(MI)) { + IsAM = true; + OpcAMDB = *R; + Pre = Post = BarrierHint(0b10000); + } else if (auto R = isAsmMB(MI)) { + OpName = (*R).first; + Operands = (*R).second; + auto B = parseAsmMB(Operands, MI); + if (!B || !BarrierHint::isValid((*B).first)) + return; + IsMB = true; + IsAsm = true; + HintOff = (*B).second; + Pre = Post = BarrierHint((*B).first); + } else if (auto R = isAsmAM(MI)) { + OpName = (*R).first; + Operands = (*R).second; + IsAM = true; + IsAsm = true; + Pre = Post = BarrierHint(0b10000); + } + } + + static std::optional<std::pair<unsigned, unsigned>> + parseAsmMB(StringRef Operand, MachineInstr &MI) { + unsigned Hint, HintOff = 0; + // DBAR N | 0xN + if (!Operand.starts_with('$')) { + if (Operand.getAsInteger(0, Hint)) + return std::nullopt; + return std::pair(Hint, HintOff); + } + // DBAR $N + unsigned N = 0, Off, AsmDescOp; + if (Operand.drop_front().getAsInteger(0, Off)) + return std::nullopt; + AsmDescOp = InlineAsm::MIOp_FirstOperand; + while (AsmDescOp != MI.getNumOperands()) { + const MachineOperand &MO = MI.getOperand(AsmDescOp); + assert(MO.isImm() && "Unexpected operand type!"); + const InlineAsm::Flag F(MO.getImm()); + if (N == Off) { + assert(F.isImmKind() && "Unexpected flag kind!"); + HintOff = AsmDescOp + 1; + Hint = MI.getOperand(HintOff).getImm(); + return std::pair(Hint, HintOff); + } + AsmDescOp += 1 + F.getNumOperandRegisters(); + ++N; + } + return std::nullopt; + } + + MachineInstr *MI; + BarrierHint Pre; + BarrierHint Post; + StringRef OpName; + StringRef Operands; + union { + unsigned OpcAMDB; + unsigned HintOff; + unsigned Data; + }; + bool IsMB; + bool IsAM; + bool IsAsm; +}; + +class LoongArchMemoryBarrierOpt : public MachineFunctionPass { +public: + static char ID; + + LoongArchMemoryBarrierOpt() : MachineFunctionPass(ID) {} + + StringRef getPassName() const override { + return LOONGARCH_MEMORY_BARRIER_OPT_NAME; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired<MachineDominatorTreeWrapperPass>(); + AU.addPreserved<MachineDominatorTreeWrapperPass>(); + AU.addRequired<MachinePostDominatorTreeWrapperPass>(); + AU.addPreserved<MachinePostDominatorTreeWrapperPass>(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + bool runOnMachineFunction(MachineFunction &Fn) override; + +private: + enum : unsigned { + CandidateA = 1u << 0, + CandidateB = 1u << 1, + }; + + unsigned resolveBarrierRedundancy(const MachineInstr *A, + const MachineInstr *B) const; + bool eliminateRedundantBarrier(InstBarrier &IA, InstBarrier &IB) const; + + MachineFunction *MF; + const MachineDominatorTree *MDT; + const MachinePostDominatorTree *MPDT; +}; + +static bool checkAllPathSafe(const MachineBasicBlock *MBBA, + const MachineBasicBlock *MBBB, bool IsAToB) { + const MachineBasicBlock *Start = IsAToB ? MBBA : MBBB; + const MachineBasicBlock *End = IsAToB ? MBBB : MBBA; + + SmallVector<const MachineBasicBlock *, 16> Worklist; + DenseSet<const MachineBasicBlock *> Visited; + + Worklist.push_back(Start); + Visited.insert(Start); + + while (!Worklist.empty()) { + const MachineBasicBlock *BB = Worklist.pop_back_val(); + + if (BB == End) + continue; + + if (BB != Start) { + for (const MachineInstr &MI : *BB) { + if (!isSafeToSkip(MI)) + return false; + } + } + + if (IsAToB) { + for (const MachineBasicBlock *Succ : BB->successors()) { + if (Visited.insert(Succ).second) + Worklist.push_back(Succ); + } + } else { + for (const MachineBasicBlock *Pred : BB->predecessors()) { + if (Visited.insert(Pred).second) + Worklist.push_back(Pred); + } + } + } + + return true; +} + +// Returns a bitmask indicating removal candidates: A (bit 1) and B (bit 2). +unsigned LoongArchMemoryBarrierOpt::resolveBarrierRedundancy( + const MachineInstr *A, const MachineInstr *B) const { + const MachineBasicBlock *MBBA = A->getParent(); + const MachineBasicBlock *MBBB = B->getParent(); + + if (MBBA == MBBB) { + /* A -> B */ + for (auto It = std::next(A->getIterator()); It != MBBA->end(); ++It) { + if (It == B->getIterator()) + return CandidateA | CandidateB; + if (!isSafeToSkip(*It)) + return 0; + } + return 0; + } + + // Cross-block walk + bool ADomB = MDT->dominates(MBBA, MBBB); + bool BPostDomA = MPDT->dominates(MBBB, MBBA); + unsigned Mask = 0; + if (!ADomB && !BPostDomA) + return 0; + + /* A -> MBBA->end() */ + for (auto It = std::next(A->getIterator()); It != MBBA->end(); ++It) + if (!isSafeToSkip(*It)) + return 0; + /* B -> MBBB->begin() */ + for (auto It = MBBB->begin(); It != B->getIterator(); ++It) + if (!isSafeToSkip(*It)) + return 0; + + /* MBBA -> MBBB */ + if (BPostDomA) + if (checkAllPathSafe(MBBA, MBBB, true /*IsAToB*/)) + Mask |= CandidateA; + + /* MBBB -> MBBA */ + if (ADomB) + if (checkAllPathSafe(MBBA, MBBB, false /*IsAToB*/)) + Mask |= CandidateB; + + return Mask; +} + +// Update DBAR hint +static void updateMB(InstBarrier &I, BarrierHint Hint, MachineFunction *MF) { + assert(I.IsMB && "Unexpected!"); + I.Pre = I.Post = Hint; + if (!I.IsAsm) { + I.MI->getOperand(0).setImm(Hint.Hint); + return; + } + if (I.HintOff) { + I.MI->getOperand(I.HintOff).setImm(Hint.Hint); + return; + } + MachineOperand &MO = I.MI->getOperand(InlineAsm::MIOp_AsmString); + auto New = I.OpName.str() + " " + llvm::utostr(Hint.Hint); + auto Sym = MF->createExternalSymbolName(New); + MO = MachineOperand::CreateES(Sym); +} + +// Replace AMO to AMO_DB +static void replaceAM(InstBarrier &I, MachineFunction *MF) { + if (!I.IsAM) + return; + if (I.OpcAMDB) { + auto &ST = MF->getSubtarget<LoongArchSubtarget>(); + I.MI->setDesc(ST.getInstrInfo()->get(I.OpcAMDB)); + return; + } + if (!I.IsAsm) + return; + MachineOperand &MO = I.MI->getOperand(InlineAsm::MIOp_AsmString); + auto New = I.OpName.str() + " " + I.Operands.str(); + auto Sym = MF->createExternalSymbolName(New); + MO = MachineOperand::CreateES(Sym); +} + +bool LoongArchMemoryBarrierOpt::eliminateRedundantBarrier( + InstBarrier &IA, InstBarrier &IB) const { + MachineInstr *A = IA.MI; + MachineInstr *B = IB.MI; + + if (!A || !B) + return false; // Already erased + if (A == B) + return false; + + unsigned Mask = resolveBarrierRedundancy(A, B); + if (!Mask) + return false; + + auto eraseOrReplaceWithNop = [&](MachineInstr *MI) { + if (ReplaceEliminatedMBToNop) { + auto &ST = MF->getSubtarget<LoongArchSubtarget>(); + BuildMI(*MI->getParent(), MI->getIterator(), MI->getDebugLoc(), + ST.getInstrInfo()->get(LoongArch::ANDI), LoongArch::R0) + .addReg(LoongArch::R0) + .addImm(0); + } + MI->eraseFromParent(); + }; + + // A B + // DBAR x + DBAR y -> DBAR (x & y) + // DBAR x + AMO_DB -> AMO_DB + // DBAR x + AMO -> AMO_DB + // DBAR x + LL -> LL + if ((Mask & CandidateA) && IA.IsMB) { + if (!IB.Pre.subsumes(IA.Post)) { + if (!IB.IsMB || (RequireNoPathBypass && !(Mask & CandidateB))) + return false; + updateMB(IB, IB.Pre.merge(IA.Post), MF); + } + replaceAM(IB, MF); + eraseOrReplaceWithNop(A); + IA.MI = nullptr; + return true; + } + + // A B + // DBAR x + DBAR y -> DBAR (x & y) + // AMO_DB + DBAR x -> AMO_DB + // AMO + DBAR x -> AMO_DB + // SC + DBAR x -> SC + if ((Mask & CandidateB) && IB.IsMB) { + if (!IA.Post.subsumes(IB.Pre)) { + if (!IA.IsMB || (RequireNoPathBypass && !(Mask & CandidateA))) + return false; + updateMB(IA, IA.Post.merge(IB.Pre), MF); + } + replaceAM(IA, MF); + eraseOrReplaceWithNop(B); + IB.MI = nullptr; + return true; + } + + return false; +} + +bool LoongArchMemoryBarrierOpt::runOnMachineFunction(MachineFunction &Fn) { + if (skipFunction(Fn.getFunction())) + return false; + + MF = &Fn; + MDT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree(); + MPDT = &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree(); + + SmallVector<InstBarrier, 32> Sites; + bool Changed = false; + + for (MachineBasicBlock &MBB : Fn) + for (MachineInstr &MI : MBB) { + InstBarrier IB(MI); + if (IB.IsMB || IB.IsAM) + Sites.push_back(IB); + } + + for (size_t a = 0; a < Sites.size(); ++a) { + for (size_t b = a + 1; b < Sites.size(); ++b) { + InstBarrier &IA = Sites[a]; + InstBarrier &IB = Sites[b]; + Changed |= eliminateRedundantBarrier(IA, IB); + Changed |= eliminateRedundantBarrier(IB, IA); + } + } + + return Changed; +} +} // namespace + +char LoongArchMemoryBarrierOpt::ID = 0; +INITIALIZE_PASS_BEGIN(LoongArchMemoryBarrierOpt, DEBUG_TYPE, + LOONGARCH_MEMORY_BARRIER_OPT_NAME, false, false) +INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass) +INITIALIZE_PASS_END(LoongArchMemoryBarrierOpt, DEBUG_TYPE, + LOONGARCH_MEMORY_BARRIER_OPT_NAME, false, false) + +FunctionPass *llvm::createLoongArchMemoryBarrierOptPass() { + return new LoongArchMemoryBarrierOpt(); +} diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp index 0bb73a88bd6fd..faaf33a9162bf 100644 --- a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp @@ -38,6 +38,7 @@ LLVMInitializeLoongArchTarget() { RegisterTargetMachine<LoongArchTargetMachine> Y(getTheLoongArch64Target()); auto *PR = PassRegistry::getPassRegistry(); initializeLoongArchDeadRegisterDefinitionsPass(*PR); + initializeLoongArchMemoryBarrierOptPass(*PR); initializeLoongArchMergeBaseOffsetOptPass(*PR); initializeLoongArchOptWInstrsPass(*PR); initializeLoongArchPreRAExpandPseudoPass(*PR); @@ -201,6 +202,8 @@ void LoongArchPassConfig::addPreEmitPass2() { // avoiding the possibility for other passes to break the requirements for // forward progress in the LL/SC block. addPass(createLoongArchExpandAtomicPseudoPass()); + if (TM->getOptLevel() != CodeGenOptLevel::None) + addPass(createLoongArchMemoryBarrierOptPass()); } void LoongArchPassConfig::addMachineSSAOptimization() { diff --git a/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll b/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll index fa37dda01c7b1..3382e28c14891 100644 --- a/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll +++ b/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll @@ -1,14 +1,35 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s +; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefixes=CHECK,NORMAL +; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-require-no-path-bypass=false < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,BYPASS +; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-disable-inline-asm-barrier-opt=true < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,NOIASM +; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-replace-eliminated-dbar-to-nop=true < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,BTONOP ;; Positive define void @dbar_acquire_acquire() nounwind { -; CHECK-LABEL: dbar_acquire_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 20 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 20 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 20 +; BTONOP-NEXT: ret entry: fence acquire fence acquire @@ -16,11 +37,26 @@ entry: } define void @dbar_acquire_release() nounwind { -; CHECK-LABEL: dbar_acquire_release: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_release: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 16 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_release: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 16 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_release: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 16 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_release: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 16 +; BTONOP-NEXT: ret entry: fence acquire fence release @@ -28,12 +64,30 @@ entry: } define i32 @dbar_acquire_add_acquire(i32 %a) nounwind { -; CHECK-LABEL: dbar_acquire_add_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: addi.w $a0, $a0, 1 -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_add_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: addi.w $a0, $a0, 1 +; NORMAL-NEXT: dbar 20 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_add_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: addi.w $a0, $a0, 1 +; BYPASS-NEXT: dbar 20 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_add_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: addi.w $a0, $a0, 1 +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_add_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: addi.w $a0, $a0, 1 +; BTONOP-NEXT: dbar 20 +; BTONOP-NEXT: ret entry: fence acquire %0 = add i32 %a, 1 @@ -42,11 +96,26 @@ entry: } define void @dbar_release_acquire() nounwind { -; CHECK-LABEL: dbar_release_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_release_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 16 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_release_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 16 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_release_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 16 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_release_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 16 +; BTONOP-NEXT: ret entry: fence release fence acquire @@ -54,12 +123,27 @@ entry: } define void @dbar_seq_cst_acq_rel_seq_cst() nounwind { -; CHECK-LABEL: dbar_seq_cst_acq_rel_seq_cst: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 16 -; CHECK-NEXT: dbar 16 -; CHECK-NEXT: dbar 16 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_seq_cst_acq_rel_seq_cst: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 16 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_seq_cst_acq_rel_seq_cst: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 16 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_seq_cst_acq_rel_seq_cst: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 16 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_seq_cst_acq_rel_seq_cst: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 16 +; BTONOP-NEXT: ret entry: fence seq_cst fence acq_rel @@ -68,14 +152,31 @@ entry: } define void @dbar_acquire_asm_dbar_acquire() nounwind { -; CHECK-LABEL: dbar_acquire_asm_dbar_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: dbar 22 -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_dbar_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 20 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_dbar_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 20 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_dbar_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: dbar 22 +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_dbar_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 20 +; BTONOP-NEXT: ret entry: fence acquire call void asm "dbar 22", ""() @@ -84,14 +185,31 @@ entry: } define void @dbar_acquire_asm_space_dbar_acquire() nounwind { -; CHECK-LABEL: dbar_acquire_asm_space_dbar_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: dbar 22 -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_space_dbar_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 20 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_space_dbar_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 20 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_space_dbar_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: dbar 22 +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_space_dbar_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 20 +; BTONOP-NEXT: ret entry: fence acquire call void asm " dbar 22", ""() @@ -100,14 +218,31 @@ entry: } define void @dbar_acquire_asm_comment_dbar_acquire() nounwind { -; CHECK-LABEL: dbar_acquire_asm_comment_dbar_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: dbar 22 # barrier -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_comment_dbar_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 20 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_comment_dbar_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 20 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_comment_dbar_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: dbar 22 # barrier +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_comment_dbar_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 20 +; BTONOP-NEXT: ret entry: fence acquire call void asm "dbar 22 # barrier", ""() @@ -116,14 +251,31 @@ entry: } define void @dbar_acquire_asm_arg0_dbar_acquire() nounwind { -; CHECK-LABEL: dbar_acquire_asm_arg0_dbar_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: dbar 16 -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_arg0_dbar_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 16 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_arg0_dbar_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 16 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_arg0_dbar_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: dbar 16 +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_arg0_dbar_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 16 +; BTONOP-NEXT: ret entry: fence acquire call void asm sideeffect "dbar $0", "I,I,~{memory}"(i32 16, i32 22) @@ -132,14 +284,31 @@ entry: } define void @dbar_acquire_asm_arg1_dbar_acquire() nounwind { -; CHECK-LABEL: dbar_acquire_asm_arg1_dbar_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: dbar 16 -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_arg1_dbar_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 16 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_arg1_dbar_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 16 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_arg1_dbar_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: dbar 16 +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_arg1_dbar_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 16 +; BTONOP-NEXT: ret entry: fence acquire call void asm sideeffect "dbar $1", "I,I,~{memory}"(i32 22, i32 16) @@ -148,11 +317,26 @@ entry: } define void @dbar_completion_acquire() nounwind { -; CHECK-LABEL: dbar_completion_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 0 -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_completion_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 0 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_completion_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 0 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_completion_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 0 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_completion_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 0 +; BTONOP-NEXT: ret entry: call void @llvm.loongarch.dbar(i32 0) fence acquire @@ -160,11 +344,26 @@ entry: } define void @dbar_release_completion() nounwind { -; CHECK-LABEL: dbar_release_completion: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: dbar 0 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_release_completion: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: dbar 0 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_release_completion: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: dbar 0 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_release_completion: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 0 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_release_completion: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: dbar 0 +; BTONOP-NEXT: ret entry: fence release call void @llvm.loongarch.dbar(i32 0) @@ -172,12 +371,30 @@ entry: } define void @dbar_release_amswap_i8_acquire() nounwind { -; CHECK-LABEL: dbar_release_amswap_i8_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: addi.w $a0, $zero, -256 -; CHECK-NEXT: amand_db.w $a1, $a0, $zero -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_release_amswap_i8_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: addi.w $a0, $zero, -256 +; NORMAL-NEXT: amand_db.w $a1, $a0, $zero +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_release_amswap_i8_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: addi.w $a0, $zero, -256 +; BYPASS-NEXT: amand_db.w $a1, $a0, $zero +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_release_amswap_i8_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: addi.w $a0, $zero, -256 +; NOIASM-NEXT: amand_db.w $a1, $a0, $zero +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_release_amswap_i8_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: addi.w $a0, $zero, -256 +; BTONOP-NEXT: amand_db.w $a1, $a0, $zero +; BTONOP-NEXT: ret entry: fence release atomicrmw xchg ptr null, i8 0 acquire @@ -185,11 +402,26 @@ entry: } define void @dbar_release_amswap_i32_acquire() nounwind { -; CHECK-LABEL: dbar_release_amswap_i32_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: amswap_db.w $a0, $zero, $zero -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_release_amswap_i32_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: amswap_db.w $a0, $zero, $zero +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_release_amswap_i32_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_release_amswap_i32_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_release_amswap_i32_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: amswap_db.w $a0, $zero, $zero +; BTONOP-NEXT: ret entry: fence release atomicrmw xchg ptr null, i32 0 acquire @@ -197,11 +429,26 @@ entry: } define void @dbar_release_amswap_i32_monotonic() nounwind { -; CHECK-LABEL: dbar_release_amswap_i32_monotonic: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: amswap.w $a0, $zero, $zero -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_release_amswap_i32_monotonic: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: amswap_db.w $a0, $zero, $zero +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_release_amswap_i32_monotonic: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_release_amswap_i32_monotonic: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_release_amswap_i32_monotonic: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: amswap_db.w $a0, $zero, $zero +; BTONOP-NEXT: ret entry: fence release atomicrmw xchg ptr null, i32 0 monotonic @@ -221,12 +468,30 @@ entry: } define void @amswap_i8_release_dbar_acquire() nounwind { -; CHECK-LABEL: amswap_i8_release_dbar_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: addi.w $a0, $zero, -256 -; CHECK-NEXT: amand_db.w $a1, $a0, $zero -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: amswap_i8_release_dbar_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: addi.w $a0, $zero, -256 +; NORMAL-NEXT: amand_db.w $a1, $a0, $zero +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: amswap_i8_release_dbar_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: addi.w $a0, $zero, -256 +; BYPASS-NEXT: amand_db.w $a1, $a0, $zero +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: amswap_i8_release_dbar_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: addi.w $a0, $zero, -256 +; NOIASM-NEXT: amand_db.w $a1, $a0, $zero +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: amswap_i8_release_dbar_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: addi.w $a0, $zero, -256 +; BTONOP-NEXT: amand_db.w $a1, $a0, $zero +; BTONOP-NEXT: nop +; BTONOP-NEXT: ret entry: atomicrmw xchg ptr null, i8 0 release fence acquire @@ -234,11 +499,26 @@ entry: } define void @amswap_i32_release_dbar_acquire() nounwind { -; CHECK-LABEL: amswap_i32_release_dbar_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: amswap_db.w $a0, $zero, $zero -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: amswap_i32_release_dbar_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: amswap_db.w $a0, $zero, $zero +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: amswap_i32_release_dbar_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: amswap_i32_release_dbar_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: amswap_i32_release_dbar_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: amswap_db.w $a0, $zero, $zero +; BTONOP-NEXT: nop +; BTONOP-NEXT: ret entry: atomicrmw xchg ptr null, i32 0 release fence acquire @@ -258,13 +538,31 @@ entry: } define void @dbar_release_amswap_i8_release_dbar_acquire() nounwind { -; CHECK-LABEL: dbar_release_amswap_i8_release_dbar_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: addi.w $a0, $zero, -256 -; CHECK-NEXT: amand_db.w $a1, $a0, $zero -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_release_amswap_i8_release_dbar_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: addi.w $a0, $zero, -256 +; NORMAL-NEXT: amand_db.w $a1, $a0, $zero +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_release_amswap_i8_release_dbar_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: addi.w $a0, $zero, -256 +; BYPASS-NEXT: amand_db.w $a1, $a0, $zero +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_release_amswap_i8_release_dbar_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: addi.w $a0, $zero, -256 +; NOIASM-NEXT: amand_db.w $a1, $a0, $zero +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_release_amswap_i8_release_dbar_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: addi.w $a0, $zero, -256 +; BTONOP-NEXT: amand_db.w $a1, $a0, $zero +; BTONOP-NEXT: nop +; BTONOP-NEXT: ret entry: fence release atomicrmw xchg ptr null, i8 0 release @@ -273,12 +571,27 @@ entry: } define void @dbar_acquire_amswap_i32_release_dbar_acquire() nounwind { -; CHECK-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: amswap_db.w $a0, $zero, $zero -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: amswap_db.w $a0, $zero, $zero +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: amswap_db.w $a0, $zero, $zero +; BTONOP-NEXT: nop +; BTONOP-NEXT: ret entry: fence acquire atomicrmw xchg ptr null, i32 0 release @@ -301,13 +614,35 @@ entry: } define void @dbar_acquire_asm_amswap_i32_release() nounwind { -; CHECK-LABEL: dbar_acquire_asm_amswap_i32_release: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: amswap_db.w $a0, $a1, $a2 -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_amswap_i32_release: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: #APP +; NORMAL-NEXT: amswap_db.w $a0, $a1, $a2 +; NORMAL-NEXT: #NO_APP +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_amswap_i32_release: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: #APP +; BYPASS-NEXT: amswap_db.w $a0, $a1, $a2 +; BYPASS-NEXT: #NO_APP +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_amswap_i32_release: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: amswap_db.w $a0, $a1, $a2 +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_amswap_i32_release: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: #APP +; BTONOP-NEXT: amswap_db.w $a0, $a1, $a2 +; BTONOP-NEXT: #NO_APP +; BTONOP-NEXT: ret entry: fence acquire call void asm "amswap_db.w $$r4, $$r5, $$r6", ""() @@ -315,13 +650,35 @@ entry: } define void @dbar_acquire_asm_amswap_i32_monotonic() nounwind { -; CHECK-LABEL: dbar_acquire_asm_amswap_i32_monotonic: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: amswap.w $a0, $a1, $a2 -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_amswap_i32_monotonic: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: #APP +; NORMAL-NEXT: amswap_db.w $a0, $a1, $a2 +; NORMAL-NEXT: #NO_APP +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_amswap_i32_monotonic: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: #APP +; BYPASS-NEXT: amswap_db.w $a0, $a1, $a2 +; BYPASS-NEXT: #NO_APP +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_amswap_i32_monotonic: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: amswap.w $a0, $a1, $a2 +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_amswap_i32_monotonic: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: #APP +; BTONOP-NEXT: amswap_db.w $a0, $a1, $a2 +; BTONOP-NEXT: #NO_APP +; BTONOP-NEXT: ret entry: fence acquire call void asm sideeffect "amswap.w $$r4, $$r5, $$r6", ""() @@ -329,13 +686,35 @@ entry: } define void @dbar_acquire_asm_space_amswap_i32_monotonic() nounwind { -; CHECK-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: amswap.w $a0, $a1, $a2 -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: #APP +; NORMAL-NEXT: amswap_db.w $a0, $a1, $a2 +; NORMAL-NEXT: #NO_APP +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: #APP +; BYPASS-NEXT: amswap_db.w $a0, $a1, $a2 +; BYPASS-NEXT: #NO_APP +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: amswap.w $a0, $a1, $a2 +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: #APP +; BTONOP-NEXT: amswap_db.w $a0, $a1, $a2 +; BTONOP-NEXT: #NO_APP +; BTONOP-NEXT: ret entry: fence acquire call void asm sideeffect " amswap.w $$r4, $$r5, $$r6", ""() @@ -343,13 +722,35 @@ entry: } define void @dbar_acquire_asm_comment_amswap_i32_monotonic() nounwind { -; CHECK-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: amswap.w $a0, $a1, $a2 # atomic -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: #APP +; NORMAL-NEXT: amswap_db.w $a0, $a1, $a2 # atomic +; NORMAL-NEXT: #NO_APP +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: #APP +; BYPASS-NEXT: amswap_db.w $a0, $a1, $a2 # atomic +; BYPASS-NEXT: #NO_APP +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: amswap.w $a0, $a1, $a2 # atomic +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: #APP +; BTONOP-NEXT: amswap_db.w $a0, $a1, $a2 # atomic +; BTONOP-NEXT: #NO_APP +; BTONOP-NEXT: ret entry: fence acquire call void asm sideeffect "amswap.w $$r4, $$r5, $$r6 # atomic", ""() @@ -357,13 +758,35 @@ entry: } define void @dbar_acquire_asm_args_amswap_i32_monotonic() nounwind { -; CHECK-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: amswap.w $a0, $zero, $zero -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: #APP +; NORMAL-NEXT: amswap_db.w $a0, $zero, $zero +; NORMAL-NEXT: #NO_APP +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: #APP +; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero +; BYPASS-NEXT: #NO_APP +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: amswap.w $a0, $zero, $zero +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: #APP +; BTONOP-NEXT: amswap_db.w $a0, $zero, $zero +; BTONOP-NEXT: #NO_APP +; BTONOP-NEXT: ret entry: fence acquire call i32 asm sideeffect "amswap.w $0, $1, $2", "=r,r,r,~{memory}"(i32 0, i32 0) @@ -371,13 +794,35 @@ entry: } define void @dbar_acquire_asm_reg_args_amswap_i32_monotonic() nounwind { -; CHECK-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: #APP -; CHECK-NEXT: amswap_db.w $ra, $zero, $zero -; CHECK-NEXT: #NO_APP -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: #APP +; NORMAL-NEXT: amswap_db.w $ra, $zero, $zero +; NORMAL-NEXT: #NO_APP +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: #APP +; BYPASS-NEXT: amswap_db.w $ra, $zero, $zero +; BYPASS-NEXT: #NO_APP +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: #APP +; NOIASM-NEXT: amswap_db.w $ra, $zero, $zero +; NOIASM-NEXT: #NO_APP +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: #APP +; BTONOP-NEXT: amswap_db.w $ra, $zero, $zero +; BTONOP-NEXT: #NO_APP +; BTONOP-NEXT: ret entry: fence acquire call void asm sideeffect "amswap_db.w $$r1, $0, $1", "r,r,~{memory}"(i32 0, i32 0) @@ -385,25 +830,81 @@ entry: } define void @dbar_acquire_cmpxchg_i32_dbar_release() nounwind { -; CHECK-LABEL: dbar_acquire_cmpxchg_i32_dbar_release: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ori $a0, $zero, 1 -; CHECK-NEXT: .LBB28_1: # %entry -; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ll.w $a1, $zero, 0 -; CHECK-NEXT: bne $a1, $zero, .LBB28_3 -; CHECK-NEXT: # %bb.2: # %entry -; CHECK-NEXT: # in Loop: Header=BB28_1 Depth=1 -; CHECK-NEXT: move $a2, $a0 -; CHECK-NEXT: sc.w $a2, $zero, 0 -; CHECK-NEXT: beq $a2, $zero, .LBB28_1 -; CHECK-NEXT: b .LBB28_4 -; CHECK-NEXT: .LBB28_3: # %entry -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: .LBB28_4: # %entry -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: ret +; NORMAL-LABEL: dbar_acquire_cmpxchg_i32_dbar_release: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: ori $a0, $zero, 1 +; NORMAL-NEXT: .LBB28_1: # %entry +; NORMAL-NEXT: # =>This Inner Loop Header: Depth=1 +; NORMAL-NEXT: ll.w $a1, $zero, 0 +; NORMAL-NEXT: bne $a1, $zero, .LBB28_3 +; NORMAL-NEXT: # %bb.2: # %entry +; NORMAL-NEXT: # in Loop: Header=BB28_1 Depth=1 +; NORMAL-NEXT: move $a2, $a0 +; NORMAL-NEXT: sc.w $a2, $zero, 0 +; NORMAL-NEXT: beq $a2, $zero, .LBB28_1 +; NORMAL-NEXT: b .LBB28_4 +; NORMAL-NEXT: .LBB28_3: # %entry +; NORMAL-NEXT: dbar 20 +; NORMAL-NEXT: .LBB28_4: # %entry +; NORMAL-NEXT: dbar 18 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: dbar_acquire_cmpxchg_i32_dbar_release: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: ori $a0, $zero, 1 +; BYPASS-NEXT: .LBB28_1: # %entry +; BYPASS-NEXT: # =>This Inner Loop Header: Depth=1 +; BYPASS-NEXT: ll.w $a1, $zero, 0 +; BYPASS-NEXT: bne $a1, $zero, .LBB28_3 +; BYPASS-NEXT: # %bb.2: # %entry +; BYPASS-NEXT: # in Loop: Header=BB28_1 Depth=1 +; BYPASS-NEXT: move $a2, $a0 +; BYPASS-NEXT: sc.w $a2, $zero, 0 +; BYPASS-NEXT: beq $a2, $zero, .LBB28_1 +; BYPASS-NEXT: b .LBB28_4 +; BYPASS-NEXT: .LBB28_3: # %entry +; BYPASS-NEXT: .LBB28_4: # %entry +; BYPASS-NEXT: dbar 16 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: dbar_acquire_cmpxchg_i32_dbar_release: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: ori $a0, $zero, 1 +; NOIASM-NEXT: .LBB28_1: # %entry +; NOIASM-NEXT: # =>This Inner Loop Header: Depth=1 +; NOIASM-NEXT: ll.w $a1, $zero, 0 +; NOIASM-NEXT: bne $a1, $zero, .LBB28_3 +; NOIASM-NEXT: # %bb.2: # %entry +; NOIASM-NEXT: # in Loop: Header=BB28_1 Depth=1 +; NOIASM-NEXT: move $a2, $a0 +; NOIASM-NEXT: sc.w $a2, $zero, 0 +; NOIASM-NEXT: beq $a2, $zero, .LBB28_1 +; NOIASM-NEXT: b .LBB28_4 +; NOIASM-NEXT: .LBB28_3: # %entry +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: .LBB28_4: # %entry +; NOIASM-NEXT: dbar 18 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: dbar_acquire_cmpxchg_i32_dbar_release: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: nop +; BTONOP-NEXT: ori $a0, $zero, 1 +; BTONOP-NEXT: .LBB28_1: # %entry +; BTONOP-NEXT: # =>This Inner Loop Header: Depth=1 +; BTONOP-NEXT: ll.w $a1, $zero, 0 +; BTONOP-NEXT: bne $a1, $zero, .LBB28_3 +; BTONOP-NEXT: # %bb.2: # %entry +; BTONOP-NEXT: # in Loop: Header=BB28_1 Depth=1 +; BTONOP-NEXT: move $a2, $a0 +; BTONOP-NEXT: sc.w $a2, $zero, 0 +; BTONOP-NEXT: beq $a2, $zero, .LBB28_1 +; BTONOP-NEXT: b .LBB28_4 +; BTONOP-NEXT: .LBB28_3: # %entry +; BTONOP-NEXT: dbar 20 +; BTONOP-NEXT: .LBB28_4: # %entry +; BTONOP-NEXT: dbar 18 +; BTONOP-NEXT: ret entry: fence acquire cmpxchg ptr null, i32 0, i32 1 acquire acquire @@ -412,17 +913,49 @@ entry: } define void @cfg_dominate(i1 %a) { -; CHECK-LABEL: cfg_dominate: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: andi $a0, $a0, 1 -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: beqz $a0, .LBB29_2 -; CHECK-NEXT: # %bb.1: -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB29_2: -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: ret +; NORMAL-LABEL: cfg_dominate: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: andi $a0, $a0, 1 +; NORMAL-NEXT: dbar 20 +; NORMAL-NEXT: beqz $a0, .LBB29_2 +; NORMAL-NEXT: # %bb.1: +; NORMAL-NEXT: ret +; NORMAL-NEXT: .LBB29_2: +; NORMAL-NEXT: dbar 18 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: cfg_dominate: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: andi $a0, $a0, 1 +; BYPASS-NEXT: dbar 16 +; BYPASS-NEXT: beqz $a0, .LBB29_2 +; BYPASS-NEXT: # %bb.1: +; BYPASS-NEXT: ret +; BYPASS-NEXT: .LBB29_2: +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: cfg_dominate: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: andi $a0, $a0, 1 +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: beqz $a0, .LBB29_2 +; NOIASM-NEXT: # %bb.1: +; NOIASM-NEXT: ret +; NOIASM-NEXT: .LBB29_2: +; NOIASM-NEXT: dbar 18 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: cfg_dominate: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: andi $a0, $a0, 1 +; BTONOP-NEXT: dbar 20 +; BTONOP-NEXT: beqz $a0, .LBB29_2 +; BTONOP-NEXT: # %bb.1: +; BTONOP-NEXT: nop +; BTONOP-NEXT: ret +; BTONOP-NEXT: .LBB29_2: +; BTONOP-NEXT: dbar 18 +; BTONOP-NEXT: ret entry: fence acquire br i1 %a, label %2, label %3 @@ -435,21 +968,65 @@ entry: } define void @cfg_postdominate(i1 %a, i1 %b, i1 %c) { -; CHECK-LABEL: cfg_postdominate: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: andi $a0, $a0, 1 -; CHECK-NEXT: beqz $a0, .LBB30_2 -; CHECK-NEXT: .LBB30_1: -; CHECK-NEXT: andi $a0, $a1, 1 -; CHECK-NEXT: dbar 20 -; CHECK-NEXT: beqz $a0, .LBB30_3 -; CHECK-NEXT: .LBB30_2: -; CHECK-NEXT: andi $a0, $a2, 1 -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: bnez $a0, .LBB30_1 -; CHECK-NEXT: .LBB30_3: -; CHECK-NEXT: dbar 18 -; CHECK-NEXT: ret +; NORMAL-LABEL: cfg_postdominate: +; NORMAL: # %bb.0: # %entry +; NORMAL-NEXT: andi $a0, $a0, 1 +; NORMAL-NEXT: beqz $a0, .LBB30_2 +; NORMAL-NEXT: .LBB30_1: +; NORMAL-NEXT: andi $a0, $a1, 1 +; NORMAL-NEXT: dbar 20 +; NORMAL-NEXT: beqz $a0, .LBB30_3 +; NORMAL-NEXT: .LBB30_2: +; NORMAL-NEXT: andi $a0, $a2, 1 +; NORMAL-NEXT: bnez $a0, .LBB30_1 +; NORMAL-NEXT: .LBB30_3: +; NORMAL-NEXT: dbar 18 +; NORMAL-NEXT: ret +; +; BYPASS-LABEL: cfg_postdominate: +; BYPASS: # %bb.0: # %entry +; BYPASS-NEXT: andi $a0, $a0, 1 +; BYPASS-NEXT: beqz $a0, .LBB30_2 +; BYPASS-NEXT: .LBB30_1: +; BYPASS-NEXT: andi $a0, $a1, 1 +; BYPASS-NEXT: beqz $a0, .LBB30_3 +; BYPASS-NEXT: .LBB30_2: +; BYPASS-NEXT: andi $a0, $a2, 1 +; BYPASS-NEXT: bnez $a0, .LBB30_1 +; BYPASS-NEXT: .LBB30_3: +; BYPASS-NEXT: dbar 16 +; BYPASS-NEXT: ret +; +; NOIASM-LABEL: cfg_postdominate: +; NOIASM: # %bb.0: # %entry +; NOIASM-NEXT: andi $a0, $a0, 1 +; NOIASM-NEXT: beqz $a0, .LBB30_2 +; NOIASM-NEXT: .LBB30_1: +; NOIASM-NEXT: andi $a0, $a1, 1 +; NOIASM-NEXT: dbar 20 +; NOIASM-NEXT: beqz $a0, .LBB30_3 +; NOIASM-NEXT: .LBB30_2: +; NOIASM-NEXT: andi $a0, $a2, 1 +; NOIASM-NEXT: bnez $a0, .LBB30_1 +; NOIASM-NEXT: .LBB30_3: +; NOIASM-NEXT: dbar 18 +; NOIASM-NEXT: ret +; +; BTONOP-LABEL: cfg_postdominate: +; BTONOP: # %bb.0: # %entry +; BTONOP-NEXT: andi $a0, $a0, 1 +; BTONOP-NEXT: beqz $a0, .LBB30_2 +; BTONOP-NEXT: .LBB30_1: +; BTONOP-NEXT: andi $a0, $a1, 1 +; BTONOP-NEXT: dbar 20 +; BTONOP-NEXT: beqz $a0, .LBB30_3 +; BTONOP-NEXT: .LBB30_2: +; BTONOP-NEXT: andi $a0, $a2, 1 +; BTONOP-NEXT: nop +; BTONOP-NEXT: bnez $a0, .LBB30_1 +; BTONOP-NEXT: .LBB30_3: +; BTONOP-NEXT: dbar 18 +; BTONOP-NEXT: ret entry: br i1 %a, label %2, label %3 2: diff --git a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll index 4a77cccb99a2e..d3bd7dfc24b4b 100644 --- a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll +++ b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll @@ -180,6 +180,9 @@ ; LAXX-NEXT: Stack Frame Layout Analysis ; LAXX-NEXT: LoongArch pseudo instruction expansion pass ; LAXX-NEXT: LoongArch atomic pseudo instruction expansion pass +; LAXX-NEXT: MachineDominator Tree Construction +; LAXX-NEXT: MachinePostDominator Tree Construction +; LAXX-NEXT: LoongArch Memory Barrier Optimisation pass ; LAXX-NEXT: Lazy Machine Block Frequency Analysis ; LAXX-NEXT: Machine Optimization Remark Emitter ; LAXX-NEXT: LoongArch Assembly Printer diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn index 810811f7d147c..bec44b7c1d140 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn @@ -49,6 +49,7 @@ static_library("LLVMLoongArchCodeGen") { "LoongArchISelLowering.cpp", "LoongArchInstrInfo.cpp", "LoongArchMCInstLower.cpp", + "LoongArchMemoryBarrierOpt.cpp", "LoongArchMergeBaseOffset.cpp", "LoongArchOptWInstrs.cpp", "LoongArchRegisterInfo.cpp", >From db862f3d5da183dab469086b3ff9ca9fc3ba0a2e Mon Sep 17 00:00:00 2001 From: WANG Rui <[email protected]> Date: Thu, 27 Aug 2026 16:10:44 +0800 Subject: [PATCH 2/4] Refactor LoongArch AMO handling with macros --- .../LoongArch/LoongArchMemoryBarrierOpt.cpp | 171 ++++-------------- 1 file changed, 39 insertions(+), 132 deletions(-) diff --git a/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp b/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp index 1f29cdc8638c1..28be20ff42a2f 100644 --- a/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp @@ -48,6 +48,34 @@ static cl::opt<bool> ReplaceEliminatedMBToNop( namespace { +#define AMO_CASES \ + CASE(AMSWAP, B) \ + CASE(AMSWAP, H) \ + CASE(AMSWAP, W) \ + CASE(AMSWAP, D) \ + CASE(AMADD, B) \ + CASE(AMADD, H) \ + CASE(AMADD, W) \ + CASE(AMADD, D) \ + CASE(AMAND, W) \ + CASE(AMAND, D) \ + CASE(AMOR, W) \ + CASE(AMOR, D) \ + CASE(AMXOR, W) \ + CASE(AMXOR, D) \ + CASE(AMMAX, W) \ + CASE(AMMAX, D) \ + CASE(AMMAX, WU) \ + CASE(AMMAX, DU) \ + CASE(AMMIN, W) \ + CASE(AMMIN, D) \ + CASE(AMMIN, WU) \ + CASE(AMMIN, DU) \ + CASE(AMCAS, B) \ + CASE(AMCAS, H) \ + CASE(AMCAS, W) \ + CASE(AMCAS, D) + static std::optional<std::pair<StringRef, StringRef>> parseMB(StringRef Asm) { auto T1 = llvm::getToken(Asm); if (!T1.first.equals_insensitive("dbar")) @@ -72,60 +100,11 @@ isAsmMB(const MachineInstr &MI) { } static StringRef getAMDB(StringRef Name) { - return StringSwitch<StringRef>(Name.lower()) - .Case("amswap.b", "amswap_db.b") - .Case("amswap.h", "amswap_db.h") - .Case("amswap.w", "amswap_db.w") - .Case("amswap.d", "amswap_db.d") - .Case("amswap_db.b", "amswap_db.b") - .Case("amswap_db.h", "amswap_db.h") - .Case("amswap_db.w", "amswap_db.w") - .Case("amswap_db.d", "amswap_db.d") - .Case("amadd.b", "amadd_db.b") - .Case("amadd.h", "amadd_db.h") - .Case("amadd.w", "amadd_db.w") - .Case("amadd.d", "amadd_db.d") - .Case("amadd_db.b", "amadd_db.b") - .Case("amadd_db.h", "amadd_db.h") - .Case("amadd_db.w", "amadd_db.w") - .Case("amadd_db.d", "amadd_db.d") - .Case("amand.w", "amand_db.w") - .Case("amand.d", "amand_db.d") - .Case("amand_db.w", "amand_db.w") - .Case("amand_db.d", "amand_db.d") - .Case("amor.w", "amor_db.w") - .Case("amor.d", "amor_db.d") - .Case("amor_db.w", "amor_db.w") - .Case("amor_db.d", "amor_db.d") - .Case("amxor.w", "amxor_db.w") - .Case("amxor.d", "amxor_db.d") - .Case("amxor_db.w", "amxor_db.w") - .Case("amxor_db.d", "amxor_db.d") - .Case("ammax.w", "ammax_db.w") - .Case("ammax.d", "ammax_db.d") - .Case("ammax_db.w", "ammax_db.w") - .Case("ammax_db.d", "ammax_db.d") - .Case("ammin.w", "ammin_db.w") - .Case("ammin.d", "ammin_db.d") - .Case("ammin_db.w", "ammin_db.w") - .Case("ammin_db.d", "ammin_db.d") - .Case("ammax.wu", "ammax_db.wu") - .Case("ammax.du", "ammax_db.du") - .Case("ammax_db.wu", "ammax_db.wu") - .Case("ammax_db.du", "ammax_db.du") - .Case("ammin.wu", "ammin_db.wu") - .Case("ammin.du", "ammin_db.du") - .Case("ammin_db.wu", "ammin_db.wu") - .Case("ammin_db.du", "ammin_db.du") - .Case("amcas.b", "amcas_db.b") - .Case("amcas.h", "amcas_db.h") - .Case("amcas.w", "amcas_db.w") - .Case("amcas.d", "amcas_db.d") - .Case("amcas_db.b", "amcas_db.b") - .Case("amcas_db.h", "amcas_db.h") - .Case("amcas_db.w", "amcas_db.w") - .Case("amcas_db.d", "amcas_db.d") - .Default({}); +#define CASE(Name, Suffix) \ + .Case(#Name "." #Suffix, #Name "_DB." #Suffix) \ + .Case(#Name "_DB." #Suffix, #Name "_DB." #Suffix) + return StringSwitch<StringRef>(Name.upper()) AMO_CASES.Default({}); +#undef CASE } static std::optional<std::pair<StringRef, StringRef>> parseAM(StringRef Asm) { @@ -187,88 +166,16 @@ static bool isSC(const MachineInstr &MI) { } static std::optional<unsigned> isAM(const MachineInstr &MI) { +#define CASE(Name, Suffix) \ + case LoongArch::Name##_##Suffix: \ + case LoongArch::Name##__DB_##Suffix: \ + return LoongArch::Name##__DB_##Suffix; switch (MI.getOpcode()) { - case LoongArch::AMSWAP_B: - case LoongArch::AMSWAP__DB_B: - return LoongArch::AMSWAP__DB_B; - case LoongArch::AMSWAP_H: - case LoongArch::AMSWAP__DB_H: - return LoongArch::AMSWAP__DB_H; - case LoongArch::AMSWAP_W: - case LoongArch::AMSWAP__DB_W: - return LoongArch::AMSWAP__DB_W; - case LoongArch::AMSWAP_D: - case LoongArch::AMSWAP__DB_D: - return LoongArch::AMSWAP__DB_D; - case LoongArch::AMADD_B: - case LoongArch::AMADD__DB_B: - return LoongArch::AMADD__DB_B; - case LoongArch::AMADD_H: - case LoongArch::AMADD__DB_H: - return LoongArch::AMADD__DB_H; - case LoongArch::AMADD_W: - case LoongArch::AMADD__DB_W: - return LoongArch::AMADD__DB_W; - case LoongArch::AMADD_D: - case LoongArch::AMADD__DB_D: - return LoongArch::AMADD__DB_D; - case LoongArch::AMAND_W: - case LoongArch::AMAND__DB_W: - return LoongArch::AMAND__DB_W; - case LoongArch::AMAND_D: - case LoongArch::AMAND__DB_D: - return LoongArch::AMAND__DB_D; - case LoongArch::AMOR_W: - case LoongArch::AMOR__DB_W: - return LoongArch::AMOR__DB_W; - case LoongArch::AMOR_D: - case LoongArch::AMOR__DB_D: - return LoongArch::AMOR__DB_D; - case LoongArch::AMXOR_W: - case LoongArch::AMXOR__DB_W: - return LoongArch::AMXOR__DB_W; - case LoongArch::AMXOR_D: - case LoongArch::AMXOR__DB_D: - return LoongArch::AMXOR__DB_D; - case LoongArch::AMMAX_W: - case LoongArch::AMMAX__DB_W: - return LoongArch::AMMAX__DB_W; - case LoongArch::AMMAX_D: - case LoongArch::AMMAX__DB_D: - return LoongArch::AMMAX__DB_D; - case LoongArch::AMMIN_W: - case LoongArch::AMMIN__DB_W: - return LoongArch::AMMIN__DB_W; - case LoongArch::AMMIN_D: - case LoongArch::AMMIN__DB_D: - return LoongArch::AMMIN__DB_D; - case LoongArch::AMMAX_WU: - case LoongArch::AMMAX__DB_WU: - return LoongArch::AMMAX__DB_WU; - case LoongArch::AMMAX_DU: - case LoongArch::AMMAX__DB_DU: - return LoongArch::AMMAX__DB_DU; - case LoongArch::AMMIN_WU: - case LoongArch::AMMIN__DB_WU: - return LoongArch::AMMIN__DB_WU; - case LoongArch::AMMIN_DU: - case LoongArch::AMMIN__DB_DU: - return LoongArch::AMMIN__DB_DU; - case LoongArch::AMCAS_B: - case LoongArch::AMCAS__DB_B: - return LoongArch::AMCAS__DB_B; - case LoongArch::AMCAS_H: - case LoongArch::AMCAS__DB_H: - return LoongArch::AMCAS__DB_H; - case LoongArch::AMCAS_W: - case LoongArch::AMCAS__DB_W: - return LoongArch::AMCAS__DB_W; - case LoongArch::AMCAS_D: - case LoongArch::AMCAS__DB_D: - return LoongArch::AMCAS__DB_D; + AMO_CASES default: return std::nullopt; } +#undef CASE } static bool isSafeToSkip(const MachineInstr &MI) { >From f3ea2a9aa5ca7fdca207dab53cc4e0fd1f71ce64 Mon Sep 17 00:00:00 2001 From: WANG Rui <[email protected]> Date: Tue, 1 Sep 2026 12:08:56 +0800 Subject: [PATCH 3/4] Add an option to merge AMOs with DBARs into AMO_DB --- .../LoongArch/LoongArchMemoryBarrierOpt.cpp | 10 +- .../CodeGen/LoongArch/memory-barrier-opt.ll | 193 ++++++++++++++++++ 2 files changed, 202 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp b/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp index 28be20ff42a2f..61c5f91e2514d 100644 --- a/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp @@ -36,6 +36,11 @@ static cl::opt<bool> RequireNoPathBypass( cl::desc("Optimize only when no paths bypass either memory barrier"), cl::init(true), cl::Hidden); +static cl::opt<bool> MergeAMOWithMB( + "loongarch-merge-amo-with-dbar", + cl::desc("Merge AMOs with DBARs into AMO_DB during optimization"), + cl::init(true), cl::Hidden); + static cl::opt<bool> DisableInlineAsm( "loongarch-disable-inline-asm-barrier-opt", cl::desc("Disable optimization of memory barriers in InlineAsm"), @@ -101,7 +106,7 @@ isAsmMB(const MachineInstr &MI) { static StringRef getAMDB(StringRef Name) { #define CASE(Name, Suffix) \ - .Case(#Name "." #Suffix, #Name "_DB." #Suffix) \ + .Case(#Name "." #Suffix, MergeAMOWithMB ? #Name "_DB." #Suffix : "") \ .Case(#Name "_DB." #Suffix, #Name "_DB." #Suffix) return StringSwitch<StringRef>(Name.upper()) AMO_CASES.Default({}); #undef CASE @@ -168,6 +173,9 @@ static bool isSC(const MachineInstr &MI) { static std::optional<unsigned> isAM(const MachineInstr &MI) { #define CASE(Name, Suffix) \ case LoongArch::Name##_##Suffix: \ + if (!MergeAMOWithMB) \ + return std::nullopt; \ + [[fallthrough]]; \ case LoongArch::Name##__DB_##Suffix: \ return LoongArch::Name##__DB_##Suffix; switch (MI.getOpcode()) { diff --git a/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll b/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll index 3382e28c14891..33b6bc42a80f3 100644 --- a/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll +++ b/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll @@ -2,6 +2,8 @@ ; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefixes=CHECK,NORMAL ; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-require-no-path-bypass=false < %s \ ; RUN: | FileCheck %s --check-prefixes=CHECK,BYPASS +; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-merge-amo-with-dbar=false < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,NOMAMO ; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-disable-inline-asm-barrier-opt=true < %s \ ; RUN: | FileCheck %s --check-prefixes=CHECK,NOIASM ; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-replace-eliminated-dbar-to-nop=true < %s \ @@ -20,6 +22,11 @@ define void @dbar_acquire_acquire() nounwind { ; BYPASS-NEXT: dbar 20 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -47,6 +54,11 @@ define void @dbar_acquire_release() nounwind { ; BYPASS-NEXT: dbar 16 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_release: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 16 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_release: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 16 @@ -76,6 +88,12 @@ define i32 @dbar_acquire_add_acquire(i32 %a) nounwind { ; BYPASS-NEXT: dbar 20 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_add_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: addi.w $a0, $a0, 1 +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_add_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: addi.w $a0, $a0, 1 @@ -106,6 +124,11 @@ define void @dbar_release_acquire() nounwind { ; BYPASS-NEXT: dbar 16 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_release_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 16 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_release_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 16 @@ -133,6 +156,11 @@ define void @dbar_seq_cst_acq_rel_seq_cst() nounwind { ; BYPASS-NEXT: dbar 16 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_seq_cst_acq_rel_seq_cst: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 16 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_seq_cst_acq_rel_seq_cst: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 16 @@ -162,6 +190,11 @@ define void @dbar_acquire_asm_dbar_acquire() nounwind { ; BYPASS-NEXT: dbar 20 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_dbar_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_dbar_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -195,6 +228,11 @@ define void @dbar_acquire_asm_space_dbar_acquire() nounwind { ; BYPASS-NEXT: dbar 20 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_space_dbar_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_space_dbar_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -228,6 +266,11 @@ define void @dbar_acquire_asm_comment_dbar_acquire() nounwind { ; BYPASS-NEXT: dbar 20 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_comment_dbar_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_comment_dbar_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -261,6 +304,11 @@ define void @dbar_acquire_asm_arg0_dbar_acquire() nounwind { ; BYPASS-NEXT: dbar 16 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_arg0_dbar_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 16 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_arg0_dbar_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -294,6 +342,11 @@ define void @dbar_acquire_asm_arg1_dbar_acquire() nounwind { ; BYPASS-NEXT: dbar 16 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_arg1_dbar_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 16 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_arg1_dbar_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -327,6 +380,11 @@ define void @dbar_completion_acquire() nounwind { ; BYPASS-NEXT: dbar 0 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_completion_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 0 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_completion_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 0 @@ -354,6 +412,11 @@ define void @dbar_release_completion() nounwind { ; BYPASS-NEXT: dbar 0 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_release_completion: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 0 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_release_completion: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 0 @@ -383,6 +446,12 @@ define void @dbar_release_amswap_i8_acquire() nounwind { ; BYPASS-NEXT: amand_db.w $a1, $a0, $zero ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_release_amswap_i8_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: addi.w $a0, $zero, -256 +; NOMAMO-NEXT: amand_db.w $a1, $a0, $zero +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_release_amswap_i8_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: addi.w $a0, $zero, -256 @@ -412,6 +481,11 @@ define void @dbar_release_amswap_i32_acquire() nounwind { ; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_release_amswap_i32_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: amswap_db.w $a0, $zero, $zero +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_release_amswap_i32_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero @@ -439,6 +513,12 @@ define void @dbar_release_amswap_i32_monotonic() nounwind { ; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_release_amswap_i32_monotonic: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 18 +; NOMAMO-NEXT: amswap.w $a0, $zero, $zero +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_release_amswap_i32_monotonic: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero @@ -480,6 +560,12 @@ define void @amswap_i8_release_dbar_acquire() nounwind { ; BYPASS-NEXT: amand_db.w $a1, $a0, $zero ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: amswap_i8_release_dbar_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: addi.w $a0, $zero, -256 +; NOMAMO-NEXT: amand_db.w $a1, $a0, $zero +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: amswap_i8_release_dbar_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: addi.w $a0, $zero, -256 @@ -509,6 +595,11 @@ define void @amswap_i32_release_dbar_acquire() nounwind { ; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: amswap_i32_release_dbar_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: amswap_db.w $a0, $zero, $zero +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: amswap_i32_release_dbar_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero @@ -550,6 +641,12 @@ define void @dbar_release_amswap_i8_release_dbar_acquire() nounwind { ; BYPASS-NEXT: amand_db.w $a1, $a0, $zero ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_release_amswap_i8_release_dbar_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: addi.w $a0, $zero, -256 +; NOMAMO-NEXT: amand_db.w $a1, $a0, $zero +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_release_amswap_i8_release_dbar_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: addi.w $a0, $zero, -256 @@ -581,6 +678,11 @@ define void @dbar_acquire_amswap_i32_release_dbar_acquire() nounwind { ; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: amswap_db.w $a0, $zero, $zero +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero @@ -628,6 +730,13 @@ define void @dbar_acquire_asm_amswap_i32_release() nounwind { ; BYPASS-NEXT: #NO_APP ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_amswap_i32_release: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: #APP +; NOMAMO-NEXT: amswap_db.w $a0, $a1, $a2 +; NOMAMO-NEXT: #NO_APP +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_amswap_i32_release: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -664,6 +773,14 @@ define void @dbar_acquire_asm_amswap_i32_monotonic() nounwind { ; BYPASS-NEXT: #NO_APP ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_amswap_i32_monotonic: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: #APP +; NOMAMO-NEXT: amswap.w $a0, $a1, $a2 +; NOMAMO-NEXT: #NO_APP +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_amswap_i32_monotonic: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -700,6 +817,14 @@ define void @dbar_acquire_asm_space_amswap_i32_monotonic() nounwind { ; BYPASS-NEXT: #NO_APP ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: #APP +; NOMAMO-NEXT: amswap.w $a0, $a1, $a2 +; NOMAMO-NEXT: #NO_APP +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -736,6 +861,14 @@ define void @dbar_acquire_asm_comment_amswap_i32_monotonic() nounwind { ; BYPASS-NEXT: #NO_APP ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: #APP +; NOMAMO-NEXT: amswap.w $a0, $a1, $a2 # atomic +; NOMAMO-NEXT: #NO_APP +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -772,6 +905,14 @@ define void @dbar_acquire_asm_args_amswap_i32_monotonic() nounwind { ; BYPASS-NEXT: #NO_APP ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: #APP +; NOMAMO-NEXT: amswap.w $a0, $zero, $zero +; NOMAMO-NEXT: #NO_APP +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -808,6 +949,13 @@ define void @dbar_acquire_asm_reg_args_amswap_i32_monotonic() nounwind { ; BYPASS-NEXT: #NO_APP ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: #APP +; NOMAMO-NEXT: amswap_db.w $ra, $zero, $zero +; NOMAMO-NEXT: #NO_APP +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: dbar 20 @@ -867,6 +1015,25 @@ define void @dbar_acquire_cmpxchg_i32_dbar_release() nounwind { ; BYPASS-NEXT: dbar 16 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: dbar_acquire_cmpxchg_i32_dbar_release: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: ori $a0, $zero, 1 +; NOMAMO-NEXT: .LBB28_1: # %entry +; NOMAMO-NEXT: # =>This Inner Loop Header: Depth=1 +; NOMAMO-NEXT: ll.w $a1, $zero, 0 +; NOMAMO-NEXT: bne $a1, $zero, .LBB28_3 +; NOMAMO-NEXT: # %bb.2: # %entry +; NOMAMO-NEXT: # in Loop: Header=BB28_1 Depth=1 +; NOMAMO-NEXT: move $a2, $a0 +; NOMAMO-NEXT: sc.w $a2, $zero, 0 +; NOMAMO-NEXT: beq $a2, $zero, .LBB28_1 +; NOMAMO-NEXT: b .LBB28_4 +; NOMAMO-NEXT: .LBB28_3: # %entry +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: .LBB28_4: # %entry +; NOMAMO-NEXT: dbar 18 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: dbar_acquire_cmpxchg_i32_dbar_release: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: ori $a0, $zero, 1 @@ -934,6 +1101,17 @@ define void @cfg_dominate(i1 %a) { ; BYPASS-NEXT: .LBB29_2: ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: cfg_dominate: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: andi $a0, $a0, 1 +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: beqz $a0, .LBB29_2 +; NOMAMO-NEXT: # %bb.1: +; NOMAMO-NEXT: ret +; NOMAMO-NEXT: .LBB29_2: +; NOMAMO-NEXT: dbar 18 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: cfg_dominate: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: andi $a0, $a0, 1 @@ -997,6 +1175,21 @@ define void @cfg_postdominate(i1 %a, i1 %b, i1 %c) { ; BYPASS-NEXT: dbar 16 ; BYPASS-NEXT: ret ; +; NOMAMO-LABEL: cfg_postdominate: +; NOMAMO: # %bb.0: # %entry +; NOMAMO-NEXT: andi $a0, $a0, 1 +; NOMAMO-NEXT: beqz $a0, .LBB30_2 +; NOMAMO-NEXT: .LBB30_1: +; NOMAMO-NEXT: andi $a0, $a1, 1 +; NOMAMO-NEXT: dbar 20 +; NOMAMO-NEXT: beqz $a0, .LBB30_3 +; NOMAMO-NEXT: .LBB30_2: +; NOMAMO-NEXT: andi $a0, $a2, 1 +; NOMAMO-NEXT: bnez $a0, .LBB30_1 +; NOMAMO-NEXT: .LBB30_3: +; NOMAMO-NEXT: dbar 18 +; NOMAMO-NEXT: ret +; ; NOIASM-LABEL: cfg_postdominate: ; NOIASM: # %bb.0: # %entry ; NOIASM-NEXT: andi $a0, $a0, 1 >From 46e5b0173885dc560a8e2e6bd8fbf1ba094c5df3 Mon Sep 17 00:00:00 2001 From: WANG Rui <[email protected]> Date: Wed, 9 Sep 2026 21:53:24 +0800 Subject: [PATCH 4/4] Fix assertion failure --- llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp b/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp index 61c5f91e2514d..c93629453058d 100644 --- a/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp @@ -272,7 +272,8 @@ struct InstBarrier { assert(MO.isImm() && "Unexpected operand type!"); const InlineAsm::Flag F(MO.getImm()); if (N == Off) { - assert(F.isImmKind() && "Unexpected flag kind!"); + if (!F.isImmKind()) + return std::nullopt; HintOff = AsmDescOp + 1; Hint = MI.getOperand(HintOff).getImm(); return std::pair(Hint, HintOff); _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
