https://github.com/ritter-x2a created https://github.com/llvm/llvm-project/pull/231854
Determining which memory accesses are ordered by a fence is already useful in the middle-end; this PR extracts the code to do that from SIMemoryLegalizer, so that it can be used in middle-end transformations. So far, unexpected suffix values for the `amdgpu-synchronize-as` were reported when they were encountered in SIMemoryLegalizer. This PR keeps that and doesn't report them in the middle end, since that would lead to a lot of repeated diagnostics for them. Disclaimer: Claude gave review comments and suggestions that I addressed during development. Preparation for ROCM-29879. Assisted-by: AI >From a9246c0458c212a27c5503ef966478870e2b6cfc Mon Sep 17 00:00:00 2001 From: Fabian Ritter <[email protected]> Date: Wed, 14 Oct 2026 06:23:47 -0400 Subject: [PATCH] [AMDGPU][NFC] Extract SIAtomicAddrSpace and fence MMRA parsing from SIMemoryLegalizer Determining which memory accesses are ordered by a fence is already useful in the middle-end; this PR extracts the code to do that from SIMemoryLegalizer, so that it can be used in middle-end transformations. So far, unexpected suffix values for the `amdgpu-synchronize-as` were reported when they were encountered in SIMemoryLegalizer. This PR keeps that and doesn't report them in the middle end, since that would lead to a lot of repeated diagnostics for them. Disclaimer: Claude gave review comments and suggestions that I addressed during development. Preparation for ROCM-29879. Assisted-by: AI --- llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 65 ++++++++++++++ llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 45 ++++++++++ llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 93 +++----------------- 3 files changed, 120 insertions(+), 83 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp index 8a4887afd3e00a..837a5f7d7cc55c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp @@ -9,6 +9,7 @@ #include "AMDGPUMemoryUtils.h" #include "Utils/AMDGPUBaseInfo.h" #include "llvm/ADT/SetOperations.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/MemorySSA.h" @@ -17,6 +18,7 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MemoryModelRelaxationAnnotations.h" #include "llvm/IR/ReplaceConstant.h" #include "llvm/Support/AMDGPUAddrSpace.h" #include "llvm/Support/AtomicOrdering.h" @@ -478,4 +480,67 @@ bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA, return false; } +SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) { + if (AS == AMDGPUAS::FLAT_ADDRESS) + return SIAtomicAddrSpace::FLAT; + if (AS == AMDGPUAS::GLOBAL_ADDRESS) + return SIAtomicAddrSpace::GLOBAL; + if (AS == AMDGPUAS::LOCAL_ADDRESS) + return SIAtomicAddrSpace::LDS; + if (AS == AMDGPUAS::PRIVATE_ADDRESS) + return SIAtomicAddrSpace::SCRATCH; + if (AS == AMDGPUAS::REGION_ADDRESS) + return SIAtomicAddrSpace::GDS; + if (AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE || + AS == AMDGPUAS::BUFFER_STRIDED_POINTER) + return SIAtomicAddrSpace::GLOBAL; + + // Constant addrspaces map to OTHER because there is no point in atomic + // operations in them, since their memory does not change during kernel + // execution. + return SIAtomicAddrSpace::OTHER; +} + +static const StringMap<SIAtomicAddrSpace> ASNames = {{ + {"global", SIAtomicAddrSpace::GLOBAL}, + {"local", SIAtomicAddrSpace::LDS}, +}}; + +std::optional<SIAtomicAddrSpace> +getSynchronizeAddrSpace(const MMRAMetadata &MMRA, + function_ref<void(StringRef)> Diag) { + static constexpr StringLiteral FenceASPrefix = "amdgpu-synchronize-as"; + + if (!MMRA) + return std::nullopt; + + SIAtomicAddrSpace Result = SIAtomicAddrSpace::NONE; + for (const auto &[Prefix, Suffix] : MMRA) { + if (Prefix != FenceASPrefix) + continue; + + if (auto It = ASNames.find(Suffix); It != ASNames.end()) { + Result |= It->second; + } else { + SmallString<128> Str; + raw_svector_ostream OS(Str); + OS << "unknown address space '" << Suffix << "'; expected one of "; + ListSeparator LS; + for (const auto &[Name, Val] : ASNames) + OS << LS << '\'' << Name << '\''; + Diag(Str); + } + } + + if (Result == SIAtomicAddrSpace::NONE) + return std::nullopt; + + return Result; +} + +std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpace(const Instruction &I) { + const auto MMRA = MMRAMetadata(I); + return getSynchronizeAddrSpace(MMRA, [](StringRef) {}); +} + } // end namespace llvm::AMDGPU diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h index 95f8c91a4dd939..e4bc43de91da80 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h @@ -10,9 +10,12 @@ #define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUMEMORYUTILS_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include <optional> namespace llvm { @@ -20,9 +23,11 @@ struct Align; class AAResults; class DataLayout; class GlobalVariable; +class Instruction; class LoadInst; class MemoryDef; class MemorySSA; +class MMRAMetadata; class Value; class Function; class CallGraph; @@ -113,6 +118,46 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA); bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA, AAResults *AA); +/// The distinct address spaces supported by the AMDGPU target for +/// atomic memory operation. Can be ORed together. +enum class SIAtomicAddrSpace { + NONE = 0u, + GLOBAL = 1u << 0, + LDS = 1u << 1, + SCRATCH = 1u << 2, + GDS = 1u << 3, + OTHER = 1u << 4, + + /// The address spaces that can be accessed by a FLAT instruction. + FLAT = GLOBAL | LDS | SCRATCH, + + /// The address spaces that support atomic instructions. + ATOMIC = GLOBAL | LDS | SCRATCH | GDS, + + /// All address spaces. + ALL = GLOBAL | LDS | SCRATCH | GDS | OTHER, + + LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL) +}; + +/// \return Return a bit set of the address spaces accessed by \p AS. +SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS); + +/// Reads \p I's MMRAs to parse the "amdgpu-synchronize-as" MMRA. +/// If this tag isn't present, or if it has no meaningful values, returns +/// \p none, otherwise returns the address spaces specified by the MD. +/// Does not diagnose unexpected MMRA tag suffixes. +std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpace(const Instruction &I); + +/// Parses the values for the "amdgpu-synchronize-as" MMRA. +/// If this tag isn't present, or if it has no meaningful values, returns +/// \p none, otherwise returns the address spaces specified by the MD. +/// If an unexpected suffix for the MMRA tag is found, \p Diag is called with +/// an error message. +std::optional<SIAtomicAddrSpace> +getSynchronizeAddrSpace(const MMRAMetadata &MMRA, + function_ref<void(StringRef)> Diag); + } // end namespace AMDGPU } // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp index 10804d65b82b0f..518956fdeb36ca 100644 --- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp +++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp @@ -15,6 +15,7 @@ #include "AMDGPU.h" #include "AMDGPUMachineModuleInfo.h" +#include "AMDGPUMemoryUtils.h" #include "GCNSubtarget.h" #include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/StringExtras.h" @@ -24,7 +25,6 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/MemoryModelRelaxationAnnotations.h" #include "llvm/IR/PassManager.h" -#include "llvm/Support/AMDGPUAddrSpace.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/Debug.h" #include "llvm/TargetParser/AMDGPUTargetParser.h" @@ -69,28 +69,6 @@ enum class SIAtomicScope { SYSTEM }; -/// The distinct address spaces supported by the AMDGPU target for -/// atomic memory operation. Can be ORed together. -enum class SIAtomicAddrSpace { - NONE = 0u, - GLOBAL = 1u << 0, - LDS = 1u << 1, - SCRATCH = 1u << 2, - GDS = 1u << 3, - OTHER = 1u << 4, - - /// The address spaces that can be accessed by a FLAT instruction. - FLAT = GLOBAL | LDS | SCRATCH, - - /// The address spaces that support atomic instructions. - ATOMIC = GLOBAL | LDS | SCRATCH | GDS, - - /// All address spaces. - ALL = GLOBAL | LDS | SCRATCH | GDS | OTHER, - - LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL) -}; - #ifndef NDEBUG static StringRef toString(SIAtomicScope S) { switch (S) { @@ -292,9 +270,6 @@ class SIMemOpAccess final { std::optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>> toSIAtomicScope(SyncScope::ID SSID, SIAtomicAddrSpace InstrAddrSpace) const; - /// \return Return a bit set of the address spaces accessed by \p AS. - SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) const; - /// \returns Info constructed from \p MI, which has at least machine memory /// operand. std::optional<SIMemOpInfo> @@ -687,50 +662,20 @@ class SIMemoryLegalizerLegacy final : public MachineFunctionPass { bool runOnMachineFunction(MachineFunction &MF) override; }; -static const StringMap<SIAtomicAddrSpace> ASNames = {{ - {"global", SIAtomicAddrSpace::GLOBAL}, - {"local", SIAtomicAddrSpace::LDS}, -}}; - -void diagnoseUnknownMMRAASName(const MachineInstr &MI, StringRef AS) { - const MachineFunction *MF = MI.getMF(); - const Function &Fn = MF->getFunction(); - SmallString<128> Str; - raw_svector_ostream OS(Str); - OS << "unknown address space '" << AS << "'; expected one of "; - ListSeparator LS; - for (const auto &[Name, Val] : ASNames) - OS << LS << '\'' << Name << '\''; - Fn.getContext().diagnose( - DiagnosticInfoUnsupported(Fn, Str.str(), MI.getDebugLoc(), DS_Warning)); -} - /// Reads \p MI's MMRAs to parse the "amdgpu-synchronize-as" MMRA. /// If this tag isn't present, or if it has no meaningful values, returns /// \p none, otherwise returns the address spaces specified by the MD. +/// Issues a diagnostic if an unexpected suffix is observed for the MMRA tag. static std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpaceMD(const MachineInstr &MI) { - static constexpr StringLiteral FenceASPrefix = "amdgpu-synchronize-as"; - - auto MMRA = MMRAMetadata(MI.getMMRAMetadata()); - if (!MMRA) - return std::nullopt; - - SIAtomicAddrSpace Result = SIAtomicAddrSpace::NONE; - for (const auto &[Prefix, Suffix] : MMRA) { - if (Prefix != FenceASPrefix) - continue; - - if (auto It = ASNames.find(Suffix); It != ASNames.end()) - Result |= It->second; - else - diagnoseUnknownMMRAASName(MI, Suffix); - } - - if (Result == SIAtomicAddrSpace::NONE) - return std::nullopt; - - return Result; + const auto MMRA = MMRAMetadata(MI.getMMRAMetadata()); + auto Diag = [&MI](StringRef S) { + const MachineFunction *MF = MI.getMF(); + const Function &Fn = MF->getFunction(); + Fn.getContext().diagnose( + DiagnosticInfoUnsupported(Fn, S, MI.getDebugLoc(), DS_Warning)); + }; + return getSynchronizeAddrSpace(MMRA, Diag); } static void diagnoseUnknownAVMetadata(const MachineInstr &MI, @@ -806,24 +751,6 @@ SIMemOpAccess::toSIAtomicScope(SyncScope::ID SSID, return std::nullopt; } -SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(unsigned AS) const { - if (AS == AMDGPUAS::FLAT_ADDRESS) - return SIAtomicAddrSpace::FLAT; - if (AS == AMDGPUAS::GLOBAL_ADDRESS) - return SIAtomicAddrSpace::GLOBAL; - if (AS == AMDGPUAS::LOCAL_ADDRESS) - return SIAtomicAddrSpace::LDS; - if (AS == AMDGPUAS::PRIVATE_ADDRESS) - return SIAtomicAddrSpace::SCRATCH; - if (AS == AMDGPUAS::REGION_ADDRESS) - return SIAtomicAddrSpace::GDS; - if (AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE || - AS == AMDGPUAS::BUFFER_STRIDED_POINTER) - return SIAtomicAddrSpace::GLOBAL; - - return SIAtomicAddrSpace::OTHER; -} - SIMemOpAccess::SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI_, const GCNSubtarget &ST) : MMI(&MMI_), ST(ST) {} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
