https://github.com/harrisonGPU created https://github.com/llvm/llvm-project/pull/219906
<sub>Stack created with <a href="https://github.com/github/gh-stack";>GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback";>Give Feedback 💬</a></sub> >From 13787e6cffa73ac8098eee2d6eefa605b546fd5f Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Mon, 31 Aug 2026 12:55:56 +0800 Subject: [PATCH] [CodeGen][AMDGPU] Allow elementwise atomic load/store at element alignment --- llvm/include/llvm/CodeGen/TargetLowering.h | 13 +- llvm/lib/CodeGen/AtomicExpandPass.cpp | 41 +++++- .../SelectionDAG/SelectionDAGBuilder.cpp | 10 +- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 17 +++ llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 + llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll | 40 ++++++ .../test/CodeGen/AMDGPU/load-atomic-global.ll | 120 ++++++++++++++++++ llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll | 24 ++++ .../CodeGen/AMDGPU/store-atomic-global.ll | 72 +++++++++++ .../AtomicExpand/AMDGPU/unaligned-atomic.ll | 37 ++++++ 10 files changed, 366 insertions(+), 12 deletions(-) diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index d8d371dfe9910..d491337f71ea7 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -2280,11 +2280,14 @@ class LLVM_ABI TargetLoweringBase { unsigned getMinCmpXchgSizeInBits() const { return MinCmpXchgSizeInBits; } /// Return true if the target supports an atomic access of \p SizeInBytes - /// bytes at the given \p Alignment. The default implementation only allows - /// naturally aligned atomics, unless setSupportsUnalignedAtomics(true) was - /// called. - virtual bool supportsAtomicAlignment(Align Alignment, - uint64_t SizeInBytes) const { + /// bytes at the given \p Alignment in \p AddrSpace. \p ElementSizeInBytes is + /// the granularity at which the access has to be indivisible; it is smaller + /// than \p SizeInBytes only for elementwise atomics. The default + /// implementation ignores both and only allows naturally aligned atomics, + /// unless setSupportsUnalignedAtomics(true) was called. + virtual bool supportsAtomicAlignment(Align Alignment, uint64_t SizeInBytes, + uint64_t ElementSizeInBytes, + unsigned AddrSpace) const { return SupportsUnalignedAtomics || Alignment.value() >= SizeInBytes; } diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp index f2ffa40030cc0..93a3949af454e 100644 --- a/llvm/lib/CodeGen/AtomicExpandPass.cpp +++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp @@ -230,6 +230,37 @@ static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) { return DL.getTypeStoreSize(CASI->getCompareOperand()->getType()); } +static unsigned getAtomicOpElementSize(LoadInst *LI) { + if (!LI->isElementwise()) + return getAtomicOpSize(LI); + return LI->getDataLayout().getTypeStoreSize(LI->getType()->getScalarType()); +} + +static unsigned getAtomicOpElementSize(StoreInst *SI) { + if (!SI->isElementwise()) + return getAtomicOpSize(SI); + return SI->getDataLayout().getTypeStoreSize( + SI->getValueOperand()->getType()->getScalarType()); +} + +template <typename Inst> static unsigned getAtomicOpElementSize(Inst *I) { + return getAtomicOpSize(I); +} + +/// Return the size \p I has to be aligned to. The element size is smaller than +/// the whole access only for elementwise atomics, and only counts if the target +/// can issue them at element alignment. +template <typename Inst> +static unsigned getRequiredAtomicSize(const TargetLowering *TLI, Inst *I) { + unsigned Size = getAtomicOpSize(I); + unsigned ElementSize = getAtomicOpElementSize(I); + if (ElementSize != Size && + TLI->supportsAtomicAlignment(I->getAlign(), Size, ElementSize, + I->getPointerAddressSpace())) + return ElementSize; + return Size; +} + /// Copy metadata that's safe to preserve when widening atomics. static void copyMetadataForAtomic(Instruction &Dest, const Instruction &Source) { @@ -265,22 +296,22 @@ static void copyMetadataForAtomic(Instruction &Dest, template <typename Inst> static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) { - unsigned Size = getAtomicOpSize(I); - Align Alignment = I->getAlign(); unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8; - return Alignment >= Size && Size <= MaxSize; + return I->getAlign() >= getRequiredAtomicSize(TLI, I) && + getAtomicOpSize(I) <= MaxSize; } template <typename Inst> static void writeUnsupportedAtomicSizeReason(const TargetLowering *TLI, Inst *I, raw_ostream &OS) { + unsigned RequiredSize = getRequiredAtomicSize(TLI, I); unsigned Size = getAtomicOpSize(I); Align Alignment = I->getAlign(); bool NeedSeparator = false; - if (Alignment < Size) { + if (Alignment < RequiredSize) { OS << "instruction alignment " << Alignment.value() - << " is smaller than the required " << Size + << " is smaller than the required " << RequiredSize << "-byte alignment for this atomic operation"; NeedSeparator = true; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 367fa58ab94a1..5805c1bcdcd5a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5368,8 +5368,11 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); + EVT ElementVT = I.isElementwise() ? MemVT.getScalarType() : MemVT; - if (!TLI.supportsAtomicAlignment(I.getAlign(), MemVT.getSizeInBits() / 8)) + if (!TLI.supportsAtomicAlignment(I.getAlign(), MemVT.getSizeInBits() / 8, + ElementVT.getSizeInBits() / 8, + I.getPointerAddressSpace())) report_fatal_error("Cannot generate unaligned atomic load"); auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo); @@ -5404,8 +5407,11 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType()); + EVT ElementVT = I.isElementwise() ? MemVT.getScalarType() : MemVT; - if (!TLI.supportsAtomicAlignment(I.getAlign(), MemVT.getSizeInBits() / 8)) + if (!TLI.supportsAtomicAlignment(I.getAlign(), MemVT.getSizeInBits() / 8, + ElementVT.getSizeInBits() / 8, + I.getPointerAddressSpace())) report_fatal_error("Cannot generate unaligned atomic store"); auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index d9810e3d9fbd9..5e9244cad7cff 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -2361,6 +2361,23 @@ bool SITargetLowering::allowsMisalignedMemoryAccesses( Alignment, Flags, IsFast); } +bool SITargetLowering::supportsAtomicAlignment(Align Alignment, + uint64_t SizeInBytes, + uint64_t ElementSizeInBytes, + unsigned AddrSpace) const { + // Each naturally aligned dword is separately atomic. Only global and flat + // opt in: the LDS 4 byte rule in allowsMisalignedMemoryAccessesImpl assumes + // lowering to ds_read2_b32, which never happens for an atomic, and buffer + // accesses need natural alignment for out-of-bounds semantics. + if (ElementSizeInBytes < SizeInBytes && ElementSizeInBytes >= 4 && + (AMDGPU::isExtendedGlobalAddrSpace(AddrSpace) || + AddrSpace == AMDGPUAS::FLAT_ADDRESS)) + return Alignment.value() >= ElementSizeInBytes; + + return TargetLoweringBase::supportsAtomicAlignment( + Alignment, SizeInBytes, ElementSizeInBytes, AddrSpace); +} + EVT SITargetLowering::getOptimalMemOpType( LLVMContext &Context, const MemOp &Op, const AttributeList &FuncAttributes) const { diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index 8ece9f279a100..d7e37825550e0 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -391,6 +391,10 @@ class SITargetLowering final : public AMDGPUTargetLowering { MachineMemOperand::Flags Flags = MachineMemOperand::MONone, unsigned *IsFast = nullptr) const override; + bool supportsAtomicAlignment(Align Alignment, uint64_t SizeInBytes, + uint64_t ElementSizeInBytes, + unsigned AddrSpace) const override; + EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &FuncAttributes) const override; diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll index 69e3378ae08b9..5e9a593de9a74 100644 --- a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll +++ b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll @@ -48,6 +48,46 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(0) %p, pt ret void } +define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_elementwise(ptr addrspace(0) %p, ptr addrspace(1) %out) { +; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_elementwise: +; GFX9: ; %bb.0: +; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: global_store_dword v[2:3], v0, off +; GFX9-NEXT: s_endpgm +; +; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_elementwise: +; GFX10: ; %bb.0: +; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc +; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX10-NEXT: global_store_dword v[2:3], v0, off +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_elementwise: +; GFX11: ; %bb.0: +; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX11-NEXT: global_store_b32 v[2:3], v0, off +; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_elementwise: +; GFX12: ; %bb.0: +; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX12-NEXT: global_store_b32 v[2:3], v0, off +; GFX12-NEXT: s_endpgm + %a0 = load atomic elementwise <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4 + %num1 = extractelement <2 x float> %a0, i32 0 + %num2 = extractelement <2 x float> %a0, i32 1 + %res = fadd float %num1, %num2 + store float %res, ptr addrspace(1) %out, align 4 + ret void +} + define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) { ; GFX9-LABEL: atomic_load_f16x2_monotonic_agent: ; GFX9: ; %bb.0: diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll index 5ac3ca4786e92..4860f6b7cad1a 100644 --- a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll +++ b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll @@ -48,6 +48,86 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(1) %p, pt ret void } +define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_elementwise(ptr addrspace(1) %p, ptr addrspace(1) %out) { +; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_elementwise: +; GFX9: ; %bb.0: +; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: global_store_dword v[2:3], v0, off +; GFX9-NEXT: s_endpgm +; +; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_elementwise: +; GFX10: ; %bb.0: +; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX10-NEXT: global_store_dword v[2:3], v0, off +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_elementwise: +; GFX11: ; %bb.0: +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX11-NEXT: global_store_b32 v[2:3], v0, off +; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_elementwise: +; GFX12: ; %bb.0: +; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX12-NEXT: global_store_b32 v[2:3], v0, off +; GFX12-NEXT: s_endpgm + %a0 = load atomic elementwise <2 x float>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4 + %num1 = extractelement <2 x float> %a0, i32 0 + %num2 = extractelement <2 x float> %a0, i32 1 + %res = fadd float %num1, %num2 + store float %res, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_cs void @atomic_load_f32x2_unordered_agent_elementwise(ptr addrspace(1) %p, ptr addrspace(1) %out) { +; GFX9-LABEL: atomic_load_f32x2_unordered_agent_elementwise: +; GFX9: ; %bb.0: +; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: global_store_dword v[2:3], v0, off +; GFX9-NEXT: s_endpgm +; +; GFX10-LABEL: atomic_load_f32x2_unordered_agent_elementwise: +; GFX10: ; %bb.0: +; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX10-NEXT: global_store_dword v[2:3], v0, off +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: atomic_load_f32x2_unordered_agent_elementwise: +; GFX11: ; %bb.0: +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX11-NEXT: global_store_b32 v[2:3], v0, off +; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: atomic_load_f32x2_unordered_agent_elementwise: +; GFX12: ; %bb.0: +; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX12-NEXT: global_store_b32 v[2:3], v0, off +; GFX12-NEXT: s_endpgm + %a0 = load atomic elementwise <2 x float>, ptr addrspace(1) %p syncscope("agent") unordered, align 4 + %num1 = extractelement <2 x float> %a0, i32 0 + %num2 = extractelement <2 x float> %a0, i32 1 + %res = fadd float %num1, %num2 + store float %res, ptr addrspace(1) %out, align 4 + ret void +} + define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) { ; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent: ; GFX9: ; %bb.0: @@ -134,6 +214,46 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(1) %p ret void } +define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront_elementwise(ptr addrspace(1) %p, ptr addrspace(1) %out) { +; GFX9-LABEL: atomic_load_f32x2_monotonic_wavefront_elementwise: +; GFX9: ; %bb.0: +; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: global_store_dword v[2:3], v0, off +; GFX9-NEXT: s_endpgm +; +; GFX10-LABEL: atomic_load_f32x2_monotonic_wavefront_elementwise: +; GFX10: ; %bb.0: +; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX10-NEXT: global_store_dword v[2:3], v0, off +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: atomic_load_f32x2_monotonic_wavefront_elementwise: +; GFX11: ; %bb.0: +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX11-NEXT: global_store_b32 v[2:3], v0, off +; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: atomic_load_f32x2_monotonic_wavefront_elementwise: +; GFX12: ; %bb.0: +; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX12-NEXT: global_store_b32 v[2:3], v0, off +; GFX12-NEXT: s_endpgm + %a0 = load atomic elementwise <2 x float>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4 + %num1 = extractelement <2 x float> %a0, i32 0 + %num2 = extractelement <2 x float> %a0, i32 1 + %res = fadd float %num1, %num2 + store float %res, ptr addrspace(1) %out, align 4 + ret void +} + define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) { ; GFX9-LABEL: atomic_load_f16x2_monotonic_agent: ; GFX9: ; %bb.0: diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll index 855091bb8a005..e2678e204d557 100644 --- a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll +++ b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll @@ -32,6 +32,30 @@ define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr a ret void } +define amdgpu_cs void @atomic_store_f32x2_monotonic_agent_elementwise(<2 x float> %in, ptr addrspace(0) %out) { +; GFX9-LABEL: atomic_store_f32x2_monotonic_agent_elementwise: +; GFX9: ; %bb.0: +; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; GFX9-NEXT: s_endpgm +; +; GFX10-LABEL: atomic_store_f32x2_monotonic_agent_elementwise: +; GFX10: ; %bb.0: +; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: atomic_store_f32x2_monotonic_agent_elementwise: +; GFX11: ; %bb.0: +; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] +; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: atomic_store_f32x2_monotonic_agent_elementwise: +; GFX12: ; %bb.0: +; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV +; GFX12-NEXT: s_endpgm + store atomic elementwise <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4 + ret void +} + define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) { ; GFX9-LABEL: atomic_store_f16x2_monotonic_agent: ; GFX9: ; %bb.0: diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll index 3de24b553f793..9ff69256f1b92 100644 --- a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll +++ b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll @@ -32,6 +32,54 @@ define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr a ret void } +define amdgpu_cs void @atomic_store_f32x2_monotonic_agent_elementwise(<2 x float> %in, ptr addrspace(1) %out) { +; GFX9-LABEL: atomic_store_f32x2_monotonic_agent_elementwise: +; GFX9: ; %bb.0: +; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX9-NEXT: s_endpgm +; +; GFX10-LABEL: atomic_store_f32x2_monotonic_agent_elementwise: +; GFX10: ; %bb.0: +; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: atomic_store_f32x2_monotonic_agent_elementwise: +; GFX11: ; %bb.0: +; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off +; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: atomic_store_f32x2_monotonic_agent_elementwise: +; GFX12: ; %bb.0: +; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV +; GFX12-NEXT: s_endpgm + store atomic elementwise <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_cs void @atomic_store_f32x2_unordered_agent_elementwise(<2 x float> %in, ptr addrspace(1) %out) { +; GFX9-LABEL: atomic_store_f32x2_unordered_agent_elementwise: +; GFX9: ; %bb.0: +; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX9-NEXT: s_endpgm +; +; GFX10-LABEL: atomic_store_f32x2_unordered_agent_elementwise: +; GFX10: ; %bb.0: +; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: atomic_store_f32x2_unordered_agent_elementwise: +; GFX11: ; %bb.0: +; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off +; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: atomic_store_f32x2_unordered_agent_elementwise: +; GFX12: ; %bb.0: +; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off +; GFX12-NEXT: s_endpgm + store atomic elementwise <2 x float> %in, ptr addrspace(1) %out syncscope("agent") unordered, align 4 + ret void +} + define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) { ; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent: ; GFX9: ; %bb.0: @@ -80,6 +128,30 @@ define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr ret void } +define amdgpu_cs void @atomic_store_f32x2_monotonic_wavefront_elementwise(<2 x float> %in, ptr addrspace(1) %out) { +; GFX9-LABEL: atomic_store_f32x2_monotonic_wavefront_elementwise: +; GFX9: ; %bb.0: +; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX9-NEXT: s_endpgm +; +; GFX10-LABEL: atomic_store_f32x2_monotonic_wavefront_elementwise: +; GFX10: ; %bb.0: +; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: atomic_store_f32x2_monotonic_wavefront_elementwise: +; GFX11: ; %bb.0: +; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off +; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: atomic_store_f32x2_monotonic_wavefront_elementwise: +; GFX12: ; %bb.0: +; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off +; GFX12-NEXT: s_endpgm + store atomic elementwise <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4 + ret void +} + define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) { ; GFX9-LABEL: atomic_store_f16x2_monotonic_agent: ; GFX9: ; %bb.0: diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/unaligned-atomic.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/unaligned-atomic.ll index e8884970c33e5..38a156684ff14 100644 --- a/llvm/test/Transforms/AtomicExpand/AMDGPU/unaligned-atomic.ll +++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/unaligned-atomic.ll @@ -11,3 +11,40 @@ define void @atomic_store_global_align1(ptr addrspace(1) %ptr, i32 %val) { store atomic i32 %val, ptr addrspace(1) %ptr monotonic, align 1 ret void } + +; CHECK: error: unsupported atomic load: instruction alignment 2 is smaller than the required 8-byte alignment for this atomic operation +define <2 x float> @atomic_load_global_elementwise_align2(ptr addrspace(1) %ptr) { + %val = load atomic elementwise <2 x float>, ptr addrspace(1) %ptr monotonic, align 2 + ret <2 x float> %val +} + +; CHECK: error: unsupported atomic store: instruction alignment 2 is smaller than the required 8-byte alignment for this atomic operation +define void @atomic_store_global_elementwise_align2(ptr addrspace(1) %ptr, <2 x float> %val) { + store atomic elementwise <2 x float> %val, ptr addrspace(1) %ptr monotonic, align 2 + ret void +} + +; CHECK: error: unsupported atomic load: instruction alignment 4 is smaller than the required 8-byte alignment for this atomic operation +define <2 x float> @atomic_load_global_align4(ptr addrspace(1) %ptr) { + %val = load atomic <2 x float>, ptr addrspace(1) %ptr monotonic, align 4 + ret <2 x float> %val +} + +; CHECK: error: unsupported atomic load: instruction alignment 2 is smaller than the required 4-byte alignment for this atomic operation +define <2 x half> @atomic_load_global_elementwise_f16_align2(ptr addrspace(1) %ptr) { + %val = load atomic elementwise <2 x half>, ptr addrspace(1) %ptr monotonic, align 2 + ret <2 x half> %val +} + +; The LDS 4-byte rule assumes lowering to ds_read2_b32, which atomics never get. +; CHECK: error: unsupported atomic load: instruction alignment 4 is smaller than the required 8-byte alignment for this atomic operation +define <2 x float> @atomic_load_local_elementwise_align4(ptr addrspace(3) %ptr) { + %val = load atomic elementwise <2 x float>, ptr addrspace(3) %ptr monotonic, align 4 + ret <2 x float> %val +} + +; CHECK: error: unsupported atomic load: target supports atomics up to 8 bytes, but this atomic accesses 16 bytes +define <4 x float> @atomic_load_global_elementwise_v4f32_align4(ptr addrspace(1) %ptr) { + %val = load atomic elementwise <4 x float>, ptr addrspace(1) %ptr monotonic, align 4 + ret <4 x float> %val +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
