https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/230944
Make the atomic load/store legality match the hardware operand structure. A 16-bit atomic load only produces a real 16-bit result with the true16 D16 loads, which require d16PreservesUnusedBits. With SRAMECC enabled the true16 patterns use a 32-bit load and extract the low half, so promote i16, f16 and bf16 atomic loads to an i32 extending load instead. Atomic stores remain 16-bit with real true16, since the store source is a 16-bit register regardless. Remove the true16 sramecc atomic load patterns which are now dead. Co-authored-by: Claude Opus 5.5 <[email protected]> >From 3e8cc2a6ac3f11d4b8d3521fd1721a93290d8ee6 Mon Sep 17 00:00:00 2001 From: Matt Arsenault <[email protected]> Date: Thu, 8 Oct 2026 20:41:53 +0200 Subject: [PATCH] AMDGPU: Only use 16-bit atomic loads when D16 preserves unused bits Make the atomic load/store legality match the hardware operand structure. A 16-bit atomic load only produces a real 16-bit result with the true16 D16 loads, which require d16PreservesUnusedBits. With SRAMECC enabled the true16 patterns use a 32-bit load and extract the low half, so promote i16, f16 and bf16 atomic loads to an i32 extending load instead. Atomic stores remain 16-bit with real true16, since the store source is a 16-bit register regardless. Remove the true16 sramecc atomic load patterns which are now dead. Co-authored-by: Claude Opus 5.5 <[email protected]> --- llvm/lib/Target/AMDGPU/AMDGPU.td | 4 + .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 19 +- llvm/lib/Target/AMDGPU/BUFInstructions.td | 2 +- llvm/lib/Target/AMDGPU/DSInstructions.td | 6 +- llvm/lib/Target/AMDGPU/FLATInstructions.td | 11 -- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 11 +- .../AMDGPU/atomic-load-store-16-sramecc.ll | 164 ++++++++++++++++++ 7 files changed, 193 insertions(+), 24 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/atomic-load-store-16-sramecc.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index 73464a89544cc..b66159d1a6304 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -3202,6 +3202,10 @@ def UseFakeTrue16Insts : True16PredicateClass<"Subtarget->hasTrue16BitInsts() && def UseTrue16WithSramECC : True16PredicateClass<"Subtarget->useRealTrue16Insts() && " "!Subtarget->d16PreservesUnusedBits()">; +def UseTrue16WithD16PreservesUnusedBits : + True16PredicateClass<"Subtarget->useRealTrue16Insts() && " + "Subtarget->d16PreservesUnusedBits()">; + def NotHasD16Writes32BitVgpr: Predicate<"!Subtarget->hasD16Writes32BitVgpr()">, AssemblerPredicate<(all_of FeatureTrue16BitInsts, FeatureRealTrue16Insts, (not FeatureD16Writes32BitVgpr))>; diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 188453d73a435..52fae0df0eef2 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -472,6 +472,9 @@ static bool isLoadStoreSizeLegal(const GCNSubtarget &ST, if (MemSize != RegSize && RegSize != 32) return false; + if (IsLoad && RegSize == 16 && !ST.d16PreservesUnusedBits()) + return false; + if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad, Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic)) @@ -1664,13 +1667,15 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, {V4S32, ConstantPtr, V4S32, GlobalAlign32}, {S64, ConstantPtr, S64, GlobalAlign32}}); - Actions.legalForTypesWithMemDesc(ST.useRealTrue16Insts(), /* Pred */ - {{S16, GlobalPtr, S8, GlobalAlign8}, - {S16, GlobalPtr, S16, GlobalAlign16}, - {S16, LocalPtr, S8, 8}, - {S16, LocalPtr, S16, 16}, - {S16, PrivatePtr, S8, 8}, - {S16, PrivatePtr, S16, 16}}); + if (IsStore || ST.d16PreservesUnusedBits()) { + Actions.legalForTypesWithMemDesc(ST.useRealTrue16Insts(), /* Pred */ + {{S16, GlobalPtr, S8, GlobalAlign8}, + {S16, GlobalPtr, S16, GlobalAlign16}, + {S16, LocalPtr, S8, 8}, + {S16, LocalPtr, S16, 16}, + {S16, PrivatePtr, S8, 8}, + {S16, PrivatePtr, S16, 16}}); + } Actions.legalIf( [=](const LegalityQuery &Query) -> bool { diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 3a428e1923f77..0f80d072e5282 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -1306,7 +1306,7 @@ defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < } // End HasD16LoadStore -let True16Predicate = UseRealTrue16Insts in { +let True16Predicate = UseTrue16WithD16PreservesUnusedBits in { defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE_D16_t16", i16, atomic_load_aext_8_global>; defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE_D16_t16", i16, atomic_load_zext_8_global>; foreach vt = Reg16Types.types in diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 7f123ad962e4d..4275697d31a85 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -969,9 +969,6 @@ multiclass DSReadPat_True16<DS_Pseudo inst, ValueType vt, string frag> { let OtherPredicates = [NotLDSRequiresM0Init, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in { def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>; } - let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = UseTrue16WithSramECC in { - def : DSReadPat_t16<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>; - } } multiclass DSReadPat_t16<DS_Pseudo inst, ValueType vt, string frag> { @@ -984,6 +981,9 @@ multiclass DSReadPat_t16<DS_Pseudo inst, ValueType vt, string frag> { def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>; } defm : DSReadPat_True16<inst, vt, frag>; + let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = UseTrue16WithSramECC in { + def : DSReadPat_t16<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>; + } let OtherPredicates = [HasGDS], True16Predicate = NotUseRealTrue16Insts in { def : DSReadPat<inst, vt, !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0"), diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index 585e5619d51a5..84e274292897c 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -2076,11 +2076,6 @@ let True16Predicate = UseTrue16WithSramECC in { defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>; defm : FlatLoadPats_t16 <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; defm : FlatLoadPats_t16 <FLAT_LOAD_USHORT, load_flat, i16>; - defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i16>; - defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i16>; - foreach vt = Reg16Types.types in - defm : FlatLoadPats_t16 <FLAT_LOAD_USHORT, atomic_load_flat, vt>; - defm : FlatLoadPats_t16 <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i16>; } let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predicate = UseRealTrue16Insts in { @@ -2234,12 +2229,6 @@ defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_SSHORT, atomic_load_sext_16_global, i32>; defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i32>; defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, load_global, i16>; -defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i16>; -defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i16>; -defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i16>; -foreach vt = Reg16Types.types in -defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_global, vt>; -defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i16>; } let OtherPredicates = [D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in { diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index d743a5d06adf5..7280e7fdbbefa 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -659,10 +659,17 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::STORE, MVT::bf16, Promote); AddPromotedToType(ISD::STORE, MVT::bf16, MVT::i16); - if (!Subtarget->useRealTrue16Insts()) { + if (!Subtarget->useRealTrue16Insts() || + !Subtarget->d16PreservesUnusedBits()) { for (MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) { - setOperationAction({ISD::ATOMIC_LOAD, ISD::ATOMIC_STORE}, VT, Promote); + setOperationAction(ISD::ATOMIC_LOAD, VT, Promote); AddPromotedToType(ISD::ATOMIC_LOAD, VT, MVT::i32); + } + } + + if (!Subtarget->useRealTrue16Insts()) { + for (MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) { + setOperationAction(ISD::ATOMIC_STORE, VT, Promote); AddPromotedToType(ISD::ATOMIC_STORE, VT, MVT::i32); } } diff --git a/llvm/test/CodeGen/AMDGPU/atomic-load-store-16-sramecc.ll b/llvm/test/CodeGen/AMDGPU/atomic-load-store-16-sramecc.ll new file mode 100644 index 0000000000000..8512cd50990e3 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/atomic-load-store-16-sramecc.ll @@ -0,0 +1,164 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: sed 's/SRAMECC/0/' %s | llc -mtriple=amdgpu12.50 -mattr=+real-true16 | FileCheck -check-prefix=SRAMECC-OFF %s +; RUN: sed 's/SRAMECC/1/' %s | llc -mtriple=amdgpu12.50 -mattr=+real-true16 | FileCheck -check-prefix=SRAMECC-ON %s +; RUN: sed 's/SRAMECC/0/' %s | llc -global-isel -mtriple=amdgpu12.50 -mattr=+real-true16 | FileCheck -check-prefix=SRAMECC-OFF %s +; RUN: sed 's/SRAMECC/1/' %s | llc -global-isel -mtriple=amdgpu12.50 -mattr=+real-true16 | FileCheck -check-prefix=SRAMECC-ON %s + +define i16 @atomic_load_i16_global(ptr addrspace(1) %p) { +; SRAMECC-OFF-LABEL: atomic_load_i16_global: +; SRAMECC-OFF: ; %bb.0: +; SRAMECC-OFF-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-OFF-NEXT: global_load_d16_b16 v0, v[0:1], off scope:SCOPE_SYS +; SRAMECC-OFF-NEXT: s_wait_loadcnt 0x0 +; SRAMECC-OFF-NEXT: s_set_pc_i64 s[30:31] +; +; SRAMECC-ON-LABEL: atomic_load_i16_global: +; SRAMECC-ON: ; %bb.0: +; SRAMECC-ON-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-ON-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-ON-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS +; SRAMECC-ON-NEXT: s_wait_loadcnt 0x0 +; SRAMECC-ON-NEXT: s_set_pc_i64 s[30:31] + %v = load atomic i16, ptr addrspace(1) %p monotonic, align 2 + ret i16 %v +} + +define half @atomic_load_f16_global(ptr addrspace(1) %p) { +; SRAMECC-OFF-LABEL: atomic_load_f16_global: +; SRAMECC-OFF: ; %bb.0: +; SRAMECC-OFF-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-OFF-NEXT: global_load_d16_b16 v0, v[0:1], off scope:SCOPE_SYS +; SRAMECC-OFF-NEXT: s_wait_loadcnt 0x0 +; SRAMECC-OFF-NEXT: s_set_pc_i64 s[30:31] +; +; SRAMECC-ON-LABEL: atomic_load_f16_global: +; SRAMECC-ON: ; %bb.0: +; SRAMECC-ON-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-ON-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-ON-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS +; SRAMECC-ON-NEXT: s_wait_loadcnt 0x0 +; SRAMECC-ON-NEXT: s_set_pc_i64 s[30:31] + %v = load atomic half, ptr addrspace(1) %p monotonic, align 2 + ret half %v +} + +define bfloat @atomic_load_bf16_global(ptr addrspace(1) %p) { +; SRAMECC-OFF-LABEL: atomic_load_bf16_global: +; SRAMECC-OFF: ; %bb.0: +; SRAMECC-OFF-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-OFF-NEXT: global_load_d16_b16 v0, v[0:1], off scope:SCOPE_SYS +; SRAMECC-OFF-NEXT: s_wait_loadcnt 0x0 +; SRAMECC-OFF-NEXT: s_set_pc_i64 s[30:31] +; +; SRAMECC-ON-LABEL: atomic_load_bf16_global: +; SRAMECC-ON: ; %bb.0: +; SRAMECC-ON-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-ON-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-ON-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS +; SRAMECC-ON-NEXT: s_wait_loadcnt 0x0 +; SRAMECC-ON-NEXT: s_set_pc_i64 s[30:31] + %v = load atomic bfloat, ptr addrspace(1) %p monotonic, align 2 + ret bfloat %v +} + +define i16 @atomic_load_i16_flat(ptr %p) { +; SRAMECC-OFF-LABEL: atomic_load_i16_flat: +; SRAMECC-OFF: ; %bb.0: +; SRAMECC-OFF-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-OFF-NEXT: flat_load_d16_b16 v0, v[0:1] scope:SCOPE_SYS +; SRAMECC-OFF-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_set_pc_i64 s[30:31] +; +; SRAMECC-ON-LABEL: atomic_load_i16_flat: +; SRAMECC-ON: ; %bb.0: +; SRAMECC-ON-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-ON-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-ON-NEXT: flat_load_u16 v0, v[0:1] scope:SCOPE_SYS +; SRAMECC-ON-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-ON-NEXT: s_set_pc_i64 s[30:31] + %v = load atomic i16, ptr %p monotonic, align 2 + ret i16 %v +} + +define half @atomic_load_f16_local(ptr addrspace(3) %p) { +; SRAMECC-OFF-LABEL: atomic_load_f16_local: +; SRAMECC-OFF: ; %bb.0: +; SRAMECC-OFF-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-OFF-NEXT: ds_load_u16_d16 v0, v0 +; SRAMECC-OFF-NEXT: s_wait_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_set_pc_i64 s[30:31] +; +; SRAMECC-ON-LABEL: atomic_load_f16_local: +; SRAMECC-ON: ; %bb.0: +; SRAMECC-ON-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-ON-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-ON-NEXT: ds_load_u16 v0, v0 +; SRAMECC-ON-NEXT: s_wait_dscnt 0x0 +; SRAMECC-ON-NEXT: s_set_pc_i64 s[30:31] + %v = load atomic half, ptr addrspace(3) %p monotonic, align 2 + ret half %v +} + +define void @atomic_store_i16_global(ptr addrspace(1) %p, i16 %v) { +; SRAMECC-OFF-LABEL: atomic_store_i16_global: +; SRAMECC-OFF: ; %bb.0: +; SRAMECC-OFF-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-OFF-NEXT: global_store_b16 v[0:1], v2, off scope:SCOPE_SYS +; SRAMECC-OFF-NEXT: s_set_pc_i64 s[30:31] +; +; SRAMECC-ON-LABEL: atomic_store_i16_global: +; SRAMECC-ON: ; %bb.0: +; SRAMECC-ON-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-ON-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-ON-NEXT: global_store_b16 v[0:1], v2, off scope:SCOPE_SYS +; SRAMECC-ON-NEXT: s_set_pc_i64 s[30:31] + store atomic i16 %v, ptr addrspace(1) %p monotonic, align 2 + ret void +} + +define void @atomic_store_f16_global(ptr addrspace(1) %p, half %v) { +; SRAMECC-OFF-LABEL: atomic_store_f16_global: +; SRAMECC-OFF: ; %bb.0: +; SRAMECC-OFF-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-OFF-NEXT: global_store_b16 v[0:1], v2, off scope:SCOPE_SYS +; SRAMECC-OFF-NEXT: s_set_pc_i64 s[30:31] +; +; SRAMECC-ON-LABEL: atomic_store_f16_global: +; SRAMECC-ON: ; %bb.0: +; SRAMECC-ON-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-ON-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-ON-NEXT: global_store_b16 v[0:1], v2, off scope:SCOPE_SYS +; SRAMECC-ON-NEXT: s_set_pc_i64 s[30:31] + store atomic half %v, ptr addrspace(1) %p monotonic, align 2 + ret void +} + +define void @atomic_store_bf16_local(ptr addrspace(3) %p, bfloat %v) { +; SRAMECC-OFF-LABEL: atomic_store_bf16_local: +; SRAMECC-OFF: ; %bb.0: +; SRAMECC-OFF-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-OFF-NEXT: ds_store_b16 v0, v1 +; SRAMECC-OFF-NEXT: s_wait_dscnt 0x0 +; SRAMECC-OFF-NEXT: s_set_pc_i64 s[30:31] +; +; SRAMECC-ON-LABEL: atomic_store_bf16_local: +; SRAMECC-ON: ; %bb.0: +; SRAMECC-ON-NEXT: s_wait_loadcnt_dscnt 0x0 +; SRAMECC-ON-NEXT: s_wait_kmcnt 0x0 +; SRAMECC-ON-NEXT: ds_store_b16 v0, v1 +; SRAMECC-ON-NEXT: s_wait_dscnt 0x0 +; SRAMECC-ON-NEXT: s_set_pc_i64 s[30:31] + store atomic bfloat %v, ptr addrspace(3) %p monotonic, align 2 + ret void +} + +!llvm.module.flags = !{!0} +!0 = !{i32 1, !"amdgpu.sramecc", i32 SRAMECC} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
