https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/230943
16-bit atomic loads are really extending loads into a 32-bit register, and 16-bit atomic stores are truncating stores of a 32-bit value. Without real true16, promote i16, f16 and bf16 atomic load and store to i32 as any-extending loads and truncating stores, matching how regular loads and stores are handled. With real true16, select all 16-bit types directly using the type-generic atomic patterns, as is done for the wider types. Teach LegalizeDAG to promote ATOMIC_LOAD and ATOMIC_STORE to a wider integer type, preserving the extension type. The i16-only atomic patterns are now only needed for real true16, so remove the rest. Co-authored-by: Claude Opus 5.5 <[email protected]> >From b6c7cf8f374960f650a068f125189fe3d6ec0203 Mon Sep 17 00:00:00 2001 From: Matt Arsenault <[email protected]> Date: Thu, 8 Oct 2026 20:24:06 +0200 Subject: [PATCH] AMDGPU: Promote 16-bit atomic load/store to extending i32 16-bit atomic loads are really extending loads into a 32-bit register, and 16-bit atomic stores are truncating stores of a 32-bit value. Without real true16, promote i16, f16 and bf16 atomic load and store to i32 as any-extending loads and truncating stores, matching how regular loads and stores are handled. With real true16, select all 16-bit types directly using the type-generic atomic patterns, as is done for the wider types. Teach LegalizeDAG to promote ATOMIC_LOAD and ATOMIC_STORE to a wider integer type, preserving the extension type. The i16-only atomic patterns are now only needed for real true16, so remove the rest. Co-authored-by: Claude Opus 5.5 <[email protected]> --- llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 28 +++++++++ llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 9 ++- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 13 ---- llvm/lib/Target/AMDGPU/BUFInstructions.td | 11 +--- llvm/lib/Target/AMDGPU/DSInstructions.td | 61 +++++++++++-------- llvm/lib/Target/AMDGPU/FLATInstructions.td | 35 +++++------ llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 8 +++ llvm/test/CodeGen/AMDGPU/gds-load-store.ll | 6 -- 8 files changed, 91 insertions(+), 80 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index ffc9173e767fa..dbb7811b0a044 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -6220,6 +6220,18 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { case ISD::ATOMIC_STORE: { AtomicSDNode *AM = cast<AtomicSDNode>(Node); SDLoc SL(Node); + if (AM->getOpcode() == ISD::ATOMIC_STORE && NVT.bitsGT(OVT)) { + assert(NVT.isInteger() && "unexpected promotion type"); + MVT IntVT = MVT::getIntegerVT(OVT.getSizeInBits()); + EVT MemVT = AM->getMemoryVT().changeTypeToInteger(); + SDValue IntVal = DAG.getNode(ISD::BITCAST, SL, IntVT, AM->getVal()); + SDValue ExtVal = DAG.getNode(ISD::ANY_EXTEND, SL, NVT, IntVal); + Results.push_back(DAG.getAtomic(ISD::ATOMIC_STORE, SL, MemVT, + AM->getChain(), ExtVal, AM->getBasePtr(), + AM->getMemOperand())); + break; + } + SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); assert(NVT.getSizeInBits() == OVT.getSizeInBits() && "unexpected promotion type"); @@ -6247,6 +6259,22 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { case ISD::ATOMIC_LOAD: { AtomicSDNode *AM = cast<AtomicSDNode>(Node); SDLoc SL(Node); + if (NVT.bitsGT(OVT)) { + assert(NVT.isInteger() && "unexpected promotion type"); + MVT IntVT = MVT::getIntegerVT(OVT.getSizeInBits()); + EVT MemVT = AM->getMemoryVT().changeTypeToInteger(); + ISD::LoadExtType ExtType = AM->getExtensionType(); + if (ExtType == ISD::NON_EXTLOAD) + ExtType = ISD::EXTLOAD; + SDValue NewAtomic = + DAG.getAtomicLoad(ExtType, SL, MemVT, NVT, AM->getChain(), + AM->getBasePtr(), AM->getMemOperand()); + SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, IntVT, NewAtomic); + Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, Trunc)); + Results.push_back(NewAtomic.getValue(1)); + break; + } + assert(NVT.getSizeInBits() == OVT.getSizeInBits() && "unexpected promotion type"); assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index a2e6a4bf485b5..dc2abb60567ff 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -355,13 +355,12 @@ bool AMDGPUDAGToDAGISel::widenRegionLoad16(SDNode *N) const { if (ExtType == ISD::NON_EXTLOAD) ExtType = ISD::EXTLOAD; + EVT MemVT = Mem->getMemoryVT().changeTypeToInteger(); SDValue NewLoad = Ld ? CurDAG->getExtLoad(ExtType, SL, MVT::i32, Mem->getChain(), - Mem->getBasePtr(), Mem->getMemoryVT(), - Mem->getMemOperand()) - : CurDAG->getAtomicLoad(ExtType, SL, Mem->getMemoryVT(), MVT::i32, - Mem->getChain(), Mem->getBasePtr(), - Mem->getMemOperand()); + Mem->getBasePtr(), MemVT, Mem->getMemOperand()) + : CurDAG->getAtomicLoad(ExtType, SL, MemVT, MVT::i32, Mem->getChain(), + Mem->getBasePtr(), Mem->getMemOperand()); SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, SL, MVT::i16, NewLoad); SDValue Ops[] = {CurDAG->getBitcast(VT, Trunc), NewLoad.getValue(1)}; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index 3d497278ec6f5..c9ef618e1e7b3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -157,19 +157,6 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, setOperationAction(ISD::LOAD, MVT::i128, Promote); AddPromotedToType(ISD::LOAD, MVT::i128, MVT::v4i32); - // TODO: Would be better to consume as directly legal - setOperationAction(ISD::ATOMIC_LOAD, MVT::f16, Promote); - AddPromotedToType(ISD::ATOMIC_LOAD, MVT::f16, MVT::i16); - - setOperationAction(ISD::ATOMIC_LOAD, MVT::bf16, Promote); - AddPromotedToType(ISD::ATOMIC_LOAD, MVT::bf16, MVT::i16); - - setOperationAction(ISD::ATOMIC_STORE, MVT::f16, Promote); - AddPromotedToType(ISD::ATOMIC_STORE, MVT::f16, MVT::i16); - - setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote); - AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16); - // There are no 64-bit extloads. These should be done as a 32-bit extload and // an extension to 64-bit. for (MVT VT : MVT::integer_valuetypes()) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index e7f0c934eb9cc..3a428e1923f77 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -1306,16 +1306,11 @@ defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < } // End HasD16LoadStore -let True16Predicate = NotUseRealTrue16Insts in { -defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_aext_8_global>; -defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_zext_8_global>; -defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_nonext_16_global>; -} - let True16Predicate = UseRealTrue16Insts in { defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE_D16_t16", i16, atomic_load_aext_8_global>; defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE_D16_t16", i16, atomic_load_zext_8_global>; -defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SHORT_D16_t16", i16, atomic_load_nonext_16_global>; +foreach vt = Reg16Types.types in +defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SHORT_D16_t16", vt, atomic_load_global>; } let SubtargetPredicate = isNotGFX940Plus in @@ -2239,9 +2234,7 @@ multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo In } let SubtargetPredicate = isGFX6GFX7 in { defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; -defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; -defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; } // End Predicates = isGFX6GFX7 diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 3ff2ef169ee8f..7f123ad962e4d 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -965,6 +965,15 @@ multiclass DSReadPat_mc<DS_Pseudo inst, ValueType vt, string frag> { } } +multiclass DSReadPat_True16<DS_Pseudo inst, ValueType vt, string frag> { + let OtherPredicates = [NotLDSRequiresM0Init, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in { + def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>; + } + let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = UseTrue16WithSramECC in { + def : DSReadPat_t16<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>; + } +} + multiclass DSReadPat_t16<DS_Pseudo inst, ValueType vt, string frag> { let OtherPredicates = [LDSRequiresM0Init] in { @@ -974,12 +983,7 @@ multiclass DSReadPat_t16<DS_Pseudo inst, ValueType vt, string frag> { let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = NotUseRealTrue16Insts in { def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>; } - let OtherPredicates = [NotLDSRequiresM0Init, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in { - def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>; - } - let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = UseTrue16WithSramECC in { - def : DSReadPat_t16<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>; - } + defm : DSReadPat_True16<inst, vt, frag>; let OtherPredicates = [HasGDS], True16Predicate = NotUseRealTrue16Insts in { def : DSReadPat<inst, vt, !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0"), @@ -1012,13 +1016,14 @@ foreach vt = VReg_64.RegTypes in { defm : DSReadPat_mc <DS_READ_B64, vt, "atomic_load_local">; } -defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_aext_8_local">; +defm : DSReadPat_True16 <DS_READ_U8, i16, "atomic_load_aext_8_local">; defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_aext_8_local">; -defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_zext_8_local">; +defm : DSReadPat_True16 <DS_READ_U8, i16, "atomic_load_zext_8_local">; defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_zext_8_local">; -defm : DSReadPat_t16 <DS_READ_I8, i16, "atomic_load_sext_8_local">; +defm : DSReadPat_True16 <DS_READ_I8, i16, "atomic_load_sext_8_local">; defm : DSReadPat_mc <DS_READ_I8, i32, "atomic_load_sext_8_local">; -defm : DSReadPat_t16 <DS_READ_U16, i16, "atomic_load_nonext_16_local">; +foreach vt = Reg16Types.types in +defm : DSReadPat_True16 <DS_READ_U16, vt, "atomic_load_local">; defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_aext_16_local">; defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_zext_16_local">; defm : DSReadPat_mc <DS_READ_I16, i32, "atomic_load_sext_16_local">; @@ -1066,29 +1071,32 @@ multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> { } } +multiclass DSWritePat_True16 <DS_Pseudo inst, ValueType vt, string frag> { + let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = UseRealTrue16Insts in { + def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>; + } + + defvar region_frag = !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0"); + let OtherPredicates = [HasGDS], True16Predicate = UseRealTrue16Insts in { + def : DSWritePat_t16<inst, vt, region_frag, /*gds=*/1>; + } +} + multiclass DSWritePat_t16 <DS_Pseudo inst, ValueType vt, string frag> { let OtherPredicates = [LDSRequiresM0Init] in { def : DSWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>; } - let OtherPredicates = [NotLDSRequiresM0Init] in { - let True16Predicate = NotUseRealTrue16Insts in { - def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>; - } - let True16Predicate = UseRealTrue16Insts in { - def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>; - } + let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = NotUseRealTrue16Insts in { + def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>; } defvar region_frag = !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0"); - let OtherPredicates = [HasGDS] in { - let True16Predicate = NotUseRealTrue16Insts in { - def : DSWritePat<inst, vt, region_frag, /*gds=*/1>; - } - let True16Predicate = UseRealTrue16Insts in { - def : DSWritePat_t16<inst, vt, region_frag, /*gds=*/1>; - } + let OtherPredicates = [HasGDS], True16Predicate = NotUseRealTrue16Insts in { + def : DSWritePat<inst, vt, region_frag, /*gds=*/1>; } + + defm : DSWritePat_True16<inst, vt, frag>; } defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">; @@ -1105,9 +1113,10 @@ foreach vt = VReg_64.RegTypes in { defm : DSWritePat_mc <DS_WRITE_B64, vt, "atomic_store_local">; } -defm : DSWritePat_t16 <DS_WRITE_B8, i16, "atomic_store_8_local">; +defm : DSWritePat_True16 <DS_WRITE_B8, i16, "atomic_store_8_local">; defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">; -defm : DSWritePat_t16 <DS_WRITE_B16, i16, "atomic_store_16_local">; +foreach vt = Reg16Types.types in +defm : DSWritePat_True16 <DS_WRITE_B16, vt, "atomic_store_local">; defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">; let OtherPredicates = [HasD16LoadStore] in { diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index e6c7618a37a64..585e5619d51a5 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -2052,7 +2052,6 @@ defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i32>; defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i32>; defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_aext_16_flat, i32>; defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_zext_16_flat, i32>; -defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_zext_16_flat, i16>; defm : FlatLoadPats <FLAT_LOAD_UBYTE, extloadi8_flat, i32>; defm : FlatLoadPats <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>; defm : FlatLoadPats <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>; @@ -2068,14 +2067,8 @@ let True16Predicate = NotUseRealTrue16Insts in { defm : FlatLoadPats <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>; defm : FlatLoadPats <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; defm : FlatLoadPats <FLAT_LOAD_USHORT, load_flat, i16>; - defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i16>; - defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i16>; - defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_nonext_16_flat, i16>; - defm : FlatLoadPats <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i16>; defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i16>; defm : FlatStorePats <FLAT_STORE_SHORT, store_flat, i16>; - defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i16>; - defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i16>; } let True16Predicate = UseTrue16WithSramECC in { @@ -2085,7 +2078,8 @@ let True16Predicate = UseTrue16WithSramECC in { defm : FlatLoadPats_t16 <FLAT_LOAD_USHORT, load_flat, i16>; defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i16>; defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i16>; - defm : FlatLoadPats_t16 <FLAT_LOAD_USHORT, atomic_load_nonext_16_flat, i16>; + foreach vt = Reg16Types.types in + defm : FlatLoadPats_t16 <FLAT_LOAD_USHORT, atomic_load_flat, vt>; defm : FlatLoadPats_t16 <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i16>; } @@ -2096,7 +2090,8 @@ let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predi defm : FlatLoadPats_D16_t16<FLAT_LOAD_SHORT_D16_t16, load_flat, i16>; defm : FlatLoadPats_D16_t16<FLAT_LOAD_UBYTE_D16_t16, atomic_load_aext_8_flat, i16>; defm : FlatLoadPats_D16_t16<FLAT_LOAD_UBYTE_D16_t16, atomic_load_zext_8_flat, i16>; - defm : FlatLoadPats_D16_t16<FLAT_LOAD_SHORT_D16_t16, atomic_load_nonext_16_flat, i16>; + foreach vt = Reg16Types.types in + defm : FlatLoadPats_D16_t16<FLAT_LOAD_SHORT_D16_t16, atomic_load_flat, vt>; defm : FlatLoadPats_D16_t16<FLAT_LOAD_SBYTE_D16_t16, atomic_load_sext_8_flat, i16>; } // End let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predicate = UseRealTrue16Insts @@ -2104,7 +2099,8 @@ let OtherPredicates = [HasFlatAddressSpace], True16Predicate = UseRealTrue16Inst defm : FlatStorePats_t16 <FLAT_STORE_BYTE, truncstorei8_flat, i16>; defm : FlatStorePats_t16 <FLAT_STORE_SHORT, store_flat, i16>; defm : FlatStorePats_t16 <FLAT_STORE_BYTE, atomic_store_8_flat, i16>; - defm : FlatStorePats_t16 <FLAT_STORE_SHORT, atomic_store_16_flat, i16>; + foreach vt = Reg16Types.types in + defm : FlatStorePats_t16 <FLAT_STORE_SHORT, atomic_store_flat, vt>; } defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i32>; @@ -2229,11 +2225,6 @@ defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, atomic_load_sext_16_global, i32>; defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i32>; defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>; -defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i16>; -defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i16>; -defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i16>; -defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_nonext_16_global, i16>; -defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i16>; } let True16Predicate = UseTrue16WithSramECC in { @@ -2246,7 +2237,8 @@ defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, load_global, i16>; defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i16>; defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i16>; defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i16>; -defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_nonext_16_global, i16>; +foreach vt = Reg16Types.types in +defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_global, vt>; defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i16>; } @@ -2258,12 +2250,14 @@ defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", load_global, i16>; defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", atomic_load_aext_8_global, i16>; defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", atomic_load_zext_8_global, i16>; defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SBYTE_D16", atomic_load_sext_8_global, i16>; -defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", atomic_load_nonext_16_global, i16>; +foreach vt = Reg16Types.types in +defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", atomic_load_global, vt>; defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", atomic_load_zext_16_global, i16>; defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", truncstorei8_global, i16>; defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", store_global, i16>; defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", atomic_store_8_global, i16>; -defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", atomic_store_16_global, i16>; +foreach vt = Reg16Types.types in +defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", atomic_store_global, vt>; } // end OtherPredicates = [HasFlatGlobalInsts, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts // There is no distinction for atomic load lowering during selection; @@ -2299,15 +2293,14 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>; let OtherPredicates = [HasFlatGlobalInsts], True16Predicate = NotUseRealTrue16Insts in { defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>; defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>; -defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>; -defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>; } let OtherPredicates = [HasFlatGlobalInsts], True16Predicate = UseRealTrue16Insts in { defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_BYTE", truncstorei8_global, i16>; defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_SHORT", store_global, i16>; defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_BYTE", atomic_store_8_global, i16>; -defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_SHORT", atomic_store_16_global, i16>; +foreach vt = Reg16Types.types in +defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_SHORT", atomic_store_global, vt>; } let OtherPredicates = [HasD16LoadStore] in { diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index fa7c49528a820..d743a5d06adf5 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -659,6 +659,14 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::STORE, MVT::bf16, Promote); AddPromotedToType(ISD::STORE, MVT::bf16, MVT::i16); + if (!Subtarget->useRealTrue16Insts()) { + for (MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) { + setOperationAction({ISD::ATOMIC_LOAD, ISD::ATOMIC_STORE}, VT, Promote); + AddPromotedToType(ISD::ATOMIC_LOAD, VT, MVT::i32); + AddPromotedToType(ISD::ATOMIC_STORE, VT, MVT::i32); + } + } + // F16 - VOP1 Actions. setOperationAction({ISD::FP_ROUND, ISD::STRICT_FP_ROUND, ISD::FCOS, ISD::FSIN, ISD::FROUND}, diff --git a/llvm/test/CodeGen/AMDGPU/gds-load-store.ll b/llvm/test/CodeGen/AMDGPU/gds-load-store.ll index f47811c8ee4e9..3fb0f4596a162 100644 --- a/llvm/test/CodeGen/AMDGPU/gds-load-store.ll +++ b/llvm/test/CodeGen/AMDGPU/gds-load-store.ll @@ -1521,7 +1521,6 @@ define i32 @atomic_load_zext_i8(ptr addrspace(2) %ptr) { ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: buffer_wbinvl1_vol ; GFX9-NEXT: s_waitcnt expcnt(0) -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: atomic_load_zext_i8: @@ -1534,7 +1533,6 @@ define i32 @atomic_load_zext_i8(ptr addrspace(2) %ptr) { ; GFX10-NEXT: buffer_gl1_inv ; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_waitcnt expcnt(0) -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: atomic_load_zext_i8: @@ -1560,7 +1558,6 @@ define i32 @atomic_load_zext_i8(ptr addrspace(2) %ptr) { ; GFX11-FAKE16-NEXT: buffer_gl1_inv ; GFX11-FAKE16-NEXT: buffer_gl0_inv ; GFX11-FAKE16-NEXT: s_waitcnt expcnt(0) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %load = load atomic i8, ptr addrspace(2) %ptr seq_cst, align 1 %ext = zext i8 %load to i32 @@ -1662,7 +1659,6 @@ define i32 @atomic_load_zext_i16(ptr addrspace(2) %ptr) { ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: buffer_wbinvl1_vol ; GFX9-NEXT: s_waitcnt expcnt(0) -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: atomic_load_zext_i16: @@ -1675,7 +1671,6 @@ define i32 @atomic_load_zext_i16(ptr addrspace(2) %ptr) { ; GFX10-NEXT: buffer_gl1_inv ; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_waitcnt expcnt(0) -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: atomic_load_zext_i16: @@ -1701,7 +1696,6 @@ define i32 @atomic_load_zext_i16(ptr addrspace(2) %ptr) { ; GFX11-FAKE16-NEXT: buffer_gl1_inv ; GFX11-FAKE16-NEXT: buffer_gl0_inv ; GFX11-FAKE16-NEXT: s_waitcnt expcnt(0) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %load = load atomic i16, ptr addrspace(2) %ptr seq_cst, align 2 %ext = zext i16 %load to i32 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
