llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-amdgpu Author: Nicolai Hähnle (nhaehnle) <details> <summary>Changes</summary> Some of the flag handling seems a bit inconsistent and dodgy, but this is meant to be a pure refactoring for now. --- **Stack**: - [3/3] #<!-- -->179469 ⬅ - [2/3] #<!-- -->175846 - [1/3] #<!-- -->175845 ⚠️ *Part of a stack created by [spr](https://github.com/nhaehnle/spr). Merging this PR using the GitHub UI may have unexpected results.* --- Full diff: https://github.com/llvm/llvm-project/pull/179469.diff 1 Files Affected: - (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+44-48) ``````````diff diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 2385b2c84e444..989aced9d4430 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -1344,13 +1344,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &CI, MachineFunction &MF, unsigned IntrID) const { - IntrinsicInfo Info; - Info.flags = MachineMemOperand::MONone; + MachineMemOperand::Flags Flags = MachineMemOperand::MONone; if (CI.hasMetadata(LLVMContext::MD_invariant_load)) - Info.flags |= MachineMemOperand::MOInvariant; + Flags |= MachineMemOperand::MOInvariant; if (CI.hasMetadata(LLVMContext::MD_nontemporal)) - Info.flags |= MachineMemOperand::MONonTemporal; - Info.flags |= getTargetMMOFlags(CI); + Flags |= MachineMemOperand::MONonTemporal; + Flags |= getTargetMMOFlags(CI); if (const AMDGPU::RsrcIntrinsic *RsrcIntr = AMDGPU::lookupRsrcIntrinsic(IntrID)) { @@ -1360,6 +1359,15 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, if (ME.doesNotAccessMemory()) return; + bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data; + if (!IsSPrefetch) { + auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1)); + if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE) + Flags |= MachineMemOperand::MOVolatile; + } + Flags |= MachineMemOperand::MODereferenceable; + + IntrinsicInfo Info; // TODO: Should images get their own address space? Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE; @@ -1382,14 +1390,6 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.ptrVal = RsrcArg; } - bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data; - if (!IsSPrefetch) { - auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1)); - if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE) - Info.flags |= MachineMemOperand::MOVolatile; - } - - Info.flags |= MachineMemOperand::MODereferenceable; if (ME.onlyReadsMemory()) { if (RsrcIntr->IsImage) { unsigned MaxNumLanes = 4; @@ -1412,7 +1412,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, // FIXME: What does alignment mean for an image? Info.opc = ISD::INTRINSIC_W_CHAIN; - Info.flags |= MachineMemOperand::MOLoad; + Info.flags = Flags | MachineMemOperand::MOLoad; } else if (ME.onlyWritesMemory()) { Info.opc = ISD::INTRINSIC_VOID; @@ -1425,19 +1425,18 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, } else Info.memVT = getValueType(MF.getDataLayout(), DataTy); - Info.flags |= MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOStore; } else { // Atomic, NoReturn Sampler or prefetch Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN; - Info.flags |= - MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable; - - if (!IsSPrefetch) - Info.flags |= MachineMemOperand::MOStore; switch (IntrID) { default: + Info.flags = Flags | MachineMemOperand::MOLoad; + if (!IsSPrefetch) + Info.flags |= MachineMemOperand::MOStore; + if ((RsrcIntr->IsImage && BaseOpcode->NoReturn) || IsSPrefetch) { // Fake memory access type for no return sampler intrinsics Info.memVT = MVT::i32; @@ -1457,7 +1456,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, // Don't set an offset, since the pointer value always represents the // base of the buffer. Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8); - Info.flags &= ~MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOLoad; Infos.push_back(Info); // Entry 1: Store to LDS. @@ -1469,8 +1468,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.offset = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 2)) ->getZExtValue(); Info.fallbackAddressSpace = AMDGPUAS::LOCAL_ADDRESS; - Info.flags &= ~MachineMemOperand::MOLoad; - Info.flags |= MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOStore; Infos.push_back(Info); return; } @@ -1481,7 +1479,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.memVT = memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(), std::numeric_limits<unsigned>::max()); - Info.flags &= ~MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOLoad; Infos.push_back(Info); return; } @@ -1491,6 +1489,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, return; } + IntrinsicInfo Info; switch (IntrID) { case Intrinsic::amdgcn_ds_ordered_add: case Intrinsic::amdgcn_ds_ordered_swap: { @@ -1498,7 +1497,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.memVT = MVT::getVT(CI.getType()); Info.ptrVal = CI.getOperand(0); Info.align.reset(); - Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore; const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4)); if (!Vol->isZero()) @@ -1523,7 +1522,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.memVT = MVT::getVT(CI.getType()); Info.ptrVal = CI.getOperand(0); Info.align.reset(); - Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore; const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1)); if (!Vol->isZero()) @@ -1542,7 +1541,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.memVT = MVT::i64; Info.size = 8; Info.align.reset(); - Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore; Infos.push_back(Info); return; } @@ -1558,8 +1557,8 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE; Info.align.reset(); - Info.flags |= - MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable; + Info.flags = Flags | MachineMemOperand::MOLoad | + MachineMemOperand::MODereferenceable; Infos.push_back(Info); return; } @@ -1572,9 +1571,9 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.memVT = MVT::getVT(CI.getType()); Info.ptrVal = CI.getOperand(0); Info.align.reset(); - Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore | - MachineMemOperand::MODereferenceable | - MachineMemOperand::MOVolatile; + Info.flags = + Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore | + MachineMemOperand::MODereferenceable | MachineMemOperand::MOVolatile; Infos.push_back(Info); return; } @@ -1603,7 +1602,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.memVT = MVT::getVT(CI.getType()); Info.ptrVal = CI.getOperand(0); Info.align.reset(); - Info.flags |= MachineMemOperand::MOLoad; + Info.flags = Flags | MachineMemOperand::MOLoad; Infos.push_back(Info); return; } @@ -1649,9 +1648,9 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.align = Align(4); if (IntrID == Intrinsic::amdgcn_ds_gws_barrier) - Info.flags |= MachineMemOperand::MOLoad; + Info.flags = Flags | MachineMemOperand::MOLoad; else - Info.flags |= MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOStore; Infos.push_back(Info); return; } @@ -1668,12 +1667,11 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID)); Info.ptrVal = CI.getArgOperand(0); // Global pointer Info.offset = cast<ConstantInt>(CI.getArgOperand(2))->getSExtValue(); - Info.flags |= MachineMemOperand::MOLoad; + Info.flags = Flags | MachineMemOperand::MOLoad; Infos.push_back(Info); // Entry 1: Store to LDS (same offset). - Info.flags &= ~MachineMemOperand::MOLoad; - Info.flags |= MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOStore; Info.ptrVal = CI.getArgOperand(1); // LDS pointer Infos.push_back(Info); return; @@ -1687,12 +1685,11 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID)); Info.ptrVal = CI.getArgOperand(1); // LDS pointer Info.offset = cast<ConstantInt>(CI.getArgOperand(2))->getSExtValue(); - Info.flags |= MachineMemOperand::MOLoad; + Info.flags = Flags | MachineMemOperand::MOLoad; Infos.push_back(Info); // Entry 1: Store to global (same offset). - Info.flags &= ~MachineMemOperand::MOLoad; - Info.flags |= MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOStore; Info.ptrVal = CI.getArgOperand(0); // Global pointer Infos.push_back(Info); return; @@ -1702,15 +1699,15 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue(); auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1)); bool IsVolatile = Aux->getZExtValue() & AMDGPU::CPol::VOLATILE; + if (IsVolatile) + Flags |= MachineMemOperand::MOVolatile; // Entry 0: Load from source (global/flat). Info.opc = ISD::INTRINSIC_VOID; Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8); Info.ptrVal = CI.getArgOperand(0); // Source pointer Info.offset = cast<ConstantInt>(CI.getArgOperand(3))->getSExtValue(); - Info.flags |= MachineMemOperand::MOLoad; - if (IsVolatile) - Info.flags |= MachineMemOperand::MOVolatile; + Info.flags = Flags | MachineMemOperand::MOLoad; Infos.push_back(Info); // Entry 1: Store to LDS. @@ -1719,8 +1716,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8 * Subtarget->getWavefrontSize()); Info.ptrVal = CI.getArgOperand(1); // LDS destination pointer - Info.flags &= ~MachineMemOperand::MOLoad; - Info.flags |= MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOStore; Infos.push_back(Info); return; } @@ -1741,7 +1737,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.size = 4; Info.align = Align(4); - Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore; Infos.push_back(Info); return; } @@ -1751,7 +1747,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Info.opc = ISD::INTRINSIC_VOID; Info.memVT = EVT::getIntegerVT(CI.getContext(), 8); Info.ptrVal = CI.getArgOperand(0); - Info.flags |= MachineMemOperand::MOLoad; + Info.flags = Flags | MachineMemOperand::MOLoad; Infos.push_back(Info); return; } `````````` </details> https://github.com/llvm/llvm-project/pull/179469 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
