https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/238842
>From bbbc670670658eb8d31fb190dfa2c258836e82f4 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Wed, 4 Nov 2026 14:55:18 +0100 Subject: [PATCH 1/2] [AMDGPU] Price buffer fat pointer memory operations Price buffer fat pointer accesses as global accesses, except that an aligned load is not widened and a vector of odd width elements is not split element by element. Without relaxed buffer bounds checking, a vector of elements that are a multiple of 32 bits wide is accessed in pieces no wider than its alignment and no narrower than 4 bytes, and a vector of other byte-sized or power of 2 wide elements in pieces of the largest power of 2 bytes that divides the element size. The width comes from buffer fat pointer lowering, which now exposes it. When the pointer is a buffer resource plus a constant offset, the offset and a constant number of records give the same bounds facts lowering uses to access it in wider pieces. --- llvm/lib/Target/AMDGPU/AMDGPU.h | 9 +++ .../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 73 ++++++++++--------- .../AMDGPU/AMDGPUTargetTransformInfo.cpp | 70 ++++++++++++++---- llvm/test/Analysis/CostModel/AMDGPU/load.ll | 71 +++++++++++++----- llvm/test/Analysis/CostModel/AMDGPU/store.ll | 71 +++++++++++++----- 5 files changed, 205 insertions(+), 89 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h index 75a83c55db2bc9..3fb25890b903ee 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.h +++ b/llvm/lib/Target/AMDGPU/AMDGPU.h @@ -21,6 +21,7 @@ namespace llvm { class AMDGPUTargetMachine; class LazyCallGraph; +class GCNSubtarget; class GCNTargetMachine; class TargetMachine; @@ -208,6 +209,14 @@ struct AMDGPULowerBufferFatPointersPass const TargetMachine &TM; }; +namespace AMDGPU { +/// Return the width in bits of the widest buffer intrinsic that buffer fat +/// pointer lowering uses to access \p Ty with alignment \p A. +uint64_t getMaxBufferIntrinsicWidth(const GCNSubtarget &ST, + const DataLayout &DL, Type *Ty, Align A, + bool NoWrapFromMax, bool NoPartialOOB); +} // namespace AMDGPU + void initializeAMDGPULowerIntrinsicsLegacyPass(PassRegistry &); struct AMDGPULowerIntrinsicsPass diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp index 574d39a2899a51..ca4b987f8a8e1d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp @@ -753,32 +753,6 @@ class LegalizeBufferContentTypesVisitor /// Break up the loads of a struct into the loads of its components - /// Return the maximum allowed load/store width for the given type and - /// alignment combination based on subtarget flags. - /// 1. If unaligned accesses are not enabled, then any load/store that is less - /// than word-aligned has to be handled one byte or ushort at a time. - /// 2. If relaxed OOB mode is not set, we must ensure that the in-bounds - /// part of a partially out of bounds read/write is performed correctly. This - /// means that any load that isn't naturally aligned has to be split into - /// parts that are naturally aligned, so that, after bitcasting, we don't have - /// unaligned loads that could discard valid data. - /// - /// For example, if we're loading a <8 x i8>, that's actually a load of a <2 x - /// i32>, and if we load from an align(2) address, that address might be 2 - /// bytes from the end of the buffer. The hardware will, when performing the - /// <2 x i32> load, mask off the entire first word, causing the two in-bounds - /// bytes to be masked off. However,if we know the offset can't be too close - /// to the number of records in the buffer (if known), we can skip this - /// expansion. - /// - /// Unlike the complete disablement of unaligned accesses from point 1, - /// this does not apply to unaligned scalars, but will apply to cases like - /// `load <2 x i32>, align 4` since the left elemenvt might be out of bounds. - /// Note that if the we know that the base offset is known to be - /// less than `uint32_max - byte_size(Ty)`, we can skip these alignment - /// checks. - uint64_t maxIntrinsicWidth(Type *Ty, Align A, OobProperties OobProps); - /// Convert a vector or scalar type that can't be operated on by buffer /// intrinsics to one that would be legal through bitcasts and/or truncation. /// Uses the wider of i32, i16, or i8 where possible, clamping to the maximum @@ -950,14 +924,39 @@ LegalizeBufferContentTypesVisitor::analyzeOobProperties(Value *Ptr, Type *Ty, return Result; } -uint64_t -LegalizeBufferContentTypesVisitor::maxIntrinsicWidth(Type *T, Align A, - OobProperties OobProps) { +/// Return the maximum allowed load/store width for the given type and +/// alignment combination based on subtarget flags. +/// 1. If unaligned accesses are not enabled, then any load/store that is less +/// than word-aligned has to be handled one byte or ushort at a time. +/// 2. If relaxed OOB mode is not set, we must ensure that the in-bounds +/// part of a partially out of bounds read/write is performed correctly. This +/// means that any load that isn't naturally aligned has to be split into +/// parts that are naturally aligned, so that, after bitcasting, we don't have +/// unaligned loads that could discard valid data. +/// +/// For example, if we're loading a <8 x i8>, that's actually a load of a <2 x +/// i32>, and if we load from an align(2) address, that address might be 2 +/// bytes from the end of the buffer. The hardware will, when performing the +/// <2 x i32> load, mask off the entire first word, causing the two in-bounds +/// bytes to be masked off. However,if we know the offset can't be too close +/// to the number of records in the buffer (if known), we can skip this +/// expansion. +/// +/// Unlike the complete disablement of unaligned accesses from point 1, +/// this does not apply to unaligned scalars, but will apply to cases like +/// `load <2 x i32>, align 4` since the left elemenvt might be out of bounds. +/// Note that if the we know that the base offset is known to be +/// less than `uint32_max - byte_size(Ty)`, we can skip these alignment +/// checks. +uint64_t AMDGPU::getMaxBufferIntrinsicWidth(const GCNSubtarget &ST, + const DataLayout &DL, Type *T, + Align A, bool NoWrapFromMax, + bool NoPartialOOB) { Align Result(16); - if (!ST->hasUnalignedBufferAccessEnabled() && A < Align(4)) + if (!ST.hasUnalignedBufferAccessEnabled() && A < Align(4)) Result = A; auto *VT = dyn_cast<VectorType>(T); - if (!ST->hasRelaxedBufferOOBMode() && VT) { + if (!ST.hasRelaxedBufferOOBMode() && VT) { TypeSize ElemBits = DL.getTypeSizeInBits(VT->getElementType()); if (ElemBits.isKnownMultipleOf(32)) { // Word-sized operations are bounds-checked per word. So, the only case we @@ -965,7 +964,7 @@ LegalizeBufferContentTypesVisitor::maxIntrinsicWidth(Type *T, Align A, // and those can only become in-bounds on a multiple of their alignment. // Therefore, we can use the declared alignment of the operation as the // maximum width, rounding up to 4. - if (!OobProps.NoWrapFromMax) + if (!NoWrapFromMax) Result = std::min(Result, std::max(A, Align(4))); } else if ((ElemBits.isKnownMultipleOf(8) || isPowerOf2_64(ElemBits.getKnownMinValue()))) { @@ -984,7 +983,7 @@ LegalizeBufferContentTypesVisitor::maxIntrinsicWidth(Type *T, Align A, // Strict OOB checking isn't supported if the size of each element is a // non-power-of-2 value less than 8, since there's no feasible way to // apply such a strict bounds check. - if (!OobProps.NoPartialOOB) + if (!NoPartialOOB) Result = commonAlignment(Result, divideCeil(ElemBits.getKnownMinValue(), 8)); } @@ -1211,7 +1210,9 @@ bool LegalizeBufferContentTypesVisitor::visitLoadImpl( Type *ArrayAsVecType = scalarArrayTypeAsVector(PartType); OobProperties OobProps = analyzeOobProperties(OrigLI.getPointerOperand(), PartType, AggByteOff); - uint64_t MaxWidth = maxIntrinsicWidth(ArrayAsVecType, PartAlign, OobProps); + uint64_t MaxWidth = AMDGPU::getMaxBufferIntrinsicWidth( + *ST, DL, ArrayAsVecType, PartAlign, OobProps.NoWrapFromMax, + OobProps.NoPartialOOB); Type *LegalType = legalNonAggregateForMemOp(ArrayAsVecType, MaxWidth); SmallVector<VecSlice> Slices; @@ -1348,7 +1349,9 @@ std::pair<bool, bool> LegalizeBufferContentTypesVisitor::visitStoreImpl( Align PartAlign = commonAlignment(OrigSI.getAlign(), AggByteOff); OobProperties OobProps = analyzeOobProperties(OrigSI.getPointerOperand(), PartType, AggByteOff); - uint64_t MaxWidth = maxIntrinsicWidth(ArrayAsVecType, PartAlign, OobProps); + uint64_t MaxWidth = AMDGPU::getMaxBufferIntrinsicWidth( + *ST, DL, ArrayAsVecType, PartAlign, OobProps.NoWrapFromMax, + OobProps.NoPartialOOB); Type *LegalType = legalNonAggregateForMemOp(ArrayAsVecType, MaxWidth); if (LegalType != ArrayAsVecType) { NewData = makeLegalNonAggregate(NewData, LegalType, Name); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index f390226ef7297d..e59f7aff6e27df 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -2213,6 +2213,46 @@ bool GCNTTIImpl::hasUniformPointerOperand(const Instruction *I) const { return isa<Argument, Constant>(Ptr) && !isSourceOfDivergence(Ptr); } +static std::pair<bool, bool> getKnownBufferOobProperties(const GCNSubtarget &ST, + const DataLayout &DL, + const Instruction *I, + unsigned Bytes) { + const Value *Ptr = I ? getLoadStorePointerOperand(I) : nullptr; + if (!Ptr || + Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER) + return {false, false}; + APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0); + while (const auto *GEP = dyn_cast<GEPOperator>(Ptr)) { + if (!GEP->accumulateConstantOffset(DL, Offset)) + return {false, false}; + Ptr = GEP->getPointerOperand(); + } + const auto *Cast = dyn_cast<AddrSpaceCastInst>(Ptr); + if (Cast) { + if (Cast->getSrcAddressSpace() != AMDGPUAS::BUFFER_RESOURCE) + return {false, false}; + Ptr = Cast->getPointerOperand(); + } + const auto *Rsrc = dyn_cast<IntrinsicInst>(Ptr); + if (Rsrc && Rsrc->getIntrinsicID() != Intrinsic::amdgcn_make_buffer_rsrc) + Rsrc = nullptr; + if (!Cast && !Rsrc) + return {false, false}; + + bool NoWrapFromMax = + Offset.ule(APInt::getAllOnes(Offset.getBitWidth()) - Bytes); + const auto *NumRecords = + Rsrc ? dyn_cast<ConstantInt>(Rsrc->getArgOperand(2)) : nullptr; + std::optional<unsigned> NumRecordsWidth = + ST.getBufferResourceNumRecordsWidth(); + if (!NumRecords || !NumRecordsWidth) + return {NoWrapFromMax, false}; + APInt Bound = NumRecords->getValue().zextOrTrunc(*NumRecordsWidth); + APInt BoundsDiff = Bound.zext(64) - Offset.zext(64); + return {NoWrapFromMax, Bound.isAllOnes() || BoundsDiff.sge(Bytes) || + BoundsDiff.isNonPositive()}; +} + InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, @@ -2228,16 +2268,9 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, unsigned EltBits = std::max<unsigned>(8, DL.getTypeSizeInBits(EltTy)); - if (AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER) { - if (Src->isVectorTy() && EltTy->isIntegerTy(8)) - return divideCeil(DL.getTypeSizeInBits(Src) - 1, - getLoadStoreVecRegBitWidth(AddressSpace)); - return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, - CostKind, OpInfo, I); - } - auto *VecTy = dyn_cast<FixedVectorType>(Src); - if (VecTy && EltBits % 8 == 0 && !isPowerOf2_32(EltBits) && + if (VecTy && AddressSpace != AMDGPUAS::BUFFER_FAT_POINTER && + EltBits % 8 == 0 && !isPowerOf2_32(EltBits) && !TLI->shouldCombineMemoryType(TLI->getValueType(DL, Src))) { Align EltAlign = commonAlignment(Alignment, EltBits / 8); return VecTy->getNumElements() * getMemoryOpCost(Opcode, EltTy, EltAlign, @@ -2288,14 +2321,23 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, HasDwordx3 = HasDwordx3 && (Alignment >= 16 || (Alignment < 4 && HasUnalignedAccess)); } + if (AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER) { + auto [NoWrapFromMax, NoPartialOOB] = + getKnownBufferOobProperties(*ST, DL, I, Bytes); + MaxBytes = AMDGPU::getMaxBufferIntrinsicWidth(*ST, DL, Src, Alignment, + NoWrapFromMax, NoPartialOOB) / + 8; + } if (Alignment < 4 && !HasUnalignedAccess) MaxBytes = std::min<unsigned>(MaxBytes, Alignment.value()); - bool MergesDwords = ST->hasFlatInstOffsets() && - (AMDGPU::isExtendedGlobalAddrSpace(AddressSpace) || - (AddressSpace == AMDGPUAS::FLAT_ADDRESS && - !ST->hasFlatSegmentOffsetBug())); + bool MergesDwords = AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER || + (ST->hasFlatInstOffsets() && + (AMDGPU::isExtendedGlobalAddrSpace(AddressSpace) || + (AddressSpace == AMDGPUAS::FLAT_ADDRESS && + !ST->hasFlatSegmentOffsetBug()))); - if (Opcode == Instruction::Load) { + if (Opcode == Instruction::Load && + AddressSpace != AMDGPUAS::BUFFER_FAT_POINTER) { if (VecTy && Alignment >= PowerOf2Ceil(Bytes)) return divideCeil(Bytes, MaxBytes); if (EltBits <= 16 && Alignment >= 4 && Bytes % 4) { diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll index b36de41ef688ff..b94c98bc461b43 100644 --- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll +++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll @@ -765,21 +765,21 @@ define void @loads_addrspace_7(i32 %arg) { ; CHECK-LABEL: 'loads_addrspace_7' ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i32, ptr addrspace(7) poison, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load i64, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <2 x i32>, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <2 x i32>, ptr addrspace(7) poison, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <2 x i32>, ptr addrspace(7) poison, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <3 x i32>, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <3 x i32>, ptr addrspace(7) poison, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %5 = load <3 x i32>, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <3 x i32>, ptr addrspace(7) poison, align 8 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <3 x i32>, ptr addrspace(7) poison, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i32>, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) poison, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i32>, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %9 = load <4 x i32>, ptr addrspace(7) poison, align 8 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <4 x i32>, ptr addrspace(7) poison, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <2 x i64>, ptr addrspace(7) poison, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <8 x i32>, ptr addrspace(7) poison, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load <4 x i8>, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i16>, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <8 x half>, ptr addrspace(7) poison, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %16 = load <4 x i24>, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %17 = load <3 x i12>, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <2 x i64>, ptr addrspace(7) poison, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = load <8 x i32>, ptr addrspace(7) poison, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %13 = load <4 x i8>, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %14 = load <2 x i16>, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %15 = load <8 x half>, ptr addrspace(7) poison, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %16 = load <4 x i24>, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %17 = load <3 x i12>, ptr addrspace(7) poison, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; load i32, ptr addrspace(7) poison, align 4 @@ -802,16 +802,47 @@ define void @loads_addrspace_7(i32 %arg) { ret void } +define void @loads_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc) { +; CHECK-LABEL: 'loads_addrspace_7_known_bounds' +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <16 x i8>, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <8 x i16>, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012 +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %4 = load <16 x i8>, ptr addrspace(7) %tail, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <16 x i8>, ptr addrspace(7) %end, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <4 x i32>, ptr addrspace(7) %q, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %7 = load <16 x i8>, ptr addrspace(7) %q, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; + %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) + %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + load <4 x i32>, ptr addrspace(7) %p, align 4 + load <16 x i8>, ptr addrspace(7) %p, align 4 + load <8 x i16>, ptr addrspace(7) %p, align 4 + %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012 + load <16 x i8>, ptr addrspace(7) %tail, align 4 + %end = getelementptr i8, ptr addrspace(7) %p, i32 1024 + load <16 x i8>, ptr addrspace(7) %end, align 4 + %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) + load <4 x i32>, ptr addrspace(7) %q, align 4 + load <16 x i8>, ptr addrspace(7) %q, align 4 + ret void +} + define void @loads_addrspace_7_unknown_offset(ptr addrspace(7) %p) { ; CHECK-LABEL: 'loads_addrspace_7_unknown_offset' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i64>, ptr addrspace(7) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <2 x i64>, ptr addrspace(7) %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <3 x i32>, ptr addrspace(7) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <8 x i32>, ptr addrspace(7) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <3 x double>, ptr addrspace(7) %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <8 x i16>, ptr addrspace(7) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %2 = load <2 x i64>, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <2 x i64>, ptr addrspace(7) %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %4 = load <3 x i32>, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %5 = load <8 x i32>, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %6 = load <3 x double>, ptr addrspace(7) %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %7 = load <8 x i16>, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %p, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %p, align 16 ; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; diff --git a/llvm/test/Analysis/CostModel/AMDGPU/store.ll b/llvm/test/Analysis/CostModel/AMDGPU/store.ll index 7465f1bee27ced..99b7c717742aaf 100644 --- a/llvm/test/Analysis/CostModel/AMDGPU/store.ll +++ b/llvm/test/Analysis/CostModel/AMDGPU/store.ll @@ -765,21 +765,21 @@ define void @stores_addrspace_7(i32 %arg) { ; CHECK-LABEL: 'stores_addrspace_7' ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(7) poison, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i64 poison, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 8 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 8 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) poison, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i32> poison, ptr addrspace(7) poison, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> poison, ptr addrspace(7) poison, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: store <4 x i24> poison, ptr addrspace(7) poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <3 x i12> poison, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> poison, ptr addrspace(7) poison, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x i32> poison, ptr addrspace(7) poison, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> poison, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> poison, ptr addrspace(7) poison, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <4 x i24> poison, ptr addrspace(7) poison, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i12> poison, ptr addrspace(7) poison, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; store i32 poison, ptr addrspace(7) poison, align 4 @@ -802,16 +802,47 @@ define void @stores_addrspace_7(i32 %arg) { ret void } -define void @stores_addrspace_7_unknown_offset(ptr addrspace(7) %p) { -; CHECK-LABEL: 'stores_addrspace_7_unknown_offset' +define void @stores_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc) { +; CHECK-LABEL: 'stores_addrspace_7_known_bounds' +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i32> poison, ptr addrspace(7) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x double> poison, ptr addrspace(7) %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012 +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %tail, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %end, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %q, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %q, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; + %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) + %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + store <4 x i32> poison, ptr addrspace(7) %p, align 4 + store <16 x i8> poison, ptr addrspace(7) %p, align 4 + store <8 x i16> poison, ptr addrspace(7) %p, align 4 + %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012 + store <16 x i8> poison, ptr addrspace(7) %tail, align 4 + %end = getelementptr i8, ptr addrspace(7) %p, i32 1024 + store <16 x i8> poison, ptr addrspace(7) %end, align 4 + %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) + store <4 x i32> poison, ptr addrspace(7) %q, align 4 + store <16 x i8> poison, ptr addrspace(7) %q, align 4 + ret void +} + +define void @stores_addrspace_7_unknown_offset(ptr addrspace(7) %p) { +; CHECK-LABEL: 'stores_addrspace_7_unknown_offset' +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x i32> poison, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x double> poison, ptr addrspace(7) %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 16 ; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; >From a5dc7d571eff5e56f7c174b3feff5fabfa44a582 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Wed, 4 Nov 2026 23:07:13 +0100 Subject: [PATCH 2/2] apply comments --- .../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 6 +- .../AMDGPU/AMDGPUTargetTransformInfo.cpp | 12 +- .../AMDGPU/buffer-fat-pointer-oob-mode.ll | 354 ++++++++++++++++++ llvm/test/Analysis/CostModel/AMDGPU/load.ll | 22 +- llvm/test/Analysis/CostModel/AMDGPU/store.ll | 22 +- 5 files changed, 407 insertions(+), 9 deletions(-) create mode 100644 llvm/test/Analysis/CostModel/AMDGPU/buffer-fat-pointer-oob-mode.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp index ca4b987f8a8e1d..3b9ad2971ae127 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp @@ -938,14 +938,14 @@ LegalizeBufferContentTypesVisitor::analyzeOobProperties(Value *Ptr, Type *Ty, /// i32>, and if we load from an align(2) address, that address might be 2 /// bytes from the end of the buffer. The hardware will, when performing the /// <2 x i32> load, mask off the entire first word, causing the two in-bounds -/// bytes to be masked off. However,if we know the offset can't be too close +/// bytes to be masked off. However, if we know the offset can't be too close /// to the number of records in the buffer (if known), we can skip this /// expansion. /// /// Unlike the complete disablement of unaligned accesses from point 1, /// this does not apply to unaligned scalars, but will apply to cases like -/// `load <2 x i32>, align 4` since the left elemenvt might be out of bounds. -/// Note that if the we know that the base offset is known to be +/// `load <2 x i32>, align 4` since the left element might be out of bounds. +/// Note that if we know that the base offset is known to be /// less than `uint32_max - byte_size(Ty)`, we can skip these alignment /// checks. uint64_t AMDGPU::getMaxBufferIntrinsicWidth(const GCNSubtarget &ST, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index e59f7aff6e27df..b8702707380e9b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -2222,9 +2222,9 @@ static std::pair<bool, bool> getKnownBufferOobProperties(const GCNSubtarget &ST, Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER) return {false, false}; APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0); + bool KnownOffset = true; while (const auto *GEP = dyn_cast<GEPOperator>(Ptr)) { - if (!GEP->accumulateConstantOffset(DL, Offset)) - return {false, false}; + KnownOffset = KnownOffset && GEP->accumulateConstantOffset(DL, Offset); Ptr = GEP->getPointerOperand(); } const auto *Cast = dyn_cast<AddrSpaceCastInst>(Ptr); @@ -2240,6 +2240,7 @@ static std::pair<bool, bool> getKnownBufferOobProperties(const GCNSubtarget &ST, return {false, false}; bool NoWrapFromMax = + KnownOffset && Offset.ule(APInt::getAllOnes(Offset.getBitWidth()) - Bytes); const auto *NumRecords = Rsrc ? dyn_cast<ConstantInt>(Rsrc->getArgOperand(2)) : nullptr; @@ -2248,9 +2249,12 @@ static std::pair<bool, bool> getKnownBufferOobProperties(const GCNSubtarget &ST, if (!NumRecords || !NumRecordsWidth) return {NoWrapFromMax, false}; APInt Bound = NumRecords->getValue().zextOrTrunc(*NumRecordsWidth); + if (Bound.isAllOnes()) + return {NoWrapFromMax, true}; + if (!KnownOffset) + return {false, false}; APInt BoundsDiff = Bound.zext(64) - Offset.zext(64); - return {NoWrapFromMax, Bound.isAllOnes() || BoundsDiff.sge(Bytes) || - BoundsDiff.isNonPositive()}; + return {NoWrapFromMax, BoundsDiff.sge(Bytes) || BoundsDiff.isNonPositive()}; } InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, diff --git a/llvm/test/Analysis/CostModel/AMDGPU/buffer-fat-pointer-oob-mode.ll b/llvm/test/Analysis/CostModel/AMDGPU/buffer-fat-pointer-oob-mode.ll new file mode 100644 index 00000000000000..80811bfaf3f2ef --- /dev/null +++ b/llvm/test/Analysis/CostModel/AMDGPU/buffer-fat-pointer-oob-mode.ll @@ -0,0 +1,354 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py +; RUN: sed 's/BUFFER_OOB_MODE/2/' %s | opt -passes="print<cost-model>" -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa 2>&1 | FileCheck %s --check-prefix=STRICT +; RUN: sed 's/BUFFER_OOB_MODE/1/' %s | opt -passes="print<cost-model>" -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa 2>&1 | FileCheck %s --check-prefix=RELAXED + +define void @loads() { +; STRICT-LABEL: 'loads' +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i32, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load i64, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <2 x i32>, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <2 x i32>, ptr addrspace(7) poison, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %5 = load <3 x i32>, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <3 x i32>, ptr addrspace(7) poison, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <3 x i32>, ptr addrspace(7) poison, align 16 +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i32>, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %9 = load <4 x i32>, ptr addrspace(7) poison, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <4 x i32>, ptr addrspace(7) poison, align 16 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <2 x i64>, ptr addrspace(7) poison, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = load <8 x i32>, ptr addrspace(7) poison, align 32 +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %13 = load <4 x i8>, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %14 = load <2 x i16>, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %15 = load <8 x half>, ptr addrspace(7) poison, align 16 +; STRICT-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %16 = load <4 x i24>, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %17 = load <3 x i12>, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; RELAXED-LABEL: 'loads' +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i32, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load i64, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <2 x i32>, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <2 x i32>, ptr addrspace(7) poison, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <3 x i32>, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <3 x i32>, ptr addrspace(7) poison, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <3 x i32>, ptr addrspace(7) poison, align 16 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i32>, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) poison, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <4 x i32>, ptr addrspace(7) poison, align 16 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <2 x i64>, ptr addrspace(7) poison, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = load <8 x i32>, ptr addrspace(7) poison, align 32 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load <4 x i8>, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i16>, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <8 x half>, ptr addrspace(7) poison, align 16 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i24>, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %17 = load <3 x i12>, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; + load i32, ptr addrspace(7) poison, align 4 + load i64, ptr addrspace(7) poison, align 4 + load <2 x i32>, ptr addrspace(7) poison, align 4 + load <2 x i32>, ptr addrspace(7) poison, align 8 + load <3 x i32>, ptr addrspace(7) poison, align 4 + load <3 x i32>, ptr addrspace(7) poison, align 8 + load <3 x i32>, ptr addrspace(7) poison, align 16 + load <4 x i32>, ptr addrspace(7) poison, align 4 + load <4 x i32>, ptr addrspace(7) poison, align 8 + load <4 x i32>, ptr addrspace(7) poison, align 16 + load <2 x i64>, ptr addrspace(7) poison, align 8 + load <8 x i32>, ptr addrspace(7) poison, align 32 + load <4 x i8>, ptr addrspace(7) poison, align 4 + load <2 x i16>, ptr addrspace(7) poison, align 4 + load <8 x half>, ptr addrspace(7) poison, align 16 + load <4 x i24>, ptr addrspace(7) poison, align 4 + load <3 x i12>, ptr addrspace(7) poison, align 4 + ret void +} + +define void @loads_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc, i32 %off) { +; STRICT-LABEL: 'loads_known_bounds' +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <16 x i8>, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <8 x i16>, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012 +; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %4 = load <16 x i8>, ptr addrspace(7) %tail, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <16 x i8>, ptr addrspace(7) %end, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <4 x i32>, ptr addrspace(7) %q, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %7 = load <16 x i8>, ptr addrspace(7) %q, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744) +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7) +; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %u.var, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %u.var2, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off +; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %10 = load <16 x i8>, ptr addrspace(7) %p.var, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <16 x i8>, ptr addrspace(7) %direct, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; RELAXED-LABEL: 'loads_known_bounds' +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <16 x i8>, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <8 x i16>, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <16 x i8>, ptr addrspace(7) %tail, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <16 x i8>, ptr addrspace(7) %end, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <4 x i32>, ptr addrspace(7) %q, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i8>, ptr addrspace(7) %q, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %u.var, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %u.var2, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <16 x i8>, ptr addrspace(7) %p.var, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <16 x i8>, ptr addrspace(7) %direct, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; + %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) + %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + load <4 x i32>, ptr addrspace(7) %p, align 4 + load <16 x i8>, ptr addrspace(7) %p, align 4 + load <8 x i16>, ptr addrspace(7) %p, align 4 + %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012 + load <16 x i8>, ptr addrspace(7) %tail, align 4 + %end = getelementptr i8, ptr addrspace(7) %p, i32 1024 + load <16 x i8>, ptr addrspace(7) %end, align 4 + %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) + load <4 x i32>, ptr addrspace(7) %q, align 4 + load <16 x i8>, ptr addrspace(7) %q, align 4 + %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744) + %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7) + %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off + load <16 x i8>, ptr addrspace(7) %u.var, align 4 + %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off + load <4 x i32>, ptr addrspace(7) %u.var2, align 4 + %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off + load <16 x i8>, ptr addrspace(7) %p.var, align 4 + %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) + load <16 x i8>, ptr addrspace(7) %direct, align 4 + ret void +} + +define void @loads_unknown_offset(ptr addrspace(7) %p) { +; STRICT-LABEL: 'loads_unknown_offset' +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %2 = load <2 x i64>, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <2 x i64>, ptr addrspace(7) %p, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %4 = load <3 x i32>, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %5 = load <8 x i32>, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %6 = load <3 x double>, ptr addrspace(7) %p, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %7 = load <8 x i16>, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %p, align 16 +; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; RELAXED-LABEL: 'loads_unknown_offset' +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i64>, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <2 x i64>, ptr addrspace(7) %p, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <3 x i32>, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <8 x i32>, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <3 x double>, ptr addrspace(7) %p, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <8 x i16>, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %p, align 16 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; + load <4 x i32>, ptr addrspace(7) %p, align 4 + load <2 x i64>, ptr addrspace(7) %p, align 4 + load <2 x i64>, ptr addrspace(7) %p, align 8 + load <3 x i32>, ptr addrspace(7) %p, align 4 + load <8 x i32>, ptr addrspace(7) %p, align 4 + load <3 x double>, ptr addrspace(7) %p, align 8 + load <8 x i16>, ptr addrspace(7) %p, align 4 + load <16 x i8>, ptr addrspace(7) %p, align 4 + load <4 x i32>, ptr addrspace(7) %p, align 16 + ret void +} + +define void @stores() { +; STRICT-LABEL: 'stores' +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i64 poison, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 16 +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 16 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> poison, ptr addrspace(7) poison, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x i32> poison, ptr addrspace(7) poison, align 32 +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> poison, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> poison, ptr addrspace(7) poison, align 16 +; STRICT-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <4 x i24> poison, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i12> poison, ptr addrspace(7) poison, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; RELAXED-LABEL: 'stores' +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i64 poison, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 16 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 16 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) poison, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x i32> poison, ptr addrspace(7) poison, align 32 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> poison, ptr addrspace(7) poison, align 16 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i24> poison, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i12> poison, ptr addrspace(7) poison, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; + store i32 poison, ptr addrspace(7) poison, align 4 + store i64 poison, ptr addrspace(7) poison, align 4 + store <2 x i32> poison, ptr addrspace(7) poison, align 4 + store <2 x i32> poison, ptr addrspace(7) poison, align 8 + store <3 x i32> poison, ptr addrspace(7) poison, align 4 + store <3 x i32> poison, ptr addrspace(7) poison, align 8 + store <3 x i32> poison, ptr addrspace(7) poison, align 16 + store <4 x i32> poison, ptr addrspace(7) poison, align 4 + store <4 x i32> poison, ptr addrspace(7) poison, align 8 + store <4 x i32> poison, ptr addrspace(7) poison, align 16 + store <2 x i64> poison, ptr addrspace(7) poison, align 8 + store <8 x i32> poison, ptr addrspace(7) poison, align 32 + store <4 x i8> poison, ptr addrspace(7) poison, align 4 + store <2 x i16> poison, ptr addrspace(7) poison, align 4 + store <8 x half> poison, ptr addrspace(7) poison, align 16 + store <4 x i24> poison, ptr addrspace(7) poison, align 4 + store <3 x i12> poison, ptr addrspace(7) poison, align 4 + ret void +} + +define void @stores_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc, i32 %off) { +; STRICT-LABEL: 'stores_known_bounds' +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012 +; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %tail, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %end, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %q, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %q, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744) +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7) +; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %u.var, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) %u.var2, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off +; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %p.var, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %direct, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; RELAXED-LABEL: 'stores_known_bounds' +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %tail, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %end, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %q, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %q, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %u.var, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %u.var2, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %p.var, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %direct, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; + %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) + %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + store <4 x i32> poison, ptr addrspace(7) %p, align 4 + store <16 x i8> poison, ptr addrspace(7) %p, align 4 + store <8 x i16> poison, ptr addrspace(7) %p, align 4 + %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012 + store <16 x i8> poison, ptr addrspace(7) %tail, align 4 + %end = getelementptr i8, ptr addrspace(7) %p, i32 1024 + store <16 x i8> poison, ptr addrspace(7) %end, align 4 + %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) + store <4 x i32> poison, ptr addrspace(7) %q, align 4 + store <16 x i8> poison, ptr addrspace(7) %q, align 4 + %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744) + %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7) + %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off + store <16 x i8> poison, ptr addrspace(7) %u.var, align 4 + %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off + store <4 x i32> poison, ptr addrspace(7) %u.var2, align 4 + %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off + store <16 x i8> poison, ptr addrspace(7) %p.var, align 4 + %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) + store <16 x i8> poison, ptr addrspace(7) %direct, align 4 + ret void +} + +define void @stores_unknown_offset(ptr addrspace(7) %p) { +; STRICT-LABEL: 'stores_unknown_offset' +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x i32> poison, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x double> poison, ptr addrspace(7) %p, align 8 +; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4 +; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 16 +; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; RELAXED-LABEL: 'stores_unknown_offset' +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x i32> poison, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x double> poison, ptr addrspace(7) %p, align 8 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 16 +; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; + store <4 x i32> poison, ptr addrspace(7) %p, align 4 + store <2 x i64> poison, ptr addrspace(7) %p, align 4 + store <2 x i64> poison, ptr addrspace(7) %p, align 8 + store <3 x i32> poison, ptr addrspace(7) %p, align 4 + store <8 x i32> poison, ptr addrspace(7) %p, align 4 + store <3 x double> poison, ptr addrspace(7) %p, align 8 + store <8 x i16> poison, ptr addrspace(7) %p, align 4 + store <16 x i8> poison, ptr addrspace(7) %p, align 4 + store <4 x i32> poison, ptr addrspace(7) %p, align 16 + ret void +} + +!llvm.module.flags = !{!0} +!0 = !{i32 7, !"amdgpu.buffer.oob.mode", i32 BUFFER_OOB_MODE} diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll index b94c98bc461b43..ed800d3d458400 100644 --- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll +++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll @@ -802,7 +802,7 @@ define void @loads_addrspace_7(i32 %arg) { ret void } -define void @loads_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc) { +define void @loads_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc, i32 %off) { ; CHECK-LABEL: 'loads_addrspace_7_known_bounds' ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) @@ -816,6 +816,16 @@ define void @loads_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspac ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <4 x i32>, ptr addrspace(7) %q, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %7 = load <16 x i8>, ptr addrspace(7) %q, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %u.var, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %u.var2, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %10 = load <16 x i8>, ptr addrspace(7) %p.var, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <16 x i8>, ptr addrspace(7) %direct, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) @@ -830,6 +840,16 @@ define void @loads_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspac %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) load <4 x i32>, ptr addrspace(7) %q, align 4 load <16 x i8>, ptr addrspace(7) %q, align 4 + %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744) + %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7) + %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off + load <16 x i8>, ptr addrspace(7) %u.var, align 4 + %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off + load <4 x i32>, ptr addrspace(7) %u.var2, align 4 + %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off + load <16 x i8>, ptr addrspace(7) %p.var, align 4 + %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) + load <16 x i8>, ptr addrspace(7) %direct, align 4 ret void } diff --git a/llvm/test/Analysis/CostModel/AMDGPU/store.ll b/llvm/test/Analysis/CostModel/AMDGPU/store.ll index 99b7c717742aaf..eb992b307ee576 100644 --- a/llvm/test/Analysis/CostModel/AMDGPU/store.ll +++ b/llvm/test/Analysis/CostModel/AMDGPU/store.ll @@ -802,7 +802,7 @@ define void @stores_addrspace_7(i32 %arg) { ret void } -define void @stores_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc) { +define void @stores_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc, i32 %off) { ; CHECK-LABEL: 'stores_addrspace_7_known_bounds' ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) @@ -816,6 +816,16 @@ define void @stores_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspa ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %q, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %q, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %u.var, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) %u.var2, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %p.var, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %direct, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) @@ -830,6 +840,16 @@ define void @stores_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspa %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7) store <4 x i32> poison, ptr addrspace(7) %q, align 4 store <16 x i8> poison, ptr addrspace(7) %q, align 4 + %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744) + %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7) + %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off + store <16 x i8> poison, ptr addrspace(7) %u.var, align 4 + %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off + store <4 x i32> poison, ptr addrspace(7) %u.var2, align 4 + %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off + store <16 x i8> poison, ptr addrspace(7) %p.var, align 4 + %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744) + store <16 x i8> poison, ptr addrspace(7) %direct, align 4 ret void } _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
