https://github.com/MrSidims created https://github.com/llvm/llvm-project/pull/233262
Without unaligned access for the address space, an access aligned below 4 bytes is split into pieces of its alignment. LDS accesses aligned to 4 use 8-byte pieces, and take a 12-byte piece only when aligned to 16 or when unaligned DS access is enabled. --- <sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub> >From 1769ec8ee986e9d77e979bcd894f113b83c82666 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Fri, 9 Oct 2026 10:56:06 +0200 Subject: [PATCH] [AMDGPU] Price under-aligned memory operations Without unaligned access for the address space, an access aligned below 4 bytes is split into pieces of its alignment. LDS accesses aligned to 4 use 8-byte pieces, and take a 12-byte piece only when aligned to 16 or when unaligned DS access is enabled. --- .../AMDGPU/AMDGPUTargetTransformInfo.cpp | 11 +- llvm/test/Analysis/CostModel/AMDGPU/load.ll | 81 ++++---- llvm/test/Analysis/CostModel/AMDGPU/store.ll | 81 ++++---- .../SLPVectorizer/AMDGPU/packed-math.ll | 178 +++++++++++++----- .../SLPVectorizer/AMDGPU/vectorize-i8.ll | 24 ++- 5 files changed, 259 insertions(+), 116 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index bfb455ec758e6c..05726165ba1e44 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -2157,21 +2157,30 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Bytes = DL.getTypeStoreSize(Src); unsigned MaxBytes = 16; bool HasDwordx3 = ST->hasDwordx3LoadStores(); + bool HasUnalignedAccess = ST->hasUnalignedBufferAccessEnabled(); switch (AddressSpace) { case AMDGPUAS::PRIVATE_ADDRESS: MaxBytes = ST->getMaxPrivateElementSize(); + HasUnalignedAccess = ST->hasUnalignedScratchAccessEnabled(); break; case AMDGPUAS::FLAT_ADDRESS: if (!ST->hasMultiDwordFlatScratchAddressing()) MaxBytes = ST->getMaxPrivateElementSize(); + HasUnalignedAccess = ST->hasUnalignedScratchAccessEnabled(); break; case AMDGPUAS::LOCAL_ADDRESS: case AMDGPUAS::REGION_ADDRESS: - HasDwordx3 = ST->hasDS96AndDS128(); + HasUnalignedAccess = ST->hasUnalignedDSAccessEnabled(); + if (Alignment >= 4 && Alignment < 8) + MaxBytes = 8; + HasDwordx3 = ST->hasDS96AndDS128() && + (Alignment >= 16 || (Alignment < 4 && HasUnalignedAccess)); break; default: break; } + if (Alignment < 4 && !HasUnalignedAccess) + MaxBytes = std::min<unsigned>(MaxBytes, Alignment.value()); if (Opcode == Instruction::Load) { if (Alignment >= PowerOf2Ceil(Bytes)) diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll index 073e504080c49c..52e5e31eeb93f1 100644 --- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll +++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll @@ -21,20 +21,35 @@ define void @loads_i1(i32 %arg) { } define void @loads_i8(i32 %arg) { -; CHECK-LABEL: 'loads_i8' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr poison, align 1 -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i8>, ptr poison, align 1 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i8>, ptr poison, align 1 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; GFX8-LABEL: 'loads_i8' +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i8>, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %7 = load <3 x i8>, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i8>, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; GFX90APLUS-LABEL: 'loads_i8' +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr poison, align 1 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i8>, ptr poison, align 1 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i8>, ptr poison, align 1 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; load i8, ptr poison load <2 x i8>, ptr poison @@ -60,10 +75,10 @@ define void @loads_i16(i32 %arg) { ; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i16>, ptr poison, align 4 ; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i16>, ptr poison, align 8 ; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i16>, ptr poison, align 8 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i16, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i16>, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i16>, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %8 = load <4 x i16>, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load i16, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load <2 x i16>, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %7 = load <3 x i16>, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %8 = load <4 x i16>, ptr poison, align 1 ; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i16, ptr poison, align 4 ; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i16>, ptr poison, align 4 ; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <3 x i16>, ptr poison, align 4 @@ -122,10 +137,10 @@ define void @loads_i32(i32 %arg) { ; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <2 x i32>, ptr poison, align 8 ; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %3 = load <3 x i32>, ptr poison, align 16 ; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %4 = load <4 x i32>, ptr poison, align 16 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i32, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i32>, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %7 = load <3 x i32>, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i32>, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load i32, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %6 = load <2 x i32>, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %7 = load <3 x i32>, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %8 = load <4 x i32>, ptr poison, align 1 ; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i32, ptr poison, align 4 ; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = load <2 x i32>, ptr poison, align 4 ; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %11 = load <3 x i32>, ptr poison, align 4 @@ -532,26 +547,26 @@ define void @loads_addrspace_5(i32 %arg) { define void @loads_underaligned(i32 %arg) { ; GFX8-LABEL: 'loads_underaligned' -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4 -; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1 ; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; GFX90A-LABEL: 'loads_underaligned' ; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1 -; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4 -; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8 -; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4 ; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1 ; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; GFX12-LABEL: 'loads_underaligned' ; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1 -; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4 -; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8 -; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4 ; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1 ; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; diff --git a/llvm/test/Analysis/CostModel/AMDGPU/store.ll b/llvm/test/Analysis/CostModel/AMDGPU/store.ll index aa9cbd92fad559..a72eddfa70796b 100644 --- a/llvm/test/Analysis/CostModel/AMDGPU/store.ll +++ b/llvm/test/Analysis/CostModel/AMDGPU/store.ll @@ -20,20 +20,35 @@ define void @stores_i1(i32 %arg) { } define void @stores_i8(i32 %arg) { -; CHECK-LABEL: 'stores_i8' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2 -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 1 -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 1 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 1 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; GFX8-LABEL: 'stores_i8' +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i8> poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; GFX90APLUS-LABEL: 'stores_i8' +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 1 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 1 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 1 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4 +; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; store i8 poison, ptr poison store <2 x i8> poison, ptr poison @@ -59,10 +74,10 @@ define void @stores_i16(i32 %arg) { ; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4 ; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8 ; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> poison, ptr poison, align 8 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i16 poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <2 x i16> poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <3 x i16> poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <4 x i16> poison, ptr poison, align 1 ; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 4 ; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4 ; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 4 @@ -121,10 +136,10 @@ define void @stores_i32(i32 %arg) { ; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr poison, align 8 ; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr poison, align 16 ; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr poison, align 16 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store i32 poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <2 x i32> poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <3 x i32> poison, ptr poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <4 x i32> poison, ptr poison, align 1 ; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4 ; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr poison, align 4 ; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr poison, align 4 @@ -534,26 +549,26 @@ define void @stores_addrspace_5(i32 %arg) { define void @stores_underaligned(i32 %arg) { ; GFX8-LABEL: 'stores_underaligned' -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4 -; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1 ; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; GFX90A-LABEL: 'stores_underaligned' ; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1 -; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4 -; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8 -; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4 ; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1 ; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; GFX12-LABEL: 'stores_underaligned' ; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1 -; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4 -; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8 -; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4 ; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1 ; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll index 49d7ee569c8711..32a5ac77bd3472 100644 --- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll +++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll @@ -72,13 +72,30 @@ define amdgpu_kernel void @test1_as_0_0_3_v2f16(ptr %a, ptr %b, ptr addrspace(3) } define amdgpu_kernel void @test1_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) { -; GCN-LABEL: @test1_fma_v2f16( -; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 -; GCN-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2 -; GCN-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2 -; GCN-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP2]], <2 x half> [[TMP3]]) -; GCN-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[D:%.*]], align 2 -; GCN-NEXT: ret void +; GFX9-LABEL: @test1_fma_v2f16( +; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 +; GFX9-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2 +; GFX9-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2 +; GFX9-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP2]], <2 x half> [[TMP3]]) +; GFX9-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[D:%.*]], align 2 +; GFX9-NEXT: ret void +; +; VI-LABEL: @test1_fma_v2f16( +; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2 +; VI-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2 +; VI-NEXT: [[I2:%.*]] = load half, ptr addrspace(3) [[C:%.*]], align 2 +; VI-NEXT: [[FMA0:%.*]] = call half @llvm.fma.f16(half [[I0]], half [[I1]], half [[I2]]) +; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1 +; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2 +; VI-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1 +; VI-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2 +; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1 +; VI-NEXT: [[I5:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX5]], align 2 +; VI-NEXT: [[FMA1:%.*]] = call half @llvm.fma.f16(half [[I3]], half [[I4]], half [[I5]]) +; VI-NEXT: store half [[FMA0]], ptr addrspace(3) [[D:%.*]], align 2 +; VI-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[D]], i64 1 +; VI-NEXT: store half [[FMA1]], ptr addrspace(3) [[ARRAYIDX6]], align 2 +; VI-NEXT: ret void ; %i0 = load half, ptr addrspace(3) %a, align 2 %i1 = load half, ptr addrspace(3) %b, align 2 @@ -98,13 +115,24 @@ define amdgpu_kernel void @test1_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3) } define amdgpu_kernel void @mul_scalar_v2f16(ptr addrspace(3) %a, half %scalar, ptr addrspace(3) %c) { -; GCN-LABEL: @mul_scalar_v2f16( -; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 -; GCN-NEXT: [[TMP2:%.*]] = insertelement <2 x half> poison, half [[SCALAR:%.*]], i64 0 -; GCN-NEXT: [[TMP3:%.*]] = shufflevector <2 x half> [[TMP2]], <2 x half> poison, <2 x i32> zeroinitializer -; GCN-NEXT: [[TMP4:%.*]] = fmul <2 x half> [[TMP1]], [[TMP3]] -; GCN-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[C:%.*]], align 2 -; GCN-NEXT: ret void +; GFX9-LABEL: @mul_scalar_v2f16( +; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 +; GFX9-NEXT: [[TMP2:%.*]] = insertelement <2 x half> poison, half [[SCALAR:%.*]], i64 0 +; GFX9-NEXT: [[TMP3:%.*]] = shufflevector <2 x half> [[TMP2]], <2 x half> poison, <2 x i32> zeroinitializer +; GFX9-NEXT: [[TMP4:%.*]] = fmul <2 x half> [[TMP1]], [[TMP3]] +; GFX9-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[C:%.*]], align 2 +; GFX9-NEXT: ret void +; +; VI-LABEL: @mul_scalar_v2f16( +; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2 +; VI-NEXT: [[MUL:%.*]] = fmul half [[I0]], [[SCALAR:%.*]] +; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1 +; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2 +; VI-NEXT: [[MUL5:%.*]] = fmul half [[I3]], [[SCALAR]] +; VI-NEXT: store half [[MUL]], ptr addrspace(3) [[C:%.*]], align 2 +; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1 +; VI-NEXT: store half [[MUL5]], ptr addrspace(3) [[ARRAYIDX5]], align 2 +; VI-NEXT: ret void ; %i0 = load half, ptr addrspace(3) %a, align 2 %mul = fmul half %i0, %scalar @@ -118,11 +146,22 @@ define amdgpu_kernel void @mul_scalar_v2f16(ptr addrspace(3) %a, half %scalar, p } define amdgpu_kernel void @fabs_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %c) { -; GCN-LABEL: @fabs_v2f16( -; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 -; GCN-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]]) -; GCN-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2 -; GCN-NEXT: ret void +; GFX9-LABEL: @fabs_v2f16( +; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 +; GFX9-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]]) +; GFX9-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2 +; GFX9-NEXT: ret void +; +; VI-LABEL: @fabs_v2f16( +; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2 +; VI-NEXT: [[FABS0:%.*]] = call half @llvm.fabs.f16(half [[I0]]) +; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1 +; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2 +; VI-NEXT: [[FABS1:%.*]] = call half @llvm.fabs.f16(half [[I3]]) +; VI-NEXT: store half [[FABS0]], ptr addrspace(3) [[C:%.*]], align 2 +; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1 +; VI-NEXT: store half [[FABS1]], ptr addrspace(3) [[ARRAYIDX5]], align 2 +; VI-NEXT: ret void ; %i0 = load half, ptr addrspace(3) %a, align 2 %fabs0 = call half @llvm.fabs.f16(half %i0) @@ -136,14 +175,33 @@ define amdgpu_kernel void @fabs_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %c) } define amdgpu_kernel void @test1_fabs_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) { -; GCN-LABEL: @test1_fabs_fma_v2f16( -; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 -; GCN-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2 -; GCN-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2 -; GCN-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]]) -; GCN-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP4]], <2 x half> [[TMP2]], <2 x half> [[TMP3]]) -; GCN-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2 -; GCN-NEXT: ret void +; GFX9-LABEL: @test1_fabs_fma_v2f16( +; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 +; GFX9-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2 +; GFX9-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2 +; GFX9-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]]) +; GFX9-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP4]], <2 x half> [[TMP2]], <2 x half> [[TMP3]]) +; GFX9-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2 +; GFX9-NEXT: ret void +; +; VI-LABEL: @test1_fabs_fma_v2f16( +; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2 +; VI-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2 +; VI-NEXT: [[I2:%.*]] = load half, ptr addrspace(3) [[C:%.*]], align 2 +; VI-NEXT: [[I0_FABS:%.*]] = call half @llvm.fabs.f16(half [[I0]]) +; VI-NEXT: [[FMA0:%.*]] = call half @llvm.fma.f16(half [[I0_FABS]], half [[I1]], half [[I2]]) +; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1 +; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2 +; VI-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1 +; VI-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2 +; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1 +; VI-NEXT: [[I5:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX5]], align 2 +; VI-NEXT: [[I3_FABS:%.*]] = call half @llvm.fabs.f16(half [[I3]]) +; VI-NEXT: [[FMA1:%.*]] = call half @llvm.fma.f16(half [[I3_FABS]], half [[I4]], half [[I5]]) +; VI-NEXT: store half [[FMA0]], ptr addrspace(3) [[D:%.*]], align 2 +; VI-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[D]], i64 1 +; VI-NEXT: store half [[FMA1]], ptr addrspace(3) [[ARRAYIDX6]], align 2 +; VI-NEXT: ret void ; %i0 = load half, ptr addrspace(3) %a, align 2 %i1 = load half, ptr addrspace(3) %b, align 2 @@ -167,18 +225,36 @@ define amdgpu_kernel void @test1_fabs_fma_v2f16(ptr addrspace(3) %a, ptr addrspa } define amdgpu_kernel void @test1_fabs_scalar_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) { -; GCN-LABEL: @test1_fabs_scalar_fma_v2f16( -; GCN-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2 -; GCN-NEXT: [[I1_FABS:%.*]] = call half @llvm.fabs.f16(half [[I1]]) -; GCN-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1 -; GCN-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2 -; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 -; GCN-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2 -; GCN-NEXT: [[TMP3:%.*]] = insertelement <2 x half> poison, half [[I1_FABS]], i64 0 -; GCN-NEXT: [[TMP4:%.*]] = insertelement <2 x half> [[TMP3]], half [[I4]], i64 1 -; GCN-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP4]], <2 x half> [[TMP2]]) -; GCN-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2 -; GCN-NEXT: ret void +; GFX9-LABEL: @test1_fabs_scalar_fma_v2f16( +; GFX9-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2 +; GFX9-NEXT: [[I1_FABS:%.*]] = call half @llvm.fabs.f16(half [[I1]]) +; GFX9-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1 +; GFX9-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2 +; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 +; GFX9-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2 +; GFX9-NEXT: [[TMP3:%.*]] = insertelement <2 x half> poison, half [[I1_FABS]], i64 0 +; GFX9-NEXT: [[TMP4:%.*]] = insertelement <2 x half> [[TMP3]], half [[I4]], i64 1 +; GFX9-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP4]], <2 x half> [[TMP2]]) +; GFX9-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2 +; GFX9-NEXT: ret void +; +; VI-LABEL: @test1_fabs_scalar_fma_v2f16( +; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2 +; VI-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2 +; VI-NEXT: [[I2:%.*]] = load half, ptr addrspace(3) [[C:%.*]], align 2 +; VI-NEXT: [[I1_FABS:%.*]] = call half @llvm.fabs.f16(half [[I1]]) +; VI-NEXT: [[FMA0:%.*]] = call half @llvm.fma.f16(half [[I0]], half [[I1_FABS]], half [[I2]]) +; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1 +; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2 +; VI-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1 +; VI-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2 +; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1 +; VI-NEXT: [[I5:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX5]], align 2 +; VI-NEXT: [[FMA1:%.*]] = call half @llvm.fma.f16(half [[I3]], half [[I4]], half [[I5]]) +; VI-NEXT: store half [[FMA0]], ptr addrspace(3) [[D:%.*]], align 2 +; VI-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[D]], i64 1 +; VI-NEXT: store half [[FMA1]], ptr addrspace(3) [[ARRAYIDX6]], align 2 +; VI-NEXT: ret void ; %i0 = load half, ptr addrspace(3) %a, align 2 %i1 = load half, ptr addrspace(3) %b, align 2 @@ -200,11 +276,22 @@ define amdgpu_kernel void @test1_fabs_scalar_fma_v2f16(ptr addrspace(3) %a, ptr } define amdgpu_kernel void @canonicalize_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %c) { -; GCN-LABEL: @canonicalize_v2f16( -; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 -; GCN-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> [[TMP1]]) -; GCN-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2 -; GCN-NEXT: ret void +; GFX9-LABEL: @canonicalize_v2f16( +; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2 +; GFX9-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> [[TMP1]]) +; GFX9-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2 +; GFX9-NEXT: ret void +; +; VI-LABEL: @canonicalize_v2f16( +; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2 +; VI-NEXT: [[CANONICALIZE0:%.*]] = call half @llvm.canonicalize.f16(half [[I0]]) +; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1 +; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2 +; VI-NEXT: [[CANONICALIZE1:%.*]] = call half @llvm.canonicalize.f16(half [[I3]]) +; VI-NEXT: store half [[CANONICALIZE0]], ptr addrspace(3) [[C:%.*]], align 2 +; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1 +; VI-NEXT: store half [[CANONICALIZE1]], ptr addrspace(3) [[ARRAYIDX5]], align 2 +; VI-NEXT: ret void ; %i0 = load half, ptr addrspace(3) %a, align 2 %canonicalize0 = call half @llvm.canonicalize.f16(half %i0) @@ -223,6 +310,3 @@ declare half @llvm.canonicalize.f16(half) #1 attributes #0 = { nounwind } attributes #1 = { nounwind readnone } -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; GFX9: {{.*}} -; VI: {{.*}} diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll index 414ffe6cbec23e..4d20feaa6b94f5 100644 --- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll +++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll @@ -787,6 +787,10 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in ; GFX7-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop( ; GFX7-SAME: <4 x i8> [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr addrspace(3) [[OUT1:%.*]], i32 [[FLAG:%.*]]) { ; GFX7-NEXT: [[ENTRY:.*]]: +; GFX7-NEXT: [[OTHERELE0:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 0 +; GFX7-NEXT: [[OTHERELE1:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 1 +; GFX7-NEXT: [[OTHERELE2:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 2 +; GFX7-NEXT: [[OTHERELE3:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 3 ; GFX7-NEXT: br label %[[DO_BODY:.*]] ; GFX7: [[DO_BODY]]: ; GFX7-NEXT: [[TMP0:%.*]] = phi <4 x i8> [ [[INPTR0]], %[[ENTRY]] ], [ [[INPTR0]], %[[DO_BODY]] ] @@ -796,12 +800,22 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in ; GFX7-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]] ; GFX7: [[EXIT]]: ; GFX7-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 0 -; GFX7-NEXT: store <4 x i8> [[INPTR0]], ptr addrspace(3) [[GEP0]], align 1 +; GFX7-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 1 +; GFX7-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 2 +; GFX7-NEXT: [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 3 +; GFX7-NEXT: store i8 [[OTHERELE0]], ptr addrspace(3) [[GEP0]], align 1 +; GFX7-NEXT: store i8 [[OTHERELE1]], ptr addrspace(3) [[GEP1]], align 1 +; GFX7-NEXT: store i8 [[OTHERELE2]], ptr addrspace(3) [[GEP2]], align 1 +; GFX7-NEXT: store i8 [[OTHERELE3]], ptr addrspace(3) [[GEP3]], align 1 ; GFX7-NEXT: ret void ; ; GFX8-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop( ; GFX8-SAME: <4 x i8> [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr addrspace(3) [[OUT1:%.*]], i32 [[FLAG:%.*]]) { ; GFX8-NEXT: [[ENTRY:.*]]: +; GFX8-NEXT: [[OTHERELE0:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 0 +; GFX8-NEXT: [[OTHERELE1:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 1 +; GFX8-NEXT: [[OTHERELE2:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 2 +; GFX8-NEXT: [[OTHERELE3:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 3 ; GFX8-NEXT: br label %[[DO_BODY:.*]] ; GFX8: [[DO_BODY]]: ; GFX8-NEXT: [[TMP0:%.*]] = phi <4 x i8> [ [[INPTR0]], %[[ENTRY]] ], [ [[INPTR0]], %[[DO_BODY]] ] @@ -811,7 +825,13 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in ; GFX8-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]] ; GFX8: [[EXIT]]: ; GFX8-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 0 -; GFX8-NEXT: store <4 x i8> [[INPTR0]], ptr addrspace(3) [[GEP0]], align 1 +; GFX8-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 1 +; GFX8-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 2 +; GFX8-NEXT: [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 3 +; GFX8-NEXT: store i8 [[OTHERELE0]], ptr addrspace(3) [[GEP0]], align 1 +; GFX8-NEXT: store i8 [[OTHERELE1]], ptr addrspace(3) [[GEP1]], align 1 +; GFX8-NEXT: store i8 [[OTHERELE2]], ptr addrspace(3) [[GEP2]], align 1 +; GFX8-NEXT: store i8 [[OTHERELE3]], ptr addrspace(3) [[GEP3]], align 1 ; GFX8-NEXT: ret void ; ; GFX9-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop( _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
