| Issue |
242207
|
| Summary |
[X86][VectorCombine] AVX-512 even-lane shuffle codegen regression after #229662 (foldShuffleOfAdjacentLoads)
|
| Labels |
new issue
|
| Assignees |
|
| Reporter |
eaeltsin
|
### Description
Commit cc7c71f048c083fbaec7581d94124559f69317f4 (reland PR #229662: `Reland "[VectorCombine] Combine adjacent loads feeding as shuffle operands"`) introduced a codegen regression on AVX-512 targets when two adjacent 256-bit vector loads feed an even-lane shuffle (`<0, 2, 4, ... 30>`) that has multiple subvector consumers.
`VectorCombine::foldShuffleOfAdjacentLoads` combines two `<16 x i16>` loads into a single `<32 x i16>` wide load and converts the two-source shuffle into a single-source `<32 x i16> -> <16 x i16>` shuffle. However, downstream X86 ISel / DAGCombiner fails to lower this 512-bit to 256-bit shuffle as a single truncation (`vpmovdw %zmm -> %ymm`) shared across consumers. Instead, subvector extracts are pushed into the shuffle, decomposing it independently for each use and producing 12 vector instructions (including `vpermw` and multiple `vpinsrw` instructions) instead of the previous 5.
On Skylake-SP, this roughly doubles port 5 shuffle pressure and caused an observed +35% latency regression on integer sequence encoding benchmarks.
### Minimal C++ Reproducer
```cpp
#include <immintrin.h>
__attribute__((target("avx512bw,avx512vl,popcnt"))) int EncodeBlock(
const short* data, char* out, const char* shuffle_table) {
const __m256i d0 = _mm256_loadu_si256((const __m256i*)data);
const __m256i d1 = _mm256_loadu_si256((const __m256i*)(data + 16));
const __m256i even_lanes = _mm256_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14, 16,
18, 20, 22, 24, 26, 28, 30);
const __m256i v = _mm256_permutex2var_epi16(d0, even_lanes, d1);
const unsigned mask = _mm256_test_epi16_mask(v, v);
const unsigned lo_mask = mask & 0xff;
const unsigned hi_mask = mask >> 8;
const __m128i lo = _mm_shuffle_epi8(
_mm256_castsi256_si128(v),
_mm_loadu_si128((const __m128i*)(shuffle_table + lo_mask * 16)));
const __m128i hi = _mm_shuffle_epi8(
_mm256_extracti128_si256(v, 1),
_mm_loadu_si128((const __m128i*)(shuffle_table + hi_mask * 16)));
*(unsigned short*)out = (unsigned short)mask;
_mm_storeu_si128((__m128i*)(out + 2), lo);
_mm_storeu_si128((__m128i*)(out + 2 + __builtin_popcount(lo_mask) * 2), hi);
return 2 + __builtin_popcount(mask) * 2;
}
```
### Old code
```
_Z11EncodeBlockPKsPcPKc:
vmovdqu64 (%rdi), %zmm0
vpmovdw %zmm0, %ymm0
vptestmw %ymm0, %ymm0, %k0
kmovw %k0, %eax
movzbl %al, %ecx
popcntl %ecx, %edi
shll $4, %ecx
vpshufb (%rdx,%rcx), %xmm0, %xmm1
popcntl %eax, %ecx
shrl $4, %eax
andl $-16, %eax
vextracti128 $1, %ymm0, %xmm0
vpshufb (%rdx,%rax), %xmm0, %xmm0
kmovw %k0, (%rsi)
vmovdqu %xmm1, 2(%rsi)
vmovdqu %xmm0, 2(%rsi,%rdi,2)
leal 2(,%rcx,2), %eax
vzeroupper
retq
```
### New code
```
.LCPI0_0:
.short 0
.short 2
.short 4
.short 6
.short 8
.short 5
.short 6
.short 7
_Z11EncodeBlockPKsPcPKc:
vmovdqa .LCPI0_0(%rip), %xmm0
vpermw (%rdi), %ymm0, %ymm0
vmovdqu 16(%rdi), %xmm1
vpextrw $6, %xmm1, %ecx
vpextrw $4, %xmm1, %r8d
vpextrw $2, %xmm1, %r9d
vmovdqu64 (%rdi), %zmm1
vpmovdw %zmm1, %ymm1
vptestmw %ymm1, %ymm1, %k0
kmovw %k0, %eax
movzbl %al, %r10d
popcntl %r10d, %r11d
shll $4, %r10d
vpinsrw $5, %r9d, %xmm0, %xmm0
vpinsrw $6, %r8d, %xmm0, %xmm0
vpinsrw $7, %ecx, %xmm0, %xmm0
vpshufb (%rdx,%r10), %xmm0, %xmm0
popcntl %eax, %ecx
shrl $4, %eax
andl $-16, %eax
vmovdqu 32(%rdi), %ymm1
vpmovdw %ymm1, %xmm1
vpshufb (%rdx,%rax), %xmm1, %xmm1
kmovw %k0, (%rsi)
vmovdqu %xmm0, 2(%rsi)
vmovdqu %xmm1, 2(%rsi,%r11,2)
leal 2(,%rcx,2), %eax
vzeroupper
retq
```
_______________________________________________
llvm-bugs mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-bugs