| Issue |
243602
|
| Summary |
[X86][AVX512BW] Wrong code: shuffle of zext <2 x i8> and zext <2 x i16> lowered to vpunpckldq + vpmovzxbd
|
| Labels |
new issue
|
| Assignees |
|
| Reporter |
r-a-y-z-e-r
|
## Summary
With AVX512BW enabled, a `shufflevector` that combines `zext <2 x i8> to <2 x i32>` with
`zext <2 x i16> to <2 x i32>` into an `<8 x i32>` is selected as `vpunpckldq` of the two narrow
sources followed by a single `vpmovzxbd`. That widens the 16-bit elements byte by byte, so the
second word's lane receives the high byte of the first word. `-mcpu=x86-64-v3` is correct;
`+avx512f`, `+avx512vl` alone are correct; adding `+avx512bw` makes it wrong.
Found with clang 21.1.8 (Ubuntu 21.1.8-6ubuntu1), x86_64-unknown-linux-gnu.
## Reduced reproducer (llc)
```llvm
declare <3 x i8> @llvm.masked.load.v3i8.p0(ptr, i32 immarg, <3 x i1>, <3 x i8>)
define <8 x i32> @f(ptr %a, ptr %b) {
%bytes = call <3 x i8> @llvm.masked.load.v3i8.p0(
ptr %a, i32 1, <3 x i1> <i1 true, i1 false, i1 true>, <3 x i8> poison)
%pair = shufflevector <3 x i8> %bytes, <3 x i8> poison,
<2 x i32> <i32 0, i32 2>
%words = load <2 x i16>, ptr %b, align 1
%x = zext <2 x i8> %pair to <2 x i32>
%y = zext <2 x i16> %words to <2 x i32>
%result = shufflevector <2 x i32> %x, <2 x i32> %y,
<8 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 3, i32 poison, i32 poison>
ret <8 x i32> %result
}
```
`llc -O2 -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v4`:
```asm
movw $5, %ax
kmovd %eax, %k1
vmovdqu8 (%rdi), %xmm0 {%k1} {z}
vmovd (%rsi), %xmm1
vpunpckldq %xmm1, %xmm0, %xmm0
vpmovzxbd %xmm0, %ymm0 # widens the two i16 elements as four bytes
retq
```
`-mcpu=x86-64-v3` (correct):
```asm
movzbl (%rdi), %eax
vmovd %eax, %xmm0
vpinsrb $1, 2(%rdi), %xmm0, %xmm0
vmovd (%rsi), %xmm1
vpmovzxwd %xmm1, %xmm1
vpmovzxbq %xmm0, %xmm0
vinserti128 $1, %xmm1, %ymm0, %ymm0
retq
```
For `a = {11, 99, 22}`, `b = {0, 2}` the defined lanes are:
| Lane | Expected | v3 | v4 |
|---|---:|---:|---:|
| 0 | 11 | 11 | 11 |
| 2 | 22 | 22 | 22 |
| 4 | 0 | 0 | 0 |
| 5 | 2 | 2 | **0** |
Sweeping `b = {0, n}` over all 65 536 values of `n`: v3 is correct for all, v4 is wrong for 65 535.
Feature bisection from x86-64-v3: `+avx512f` correct; `+avx512f,+avx512vl` correct;
`+avx512f,+avx512bw,+avx512vl` wrong; `+avx512f,+avx512bw,+evex512` (no VL) wrong.
## Original C reproducer
The pattern came from SLP-vectorizing eight adjacent 4-input LUT evaluations,
`v[o] = (C >> (v[i0] | v[i1] << 1 | v[i2] << 2 | v[i3] << 3)) & 1;` on `uint8_t *restrict v`,
with 32-bit constants. A self-contained exhaustive test over all 2^17 Boolean inputs:
```c
/* Eight adjacent 4-input LUT lines, f (32-bit constants) and fixed (64-bit
* constants), checked against an optnone reference over all 2^17 inputs.
* Build and run: clang -O3 -march=x86-64-v4 lut_shift.c -o lut_shift && ./lut_shift
*/
#include <stdio.h>
#include <stdint.h>
__attribute__((noinline)) void f(uint8_t *restrict v) {
v[28417] = (65257u >> (v[31446] | v[26763] << 1 | v[26873] << 2 | v[24605] << 3)) & 1;
v[28418] = (233u >> (v[26763] | v[26873] << 1 | v[24605] << 2)) & 1;
v[28419] = (5632u >> (v[31852] | v[26658] << 1 | v[26875] << 2 | v[26876] << 3)) & 1;
v[28420] = (96u >> (v[26658] | v[26875] << 1 | v[26876] << 2)) & 1;
v[28421] = (45883u >> (v[31732] | v[23494] << 1 | v[26877] << 2 | v[26659] << 3)) & 1;
v[28422] = (52273u >> (v[23494] | v[26877] << 1 | v[26878] << 2 | v[26659] << 3)) & 1;
v[28423] = (5632u >> (v[31466] | v[26660] << 1 | v[26662] << 2 | v[26879] << 3)) & 1;
v[28424] = (96u >> (v[26660] | v[26662] << 1 | v[26879] << 2)) & 1;
}
#include <stdio.h>
#include <stdint.h>
__attribute__((noinline)) void fixed(uint8_t *restrict v) {
v[28417] = (65257ull >> (v[31446] | v[26763] << 1 | v[26873] << 2 | v[24605] << 3)) & 1;
v[28418] = (233ull >> (v[26763] | v[26873] << 1 | v[24605] << 2)) & 1;
v[28419] = (5632ull >> (v[31852] | v[26658] << 1 | v[26875] << 2 | v[26876] << 3)) & 1;
v[28420] = (96ull >> (v[26658] | v[26875] << 1 | v[26876] << 2)) & 1;
v[28421] = (45883ull >> (v[31732] | v[23494] << 1 | v[26877] << 2 | v[26659] << 3)) & 1;
v[28422] = (52273ull >> (v[23494] | v[26877] << 1 | v[26878] << 2 | v[26659] << 3)) & 1;
v[28423] = (5632ull >> (v[31466] | v[26660] << 1 | v[26662] << 2 | v[26879] << 3)) & 1;
v[28424] = (96ull >> (v[26660] | v[26662] << 1 | v[26879] << 2)) & 1;
}
#include <stdio.h>
#include <stdint.h>
__attribute__((noinline,optnone)) void reference(uint8_t *restrict v) {
v[28417] = (65257u >> (v[31446] | v[26763] << 1 | v[26873] << 2 | v[24605] << 3)) & 1;
v[28418] = (233u >> (v[26763] | v[26873] << 1 | v[24605] << 2)) & 1;
v[28419] = (5632u >> (v[31852] | v[26658] << 1 | v[26875] << 2 | v[26876] << 3)) & 1;
v[28420] = (96u >> (v[26658] | v[26875] << 1 | v[26876] << 2)) & 1;
v[28421] = (45883u >> (v[31732] | v[23494] << 1 | v[26877] << 2 | v[26659] << 3)) & 1;
v[28422] = (52273u >> (v[23494] | v[26877] << 1 | v[26878] << 2 | v[26659] << 3)) & 1;
v[28423] = (5632u >> (v[31466] | v[26660] << 1 | v[26662] << 2 | v[26879] << 3)) & 1;
v[28424] = (96u >> (v[26660] | v[26662] << 1 | v[26879] << 2)) & 1;
}
int main(void) {uint8_t a[40000]={0},b[40000]={0},c[40000]={0};unsigned bad=0,fixed_bad=0;
for(unsigned bits=0;bits<(1u<<17);bits++) {
a[23494]=b[23494]=c[23494]=(bits>>0)&1;
a[24605]=b[24605]=c[24605]=(bits>>1)&1;
a[26658]=b[26658]=c[26658]=(bits>>2)&1;
a[26659]=b[26659]=c[26659]=(bits>>3)&1;
a[26660]=b[26660]=c[26660]=(bits>>4)&1;
a[26662]=b[26662]=c[26662]=(bits>>5)&1;
a[26763]=b[26763]=c[26763]=(bits>>6)&1;
a[26873]=b[26873]=c[26873]=(bits>>7)&1;
a[26875]=b[26875]=c[26875]=(bits>>8)&1;
a[26876]=b[26876]=c[26876]=(bits>>9)&1;
a[26877]=b[26877]=c[26877]=(bits>>10)&1;
a[26878]=b[26878]=c[26878]=(bits>>11)&1;
a[26879]=b[26879]=c[26879]=(bits>>12)&1;
a[31446]=b[31446]=c[31446]=(bits>>13)&1;
a[31466]=b[31466]=c[31466]=(bits>>14)&1;
a[31732]=b[31732]=c[31732]=(bits>>15)&1;
a[31852]=b[31852]=c[31852]=(bits>>16)&1;
f(a);fixed(b);reference(c);
bad+=(a[28417]!=c[28417] || a[28418]!=c[28418] || a[28419]!=c[28419] || a[28420]!=c[28420] || a[28421]!=c[28421] || a[28422]!=c[28422] || a[28423]!=c[28423] || a[28424]!=c[28424]);
fixed_bad+=(b[28417]!=c[28417] || b[28418]!=c[28418] || b[28419]!=c[28419] || b[28420]!=c[28420] || b[28421]!=c[28421] || b[28422]!=c[28422] || b[28423]!=c[28423] || b[28424]!=c[28424]);
}
printf("combinations=%u original_mismatches=%u fixed_mismatches=%u\n",1u<<17,bad,fixed_bad);return fixed_bad!=0;}
```
`clang -O3 -march=x86-64-v4 lut_shift.c && ./a.out` prints
`combinations=131072 original_mismatches=57344 fixed_mismatches=0` (expected 0 mismatches).
Correct with `-march=x86-64-v3`, with `-fno-slp-vectorize`, with gcc 15.2 at `-march=x86-64-v4`.
Still wrong with `-mprefer-vector-width=256`. The SLP-vectorized IR is correct when lowered for
x86-64-v3 and wrong for x86-64-v4; the wrong `VPUNPCKLDQZ128rr`/`VPMOVZXBDZ256rr` pair is already
present after x86-isel.
We ruled out a CPU erratum (found on an AMD Zen 4-class machine): an interpreter of the emitted
instructions following the Intel SDM reproduces the hardware's results on all 131 072 inputs, and
both disagree with the C source on the same 57 344.
_______________________________________________
llvm-bugs mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-bugs