This patch improves GCC’s vectorization of __builtin_popcount for aarch64 target by adding popcount patterns for vector modes besides QImode, i.e., HImode, SImode and DImode.
With this patch, we now generate the following for HImode: cnt v1.16b, v.16b uaddlp v2.8h, v1.16b For SImode, we generate: cnt v1.16b, v.16b uaddlp v2.8h, v1.16b uaddlp v3.4s, v2.8h For V2DI, we generate: cnt v1.16b, v.16b uaddlp v2.8h, v1.16b uaddlp v3.4s, v2.8h uaddlp v4.2d, v3.4s gcc/ChangeLog: PR target/113859 * config/aarch64/aarch64-simd.md (popcount<mode>2): New define_expand. gcc/testsuite/ChangeLog: PR target/113859 * gcc.target/aarch64/popcnt-vec.c: New test. Signed-off-by: Pengxuan Zheng <quic_pzh...@quicinc.com> --- gcc/config/aarch64/aarch64-simd.md | 40 ++++++++++++++++ gcc/testsuite/gcc.target/aarch64/popcnt-vec.c | 48 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 gcc/testsuite/gcc.target/aarch64/popcnt-vec.c diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index f8bb973a278..093c32ee8ff 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -3540,6 +3540,46 @@ (define_insn "popcount<mode>2<vczle><vczbe>" [(set_attr "type" "neon_cnt<q>")] ) +(define_expand "popcount<mode>2" + [(set (match_operand:VQN 0 "register_operand" "=w") + (popcount:VQN (match_operand:VQN 1 "register_operand" "w")))] + "TARGET_SIMD" + { + rtx v = gen_reg_rtx (V16QImode); + rtx v1 = gen_reg_rtx (V16QImode); + emit_move_insn (v, gen_lowpart (V16QImode, operands[1])); + emit_insn (gen_popcountv16qi2 (v1, v)); + if (<MODE>mode == V8HImode) + { + /* For V8HI, we generate: + cnt v1.16b, v.16b + uaddlp v2.8h, v1.16b */ + emit_insn (gen_aarch64_uaddlpv16qi (operands[0], v1)); + DONE; + } + rtx v2 = gen_reg_rtx (V8HImode); + emit_insn (gen_aarch64_uaddlpv16qi (v2, v1)); + if (<MODE>mode == V4SImode) + { + /* For V4SI, we generate: + cnt v1.16b, v.16b + uaddlp v2.8h, v1.16b + uaddlp v3.4s, v2.8h */ + emit_insn (gen_aarch64_uaddlpv8hi (operands[0], v2)); + DONE; + } + /* For V2DI, we generate: + cnt v1.16b, v.16b + uaddlp v2.8h, v1.16b + uaddlp v3.4s, v2.8h + uaddlp v4.2d, v3.4s */ + rtx v3 = gen_reg_rtx (V4SImode); + emit_insn (gen_aarch64_uaddlpv8hi (v3, v2)); + emit_insn (gen_aarch64_uaddlpv4si (operands[0], v3)); + DONE; + } +) + ;; 'across lanes' max and min ops. ;; Template for outputting a scalar, so we can create __builtins which can be diff --git a/gcc/testsuite/gcc.target/aarch64/popcnt-vec.c b/gcc/testsuite/gcc.target/aarch64/popcnt-vec.c new file mode 100644 index 00000000000..4c9a1b95990 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/popcnt-vec.c @@ -0,0 +1,48 @@ +/* { dg-do compile } */ +/* { dg-options "-O2" } */ + +/* This function should produce cnt v.16b. */ +void +bar (unsigned char *__restrict b, unsigned char *__restrict d) +{ + for (int i = 0; i < 1024; i++) + d[i] = __builtin_popcount (b[i]); +} + +/* This function should produce cnt v.16b and uaddlp (Add Long Pairwise). */ +void +bar1 (unsigned short *__restrict b, unsigned short *__restrict d) +{ + for (int i = 0; i < 1024; i++) + d[i] = __builtin_popcount (b[i]); +} + +/* This function should produce cnt v.16b and 2 uaddlp (Add Long Pairwise). */ +void +bar2 (unsigned int *__restrict b, unsigned int *__restrict d) +{ + for (int i = 0; i < 1024; i++) + d[i] = __builtin_popcount (b[i]); +} + +/* This function should produce cnt v.16b and 3 uaddlp (Add Long Pairwise). */ +void +bar3 (unsigned long long *__restrict b, unsigned long long *__restrict d) +{ + for (int i = 0; i < 1024; i++) + d[i] = __builtin_popcountll (b[i]); +} + +/* SLP + This function should produce cnt v.16b and 3 uaddlp (Add Long Pairwise). */ +void +bar4 (unsigned long long *__restrict b, unsigned long long *__restrict d) +{ + d[0] = __builtin_popcountll (b[0]); + d[1] = __builtin_popcountll (b[1]); +} + +/* { dg-final { scan-assembler-not {\tbl\tpopcount} } } */ +/* { dg-final { scan-assembler-times {cnt\t} 5 } } */ +/* { dg-final { scan-assembler-times {uaddlp\t} 9 } } */ +/* { dg-final { scan-assembler-times {ldr\tq} 5 } } */ -- 2.17.1