https://gcc.gnu.org/g:7644f2a3f009da4c32f7acc66c10f947bb455ae6
commit r17-5085-g7644f2a3f009da4c32f7acc66c10f947bb455ae6 Author: Kyrylo Tkachov <[email protected]> Date: Wed Sep 16 22:29:54 2026 +0200 aarch64: Expose AdvSIMD integer multiply-add optabs AdvSIMD has integer MLA and MLS patterns, but no integer FMA optabs. A multiplication by three can therefore become two additions during expansion before combine can recognise a multiply-subtract. Add FMA and FNMA expanders for the existing AdvSIMD integer modes. Reuse the SVE preparation helper, renamed for both callers, to preserve shifts for positive, negative and lane-varying power-of-two factors. The existing tree math pass can then retain the multiply-add operation. In glibc sysdeps/aarch64/fpu/advsimd/cbrtf.c, the remainder expression e - ey * 3 changes from separate GIMPLE multiply and subtract statements to .FNMA. Before: add v28.4s, v29.4s, v29.4s add v28.4s, v28.4s, v29.4s sub v31.4s, v31.4s, v28.4s After: movi v28.4s, 0x3 mls v31.4s, v29.4s, v28.4s In that comparison the routine has 1.69% fewer instructions. Tests cover signed and unsigned modes, constant and variable factors, wraparound values and the power-of-two fallback. Bootstrapped and tested on aarch64-none-linux-gnu. Signed-off-by: Kyrylo Tkachov <[email protected]> gcc/ChangeLog: * config/aarch64/aarch64-protos.h (aarch64_prepare_sve_int_fma): Rename to aarch64_prepare_int_fma. * config/aarch64/aarch64.cc (aarch64_prepare_sve_int_fma): Likewise. * config/aarch64/aarch64-simd.md (fma<mode>4, fnma<mode>4): New VDQ_BHSI expanders. * config/aarch64/aarch64-sve.md (fma<mode>4, fnma<mode>4): Use the renamed helper. gcc/testsuite/ChangeLog: * gcc.target/aarch64/simd-int-fma.h: New file. * gcc.target/aarch64/simd-int-fma.c: New test. * gcc.target/aarch64/simd-int-fma-run.c: Likewise. * gcc.target/aarch64/simd-int-fma-shift.c: Likewise. * gcc.target/aarch64/pr122749_1.c: Remove the expected FMA failure. * gcc.target/aarch64/pr122749_2.c: Likewise. * gcc.target/aarch64/pr122749_3.c: Likewise. * gcc.target/aarch64/pr122749_8.c: Likewise. * gcc.target/aarch64/pr122749_9.c: Likewise. * gcc.target/aarch64/sve/mla_3.c: Expect AdvSIMD and SVE FMAs. Diff: --- gcc/config/aarch64/aarch64-protos.h | 2 +- gcc/config/aarch64/aarch64-simd.md | 30 ++++++++ gcc/config/aarch64/aarch64-sve.md | 4 +- gcc/config/aarch64/aarch64.cc | 4 +- gcc/testsuite/gcc.target/aarch64/pr122749_1.c | 2 +- gcc/testsuite/gcc.target/aarch64/pr122749_2.c | 2 +- gcc/testsuite/gcc.target/aarch64/pr122749_3.c | 2 +- gcc/testsuite/gcc.target/aarch64/pr122749_8.c | 2 +- gcc/testsuite/gcc.target/aarch64/pr122749_9.c | 2 +- .../gcc.target/aarch64/simd-int-fma-run.c | 81 ++++++++++++++++++++++ .../gcc.target/aarch64/simd-int-fma-shift.c | 31 +++++++++ gcc/testsuite/gcc.target/aarch64/simd-int-fma.c | 7 ++ gcc/testsuite/gcc.target/aarch64/simd-int-fma.h | 39 +++++++++++ gcc/testsuite/gcc.target/aarch64/sve/mla_3.c | 2 +- 14 files changed, 199 insertions(+), 11 deletions(-) diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h index e8ae3d427949..262b3c441057 100644 --- a/gcc/config/aarch64/aarch64-protos.h +++ b/gcc/config/aarch64/aarch64-protos.h @@ -1138,7 +1138,7 @@ void aarch64_finish_ldpstp_peephole (rtx *, bool, void aarch64_expand_sve_vec_cmp_int (rtx, rtx_code, rtx, rtx); void aarch64_expand_sve_vec_cmp_float (rtx, rtx_code, rtx, rtx); -bool aarch64_prepare_sve_int_fma (rtx *, rtx_code); +bool aarch64_prepare_int_fma (rtx *, rtx_code); bool aarch64_prepare_sve_cond_int_fma (rtx *, rtx_code); bool aarch64_granule16_memory_address_p (rtx mem); diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index 98c2da034c55..1d28c8f68f9a 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -1959,6 +1959,21 @@ DONE; }) +;; Integer addition of product, preserving shifts for power-of-two factors. +(define_expand "fma<mode>4" + [(set (match_operand:VDQ_BHSI 0 "register_operand") + (plus:VDQ_BHSI + (mult:VDQ_BHSI + (match_operand:VDQ_BHSI 1 "register_operand") + (match_operand:VDQ_BHSI 2 "nonmemory_operand")) + (match_operand:VDQ_BHSI 3 "register_operand")))] + "TARGET_SIMD" + { + if (aarch64_prepare_int_fma (operands, PLUS)) + DONE; + } +) + (define_insn "aarch64_mla<mode><vczle><vczbe>" [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w") (plus:VDQ_BHSI (mult:VDQ_BHSI @@ -2019,6 +2034,21 @@ [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")] ) +;; Integer subtraction of product. +(define_expand "fnma<mode>4" + [(set (match_operand:VDQ_BHSI 0 "register_operand") + (minus:VDQ_BHSI + (match_operand:VDQ_BHSI 3 "register_operand") + (mult:VDQ_BHSI + (match_operand:VDQ_BHSI 1 "register_operand") + (match_operand:VDQ_BHSI 2 "nonmemory_operand"))))] + "TARGET_SIMD" + { + if (aarch64_prepare_int_fma (operands, MINUS)) + DONE; + } +) + (define_insn "aarch64_mls<mode><vczle><vczbe>" [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w") (minus:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "0") diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md index a833c74493a2..88544a4fde9a 100644 --- a/gcc/config/aarch64/aarch64-sve.md +++ b/gcc/config/aarch64/aarch64-sve.md @@ -7530,7 +7530,7 @@ (match_operand:SVE_I 3 "register_operand")))] "TARGET_SVE" { - if (aarch64_prepare_sve_int_fma (operands, PLUS)) + if (aarch64_prepare_int_fma (operands, PLUS)) DONE; operands[4] = aarch64_ptrue_reg (<VPRED>mode); } @@ -7676,7 +7676,7 @@ UNSPEC_PRED_X)))] "TARGET_SVE" { - if (aarch64_prepare_sve_int_fma (operands, MINUS)) + if (aarch64_prepare_int_fma (operands, MINUS)) DONE; operands[4] = aarch64_ptrue_reg (<VPRED>mode); } diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc index e8e903d82a12..5f0556fd5169 100644 --- a/gcc/config/aarch64/aarch64.cc +++ b/gcc/config/aarch64/aarch64.cc @@ -26922,14 +26922,14 @@ aarch64_convert_mult_to_shift (rtx value, rtx_code &code) return builder.build (); } -/* Prepare for an integer SVE multiply-add or multiply-subtract pattern; +/* Prepare for an integer vector multiply-add or multiply-subtract pattern. CODE is PLUS for the former and MINUS for the latter. OPERANDS is the operands array, in the same order as for fma_optab. Return true if the function emitted all the necessary instructions, false if the caller should generate the pattern normally with the new OPERANDS array. */ bool -aarch64_prepare_sve_int_fma (rtx *operands, rtx_code code) +aarch64_prepare_int_fma (rtx *operands, rtx_code code) { machine_mode mode = GET_MODE (operands[0]); if (rtx shifts = aarch64_convert_mult_to_shift (operands[2], code)) diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_1.c b/gcc/testsuite/gcc.target/aarch64/pr122749_1.c index 25311fce4e3a..da86ea3330f4 100644 --- a/gcc/testsuite/gcc.target/aarch64/pr122749_1.c +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_1.c @@ -44,5 +44,5 @@ main (void) return 0; } -/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" } } */ /* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_2.c b/gcc/testsuite/gcc.target/aarch64/pr122749_2.c index f4a70a611176..edb49007d84f 100644 --- a/gcc/testsuite/gcc.target/aarch64/pr122749_2.c +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_2.c @@ -44,5 +44,5 @@ main (void) return 0; } -/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" } } */ /* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_3.c b/gcc/testsuite/gcc.target/aarch64/pr122749_3.c index 61bcd30be2b4..b5244ed05a80 100644 --- a/gcc/testsuite/gcc.target/aarch64/pr122749_3.c +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_3.c @@ -44,5 +44,5 @@ main (void) return 0; } -/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" } } */ /* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_8.c b/gcc/testsuite/gcc.target/aarch64/pr122749_8.c index 6aa729c13d16..a0b55390156a 100644 --- a/gcc/testsuite/gcc.target/aarch64/pr122749_8.c +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_8.c @@ -44,5 +44,5 @@ main (void) return 0; } -/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" } } */ /* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_9.c b/gcc/testsuite/gcc.target/aarch64/pr122749_9.c index d987a9936afb..b1c94bab2c56 100644 --- a/gcc/testsuite/gcc.target/aarch64/pr122749_9.c +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_9.c @@ -44,5 +44,5 @@ main (void) return 0; } -/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" } } */ /* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/simd-int-fma-run.c b/gcc/testsuite/gcc.target/aarch64/simd-int-fma-run.c new file mode 100644 index 000000000000..9f87ae065c2c --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/simd-int-fma-run.c @@ -0,0 +1,81 @@ +/* { dg-do run } */ +/* { dg-options "-O2 -fwrapv" } */ + +#include "simd-int-fma.h" +#include "simd-int-fma-shift.c" + +static const uint32_t values[] = { + 0, 1, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, + 0x7fffffff, 0x80000000, 0xffffffff, 0x55555555, 0xaaaaaaaa +}; + +#define CHECK(TYPE) \ + { \ + TYPE a, b, c; \ + unsigned int lanes = sizeof (a) / sizeof (a[0]); \ + for (unsigned int k = 0; k < lanes; ++k) \ + { \ + a[k] = values[(i + k) % count]; \ + b[k] = values[(j + 2 * k) % count]; \ + c[k] = values[(i + j + 3 * k) % count]; \ + } \ + TYPE add = add_##TYPE (a, b); \ + TYPE sub = sub_##TYPE (a, b); \ + TYPE add_variable = add_variable_##TYPE (a, b, c); \ + TYPE sub_variable = sub_variable_##TYPE (a, b, c); \ + for (unsigned int k = 0; k < lanes; ++k) \ + { \ + uint32_t sum = (uint32_t) a[k] + (uint32_t) b[k] * 3; \ + uint32_t diff = (uint32_t) a[k] - (uint32_t) b[k] * 3; \ + if (add[k] != (__typeof (a[k])) sum \ + || sub[k] != (__typeof (a[k])) diff) \ + __builtin_abort (); \ + uint32_t product = (uint32_t) b[k] * (uint32_t) c[k]; \ + sum = (uint32_t) a[k] + product; \ + diff = (uint32_t) a[k] - product; \ + if (add_variable[k] != (__typeof (a[k])) sum \ + || sub_variable[k] != (__typeof (a[k])) diff) \ + __builtin_abort (); \ + } \ + } + +#define CHECK_SHIFT(TYPE, SUFFIX, FACTOR) \ + { \ + TYPE a, b; \ + TYPE factors = (TYPE) { 0 } + FACTOR; \ + unsigned int lanes = sizeof (a) / sizeof (a[0]); \ + for (unsigned int k = 0; k < lanes; ++k) \ + { \ + a[k] = values[(i + k) % count]; \ + b[k] = values[(j + 2 * k) % count]; \ + } \ + TYPE add = add_##SUFFIX (a, b); \ + TYPE sub = sub_##SUFFIX (a, b); \ + for (unsigned int k = 0; k < lanes; ++k) \ + { \ + uint32_t product = (uint32_t) b[k] * (uint32_t) factors[k]; \ + uint32_t sum = (uint32_t) a[k] + product; \ + uint32_t diff = (uint32_t) a[k] - product; \ + if (add[k] != (__typeof (a[k])) sum \ + || sub[k] != (__typeof (a[k])) diff) \ + __builtin_abort (); \ + } \ + } + +int +main (void) +{ + unsigned int count = sizeof (values) / sizeof (values[0]); + for (unsigned int i = 0; i < count; ++i) + for (unsigned int j = 0; j < count; ++j) + { + ALL_TYPES (CHECK) + CHECK_SHIFT (int16x8_t, s16, 8) + CHECK_SHIFT (uint16x8_t, u16, 8) + CHECK_SHIFT (int32x4_t, s32, -8) + CHECK_SHIFT (uint32x4_t, u32, -8U) + CHECK_SHIFT (uint16x8_t, positive, POSITIVE_POWERS) + CHECK_SHIFT (int32x4_t, negative, NEGATIVE_POWERS) + } + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/simd-int-fma-shift.c b/gcc/testsuite/gcc.target/aarch64/simd-int-fma-shift.c new file mode 100644 index 000000000000..91822c8f5ea5 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/simd-int-fma-shift.c @@ -0,0 +1,31 @@ +/* { dg-do compile } */ +/* { dg-options "-O2" } */ + +#include <arm_neon.h> + +#define DEF_SHIFT(TYPE, SUFFIX, FACTOR) \ + TYPE __attribute__ ((noipa)) add_##SUFFIX (TYPE a, TYPE b) \ + { \ + return a + b * FACTOR; \ + } \ + TYPE __attribute__ ((noipa)) sub_##SUFFIX (TYPE a, TYPE b) \ + { \ + return a - b * FACTOR; \ + } + +DEF_SHIFT (int16x8_t, s16, 8) +DEF_SHIFT (uint16x8_t, u16, 8) +DEF_SHIFT (int32x4_t, s32, -8) +DEF_SHIFT (uint32x4_t, u32, -8U) + +#define POSITIVE_POWERS \ + ((uint16x8_t) { 1, 2, 4, 8, 16, 32, 64, 32768 }) +#define NEGATIVE_POWERS \ + ((int32x4_t) { -1, -2, -4, (-2147483647 - 1) }) + +DEF_SHIFT (uint16x8_t, positive, POSITIVE_POWERS) +DEF_SHIFT (int32x4_t, negative, NEGATIVE_POWERS) + +/* { dg-final { scan-assembler-times {\tshl\tv} 8 } } */ +/* { dg-final { scan-assembler-times {\tsshl\tv} 4 } } */ +/* { dg-final { scan-assembler-not {\tmul\t|\tmla\t|\tmls\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/simd-int-fma.c b/gcc/testsuite/gcc.target/aarch64/simd-int-fma.c new file mode 100644 index 000000000000..e8c7dd13652c --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/simd-int-fma.c @@ -0,0 +1,7 @@ +/* { dg-do compile } */ +/* { dg-options "-O2" } */ + +#include "simd-int-fma.h" + +/* { dg-final { scan-assembler-times {\tmla\tv} 24 } } */ +/* { dg-final { scan-assembler-times {\tmls\tv} 24 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/simd-int-fma.h b/gcc/testsuite/gcc.target/aarch64/simd-int-fma.h new file mode 100644 index 000000000000..aaa397f58443 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/simd-int-fma.h @@ -0,0 +1,39 @@ +#include <arm_neon.h> + +#define DEF_FMA(TYPE) \ + TYPE __attribute__ ((noipa)) \ + add_##TYPE (TYPE a, TYPE b) \ + { \ + return a + b * 3; \ + } \ + TYPE __attribute__ ((noipa)) \ + sub_##TYPE (TYPE a, TYPE b) \ + { \ + return a - b * 3; \ + } \ + TYPE __attribute__ ((noipa)) \ + add_variable_##TYPE (TYPE a, TYPE b, TYPE c) \ + { \ + return a + b * c; \ + } \ + TYPE __attribute__ ((noipa)) \ + sub_variable_##TYPE (TYPE a, TYPE b, TYPE c) \ + { \ + return a - b * c; \ + } + +#define ALL_TYPES(T) \ + T (int8x8_t) \ + T (uint8x8_t) \ + T (int8x16_t) \ + T (uint8x16_t) \ + T (int16x4_t) \ + T (uint16x4_t) \ + T (int16x8_t) \ + T (uint16x8_t) \ + T (int32x2_t) \ + T (uint32x2_t) \ + T (int32x4_t) \ + T (uint32x4_t) + +ALL_TYPES (DEF_FMA) diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mla_3.c b/gcc/testsuite/gcc.target/aarch64/sve/mla_3.c index 25e99f7d72a2..2473a0ff3a8a 100644 --- a/gcc/testsuite/gcc.target/aarch64/sve/mla_3.c +++ b/gcc/testsuite/gcc.target/aarch64/sve/mla_3.c @@ -36,4 +36,4 @@ g3 (vec a, vec b, vec c) return a * b + c; } -/* { dg-final { scan-tree-dump-times {\.FMA } 1 "optimized" } } */ +/* { dg-final { scan-tree-dump-times {\.FMA } 2 "optimized" } } */
