https://gcc.gnu.org/g:cb40e813b8f09f9d3a6000901f1373b476a20886
commit r16-7057-gcb40e813b8f09f9d3a6000901f1373b476a20886 Author: Tamar Christina <[email protected]> Date: Tue Jan 27 09:12:16 2026 +0000 middle-end: teach convert_mult_to_fma handle casts between addend and multiplicant [PR122749] The following example int foo2 (char *buf, int len) { int x; for (int i =0; i < len; i++) { x += (int) i * buf[i]; } return x; } compiled with -O3 -mcpu=neoverse-v2 used to generate a 4x unrolled MLA sequence mla z29.s, p7/m, z2.s, z0.s mla z27.s, p7/m, z4.s, z26.s mla z30.s, p7/m, z1.s, z0.s mla z28.s, p7/m, z23.s, z3.s but now generates MUL + ADD mul z2.s, z2.s, z1.s mul z4.s, z4.s, z26.s mul z1.s, z24.s, z1.s mul z3.s, z23.s, z3.s add z29.s, z2.s, z29.s add z30.s, z1.s, z30.s add z28.s, z3.s, z28.s add z0.s, z4.s, z0.s This is since the fix for r16-3328-g3182e95eda4 we now insert casts around the reduction addend. This causes convert_mult_to_fma to miss the mul + add sequence. This patch teaches it to look around the casts for the operands and only accept the conversions if it's essentially only a sign changing operations. Concretely, it converts: # vect_vec_iv_.13_49 = PHI <_50(5), { 0, 1, 2, ... }(4)> vect__3.8_38 = MEM <vector([4,4]) char> [(char *)_16]; vect__4.12_45 = (vector([4,4]) int) vect__3.8_38; vect__5.14_54 = vect__4.12_45 * vect_vec_iv_.13_49; vect_x_12.17_62 = VIEW_CONVERT_EXPR<vector([4,4]) unsigned int>(vect__5.14_54); vect_x_12.17_63 = VIEW_CONVERT_EXPR<vector([4,4]) unsigned int>(vect_x_16.15_58); vect_x_12.17_64 = vect_x_12.17_62 + vect_x_12.17_63; vect_x_12.16_65 = VIEW_CONVERT_EXPR<vector([4,4]) int>(vect_x_12.17_64); into: # vect_vec_iv_.13_49 = PHI <_50(5), { 0, 1, 2, ... }(4)> vect__3.8_38 = MEM <vector([4,4]) charD.8> [(charD.8 *)_16]; vect__4.12_45 = (vector([4,4]) intD.7) vect__3.8_38; vect_x_12.17_63 = VIEW_CONVERT_EXPR<vector([4,4]) unsigned int>(vect_x_16.15_58); _2 = (vector([4,4]) unsigned int) vect_vec_iv_.13_49; _1 = (vector([4,4]) unsigned int) vect__4.12_45; vect_x_12.17_64 = .FMA (_1, _2, vect_x_12.17_63); vect_x_12.16_65 = VIEW_CONVERT_EXPR<vector([4,4]) intD.7>(vect_x_12.17_64); thus restoring FMAs on reductions. gcc/ChangeLog: PR tree-optimization/122749 * tree-ssa-math-opts.cc (convert_mult_to_fma_1, convert_mult_to_fma): Unwrap converts around addend. gcc/testsuite/ChangeLog: PR tree-optimization/122749 * gcc.target/aarch64/pr122749_1.c: New test. * gcc.target/aarch64/pr122749_2.c: New test. * gcc.target/aarch64/pr122749_3.c: New test. * gcc.target/aarch64/pr122749_4.c: New test. * gcc.target/aarch64/pr122749_5.c: New test. * gcc.target/aarch64/pr122749_6.c: New test. * gcc.target/aarch64/pr122749_8.c: New test. * gcc.target/aarch64/pr122749_9.c: New test. * gcc.target/aarch64/sve/pr122749_1.c: New test. * gcc.target/aarch64/sve/pr122749_11.c: New test. * gcc.target/aarch64/sve/pr122749_12.c: New test. * gcc.target/aarch64/sve/pr122749_13.c: New test. * gcc.target/aarch64/sve/pr122749_14.c: New test. * gcc.target/aarch64/sve/pr122749_2.c: New test. * gcc.target/aarch64/sve/pr122749_3.c: New test. * gcc.target/aarch64/sve/pr122749_4.c: New test. * gcc.target/aarch64/sve/pr122749_5.c: New test. * gcc.target/aarch64/sve/pr122749_6.c: New test. * gcc.target/aarch64/sve/pr122749_8.c: New test. * gcc.target/aarch64/sve/pr122749_9.c: New test. Diff: --- gcc/testsuite/gcc.target/aarch64/pr122749_1.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/pr122749_2.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/pr122749_3.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/pr122749_4.c | 45 ++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/pr122749_5.c | 45 ++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/pr122749_6.c | 45 ++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/pr122749_8.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/pr122749_9.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_1.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_11.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_12.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_13.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_14.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_2.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_3.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_4.c | 48 ++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_5.c | 45 ++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_6.c | 45 ++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_8.c | 45 ++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr122749_9.c | 45 ++++++++++++++++++++ gcc/tree-ssa-math-opts.cc | 41 ++++++++++++++++++ 21 files changed, 980 insertions(+) diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_1.c b/gcc/testsuite/gcc.target/aarch64/pr122749_1.c new file mode 100644 index 000000000000..25311fce4e3a --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_1.c @@ -0,0 +1,48 @@ +/* { dg-do run { target arm_v8_neon_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef int8_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, -2, INT8_MAX, INT8_MIN, 5, -7, 3, -4 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_2.c b/gcc/testsuite/gcc.target/aarch64/pr122749_2.c new file mode 100644 index 000000000000..f4a70a611176 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_2.c @@ -0,0 +1,48 @@ +/* { dg-do run { target arm_v8_neon_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef int16_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, -2, INT16_MAX, INT16_MIN, 5, -7, 3, -4 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_3.c b/gcc/testsuite/gcc.target/aarch64/pr122749_3.c new file mode 100644 index 000000000000..61bcd30be2b4 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_3.c @@ -0,0 +1,48 @@ +/* { dg-do run { target arm_v8_neon_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef int32_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, -2, INT32_MAX, INT32_MIN, 5, -7, 3, -4 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_4.c b/gcc/testsuite/gcc.target/aarch64/pr122749_4.c new file mode 100644 index 000000000000..6089716b0ca7 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_4.c @@ -0,0 +1,45 @@ +/* { dg-do run { target arm_v8_neon_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +typedef float elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.FMA" 4 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_5.c b/gcc/testsuite/gcc.target/aarch64/pr122749_5.c new file mode 100644 index 000000000000..562dc5be8617 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_5.c @@ -0,0 +1,45 @@ +/* { dg-do run { target arm_v8_neon_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +typedef double elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1.0, 2.0, 1.0, 2.0, 1.0, 2.0 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.FMA" 2 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_6.c b/gcc/testsuite/gcc.target/aarch64/pr122749_6.c new file mode 100644 index 000000000000..3e51c5e22a18 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_6.c @@ -0,0 +1,45 @@ +/* { dg-do run { target arm_v8_neon_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fwrapv -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +typedef float elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.FMA" 4 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_8.c b/gcc/testsuite/gcc.target/aarch64/pr122749_8.c new file mode 100644 index 000000000000..6aa729c13d16 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_8.c @@ -0,0 +1,48 @@ +/* { dg-do run { target arm_v8_neon_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef uint8_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, 2, UINT8_MAX, 7, 0, UINT8_MAX, 5, 9 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_9.c b/gcc/testsuite/gcc.target/aarch64/pr122749_9.c new file mode 100644 index 000000000000..d987a9936afb --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/pr122749_9.c @@ -0,0 +1,48 @@ +/* { dg-do run { target arm_v8_neon_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef uint16_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, 2, UINT16_MAX, 7, 0, UINT16_MAX, 5, 9 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_1.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_1.c new file mode 100644 index 000000000000..32a36461fbc7 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_1.c @@ -0,0 +1,48 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef int8_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, -2, INT8_MAX, INT8_MIN, 5, -7, 3, -4 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_11.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_11.c new file mode 100644 index 000000000000..bd160dd0ebf5 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_11.c @@ -0,0 +1,48 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef uint8_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, 2, UINT8_MAX, 7, 0, UINT8_MAX, 5, 9 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_12.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_12.c new file mode 100644 index 000000000000..8f0198ce4260 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_12.c @@ -0,0 +1,48 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef uint16_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, 2, UINT16_MAX, 7, 0, UINT16_MAX, 5, 9 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_13.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_13.c new file mode 100644 index 000000000000..218afde13984 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_13.c @@ -0,0 +1,48 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef uint32_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, 2, UINT32_MAX, 7, 0, UINT32_MAX, 5, 9 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_14.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_14.c new file mode 100644 index 000000000000..1587628757e2 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_14.c @@ -0,0 +1,48 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef uint64_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, 2, UINT64_MAX, 7, 0, UINT64_MAX, 5, 9 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_2.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_2.c new file mode 100644 index 000000000000..0f5918a90235 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_2.c @@ -0,0 +1,48 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef int16_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, -2, INT16_MAX, INT16_MIN, 5, -7, 3, -4 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_3.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_3.c new file mode 100644 index 000000000000..92548cb6ec4f --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_3.c @@ -0,0 +1,48 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef int32_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, -2, INT32_MAX, INT32_MIN, 5, -7, 3, -4 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_4.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_4.c new file mode 100644 index 000000000000..6085a18bab7f --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_4.c @@ -0,0 +1,48 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +#include <limits.h> +#include <stdint.h> + +typedef int64_t elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1, -2, INT64_MAX, INT64_MIN, 5, -7, 3, -4 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_5.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_5.c new file mode 100644 index 000000000000..d61b91bb06dc --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_5.c @@ -0,0 +1,45 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +typedef float elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_6.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_6.c new file mode 100644 index 000000000000..7598f7a28bcf --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_6.c @@ -0,0 +1,45 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +typedef double elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1.0, 2.0, 1.0, 2.0, 1.0, 2.0 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_8.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_8.c new file mode 100644 index 000000000000..e1c337d44ead --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_8.c @@ -0,0 +1,45 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fwrapv -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +typedef float elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_9.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_9.c new file mode 100644 index 000000000000..13d962e2130f --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_9.c @@ -0,0 +1,45 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-additional-options "-Ofast -std=gnu99 -fwrapv -fdump-tree-vect-details -fdump-tree-widening_mul" } */ + +typedef double elem_t; + +__attribute__ ((noipa)) +elem_t +foo2 (elem_t *buf, int len) +{ + elem_t x = 0; + + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +static elem_t +reference (elem_t *buf, int len) +{ + elem_t x = 0; + +#pragma GCC novector + for (int i = 0; i < len; i++) + x += (elem_t) i * buf[i]; + + return x; +} + +int +main (void) +{ + elem_t buf[] = { 1.0, 2.0, 1.0, 2.0, 1.0, 2.0 }; + int len = sizeof (buf) / sizeof (buf[0]); + elem_t want = reference (buf, len); + elem_t got = foo2 (buf, len); + + if (want != got) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */ diff --git a/gcc/tree-ssa-math-opts.cc b/gcc/tree-ssa-math-opts.cc index 4c3fb0f4fc53..4b50a96ad3aa 100644 --- a/gcc/tree-ssa-math-opts.cc +++ b/gcc/tree-ssa-math-opts.cc @@ -3120,6 +3120,26 @@ convert_mult_to_fma_1 (tree mul_result, tree op1, tree op2) if (is_gimple_debug (use_stmt)) continue; + /* If the use is a type convert, look further into it if the operations + are the same under two's complement. */ + tree lhs_type; + if (gimple_assign_cast_p (use_stmt) + && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt))) + && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1))) + { + tree cast_lhs = gimple_get_lhs (use_stmt); + gimple *tmp_use; + use_operand_p tmp_use_p; + if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use)) + { + release_defs (use_stmt); + use_stmt = tmp_use; + result = cast_lhs; + gsi_remove (&gsi, true); + gsi = gsi_for_stmt (use_stmt); + } + } + if (is_gimple_assign (use_stmt) && gimple_assign_rhs_code (use_stmt) == NEGATE_EXPR) { @@ -3159,6 +3179,13 @@ convert_mult_to_fma_1 (tree mul_result, tree op1, tree op2) if (seq) gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT); + /* Ensure all the operands are of the same type. Use the type of the + addend as that's the statement being replaced. */ + op2 = gimple_convert (&gsi, true, GSI_SAME_STMT, + UNKNOWN_LOCATION, TREE_TYPE (addop), op2); + mulop1 = gimple_convert (&gsi, true, GSI_SAME_STMT, + UNKNOWN_LOCATION, TREE_TYPE (addop), mulop1); + if (len) fma_stmt = gimple_build_call_internal (IFN_COND_LEN_FMA, 7, cond, mulop1, op2, @@ -3419,6 +3446,20 @@ convert_mult_to_fma (gimple *mul_stmt, tree op1, tree op2, if (is_gimple_debug (use_stmt)) continue; + /* If the use is a type convert, look further into it if the operations + are the same under two's complement. */ + tree lhs_type; + if (gimple_assign_cast_p (use_stmt) + && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt))) + && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1))) + { + tree cast_lhs = gimple_get_lhs (use_stmt); + gimple *tmp_use; + use_operand_p tmp_use_p; + if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use)) + use_stmt = tmp_use; + } + /* For now restrict this operations to single basic blocks. In theory we would want to support sinking the multiplication in m = a*b;
