https://gcc.gnu.org/g:9e1c435c70a2ad4ca5285307b4a760f92c7a3426
commit r17-3899-g9e1c435c70a2ad4ca5285307b4a760f92c7a3426 Author: Kyrylo Tkachov <[email protected]> Date: Thu Sep 3 15:44:32 2026 +0200 aarch64: Use SVE2 SQXTNB and UQXTNB for saturating truncation vect_recog_sat_trunc_pattern forms .SAT_TRUNC only when the target has an sstrunc or ustrunc optab for the mode pair it selected. SVE2 has SQXTNB and UQXTNB, but only exposes them through unspecs for the ACLE, so a saturating truncation is vectorized as a minimum, or a minimum and a maximum, feeding a truncating store. The bottom forms write each result into the low half of its source element, which is where the element of the unpacked result mode lives. That holds when the source is itself unpacked, since the bits above each source element are then don't-care on both sides. Extend VNARROWQ and Vnarrowq to the SVE modes whose elements halve without changing the number of lanes, and provide the two optabs over SVE_HSDI. The unpacked source modes come out of that for free. For void f (int8_t *__restrict d, const int16_t *__restrict s, int n) { for (int i = 0; i < n; ++i) { int16_t x = s[i]; int8_t t = (int8_t) x; d[i] = -128 <= x && x <= 127 ? t : x < 0 ? -128 : 127; } } the inner loop changes from ld1h z31.h, p7/z, [x1, x3, lsl 1] smin z31.h, z31.h, #127 smax z31.h, z31.h, #-128 st1b z31.h, p7, [x0, x3] to ld1h z31.h, p7/z, [x1, x3, lsl 1] sqxtnb z31.b, z31.h st1b z31.h, p7, [x0, x3] This also makes the NARROW_CLIP form of .SAT_TRUNC reachable on aarch64, so add a runtime test for the PR tree-optimization/126981 fix on top of the target independent one. Bootstrapped and tested on aarch64-none-linux-gnu. gcc/ChangeLog: * config/aarch64/iterators.md (VNARROWQ, Vnarrowq): Add the SVE modes whose elements halve without changing the number of lanes. (Ventype): Add the unpacked SVE modes. * config/aarch64/aarch64-sve2.md (<sat_trunc_op>trunc<mode><Vnarrowq>2): New pattern. gcc/testsuite/ChangeLog: * gcc.target/aarch64/sve2/vect-sat-trunc-1.c: New test. * gcc.target/aarch64/sve2/vect-sat-trunc-1_run.c: Likewise. * gcc.target/aarch64/sve2/vect-sat-trunc-2.c: Likewise. * gcc.target/aarch64/sve2/vect-sat-trunc-2_run.c: Likewise. * gcc.target/aarch64/sve2/vect-sat-trunc-clip-1_run.c: Likewise. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/config/aarch64/aarch64-sve2.md | 13 +++++ gcc/config/aarch64/iterators.md | 15 ++++-- .../gcc.target/aarch64/sve2/vect-sat-trunc-1.c | 50 ++++++++++++++++++ .../gcc.target/aarch64/sve2/vect-sat-trunc-1_run.c | 59 ++++++++++++++++++++++ .../gcc.target/aarch64/sve2/vect-sat-trunc-2.c | 31 ++++++++++++ .../gcc.target/aarch64/sve2/vect-sat-trunc-2_run.c | 37 ++++++++++++++ .../aarch64/sve2/vect-sat-trunc-clip-1_run.c | 51 +++++++++++++++++++ 7 files changed, 253 insertions(+), 3 deletions(-) diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md index 3a1285a6a008..9e55781f10f5 100644 --- a/gcc/config/aarch64/aarch64-sve2.md +++ b/gcc/config/aarch64/aarch64-sve2.md @@ -3053,6 +3053,19 @@ ;; - UQXTNT ;; ------------------------------------------------------------------------- +;; The "bottom" narrowing instructions write each result into the low half of +;; its source element, which is where the element of the unpacked result mode +;; lives. This holds for an unpacked source too, since the bits above each +;; source element are then don't-care on both sides. +(define_insn "<sat_trunc_op>trunc<mode><Vnarrowq>2" + [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w") + (SAT_TRUNC:<VNARROWQ> + (match_operand:SVE_HSDI 1 "register_operand" "w")))] + "TARGET_SVE2" + "<shrn_op>xtnb\t%0.<Ventype>, %1.<Vetype>" + [(set_attr "sve_type" "sve_int_extract")] +) + (define_insn "@aarch64_sve_<sve_int_op><mode>" [(set (match_operand:<VNARROW> 0 "register_operand" "=w") (unspec:<VNARROW> diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md index b0571460ec7d..0c69c2148dce 100644 --- a/gcc/config/aarch64/iterators.md +++ b/gcc/config/aarch64/iterators.md @@ -2190,13 +2190,20 @@ (define_mode_attr Vnarrowd [(V4HI "v8qi") (V2SI "v4hi") (DI "v2si")]) -;; Narrowed double-modes for VQN (Used for XTN). +;; Modes with the same number of elements, each half as wide. Used for XTN +;; and for the SVE "bottom" narrowing instructions, whose result is unpacked. (define_mode_attr VNARROWQ [(V8HI "V8QI") (V4SI "V4HI") (V2DI "V2SI") (DI "SI") (SI "HI") - (HI "QI")]) + (HI "QI") + (VNx8HI "VNx8QI") (VNx4HI "VNx4QI") + (VNx2HI "VNx2QI") (VNx4SI "VNx4HI") + (VNx2SI "VNx2HI") (VNx2DI "VNx2SI")]) (define_mode_attr Vnarrowq [(V8HI "v8qi") (V4SI "v4hi") - (V2DI "v2si")]) + (V2DI "v2si") + (VNx8HI "vnx8qi") (VNx4HI "vnx4qi") + (VNx2HI "vnx2qi") (VNx4SI "vnx4hi") + (VNx2SI "vnx2hi") (VNx2DI "vnx2si")]) ;; Narrowed quad-modes for VQN (Used for XTN2). (define_mode_attr VNARROWQ2 [(V8HI "V16QI") (V4SI "V8HI") @@ -2347,7 +2354,9 @@ ;; SVE vector after narrowing. (define_mode_attr Ventype [(VNx8HI "b") (VNx8HF "b") + (VNx4HI "b") (VNx2HI "b") (VNx4SI "h") (VNx4SF "h") + (VNx2SI "h") (VNx2DI "s") (VNx2DF "s") (VNx8SI "h") (VNx16SI "b") (VNx8DI "h") (VNx16HI "b")]) diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-1.c b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-1.c new file mode 100644 index 000000000000..7c7459bfacd2 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-1.c @@ -0,0 +1,50 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -mautovec-preference=sve-only" } */ + +typedef __INT8_TYPE__ int8_t; +typedef __INT16_TYPE__ int16_t; +typedef __INT32_TYPE__ int32_t; +typedef __INT64_TYPE__ int64_t; +typedef __UINT8_TYPE__ uint8_t; +typedef __UINT16_TYPE__ uint16_t; +typedef __UINT32_TYPE__ uint32_t; +typedef __UINT64_TYPE__ uint64_t; + +#define DEF_UNSIGNED(NAME, OUT, IN, MAX) \ + void __attribute__((noipa)) \ + NAME (OUT *__restrict out, const IN *__restrict in, int n) \ + { \ + for (int i = 0; i < n; ++i) \ + { \ + IN x = in[i]; \ + out[i] = x > (IN) MAX ? (OUT) MAX : (OUT) x; \ + } \ + } + +#define DEF_SIGNED(NAME, OUT, IN, MIN, MAX) \ + void __attribute__((noipa)) \ + NAME (OUT *__restrict out, const IN *__restrict in, int n) \ + { \ + for (int i = 0; i < n; ++i) \ + { \ + IN x = in[i]; \ + OUT tmp = (OUT) x; \ + out[i] = ((IN) MIN <= x && x <= (IN) MAX \ + ? tmp : x < 0 ? (OUT) MIN : (OUT) MAX); \ + } \ + } + +DEF_UNSIGNED (u16_to_u8, uint8_t, uint16_t, 255) +DEF_UNSIGNED (u32_to_u16, uint16_t, uint32_t, 65535) +DEF_UNSIGNED (u64_to_u32, uint32_t, uint64_t, 4294967295ULL) +DEF_SIGNED (s16_to_s8, int8_t, int16_t, -128, 127) +DEF_SIGNED (s32_to_s16, int16_t, int32_t, -32768, 32767) +DEF_SIGNED (s64_to_s32, int32_t, int64_t, -2147483647 - 1, 2147483647) + +/* { dg-final { scan-assembler-times {\tuqxtnb\tz[0-9]+\.b, z[0-9]+\.h\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tuqxtnb\tz[0-9]+\.h, z[0-9]+\.s\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tuqxtnb\tz[0-9]+\.s, z[0-9]+\.d\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tsqxtnb\tz[0-9]+\.b, z[0-9]+\.h\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tsqxtnb\tz[0-9]+\.h, z[0-9]+\.s\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tsqxtnb\tz[0-9]+\.s, z[0-9]+\.d\n} 1 } } */ +/* { dg-final { scan-assembler-not {\t[su]min\tz} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-1_run.c new file mode 100644 index 000000000000..62da9e64b663 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-1_run.c @@ -0,0 +1,59 @@ +/* { dg-do run } */ +/* { dg-require-effective-target aarch64_sve2_hw } */ +/* { dg-options "-O3 -mautovec-preference=sve-only" } */ + +#include "vect-sat-trunc-1.c" + +/* Not a multiple of any SVE vector length, so that the loops run a + predicated tail iteration. */ +#define N 257 + +#define INIT(IN, ARR) \ + for (int i = 0; i < N; ++i) \ + (ARR)[i] = (i & 1) ? (IN) (i - N / 2) \ + : (IN) (i * 0x9e3779b97f4a7c15ULL) + +#define CHECK_UNSIGNED(NAME, OUT, IN, MAX) \ + do { \ + IN in[N]; \ + OUT out[N]; \ + INIT (IN, in); \ + NAME (out, in, N); \ + _Pragma ("GCC novector") \ + for (int i = 0; i < N; ++i) \ + { \ + IN x = in[i]; \ + if (out[i] != (OUT) (x > (IN) MAX ? (IN) MAX : x)) \ + __builtin_abort (); \ + } \ + } while (0) + +#define CHECK_SIGNED(NAME, OUT, IN, MIN, MAX) \ + do { \ + IN in[N]; \ + OUT out[N]; \ + INIT (IN, in); \ + NAME (out, in, N); \ + _Pragma ("GCC novector") \ + for (int i = 0; i < N; ++i) \ + { \ + IN x = in[i]; \ + OUT tmp = (OUT) x; \ + OUT ref = ((IN) MIN <= x && x <= (IN) MAX \ + ? tmp : x < 0 ? (OUT) MIN : (OUT) MAX); \ + if (out[i] != ref) \ + __builtin_abort (); \ + } \ + } while (0) + +int +main (void) +{ + CHECK_UNSIGNED (u16_to_u8, uint8_t, uint16_t, 255); + CHECK_UNSIGNED (u32_to_u16, uint16_t, uint32_t, 65535); + CHECK_UNSIGNED (u64_to_u32, uint32_t, uint64_t, 4294967295ULL); + CHECK_SIGNED (s16_to_s8, int8_t, int16_t, -128, 127); + CHECK_SIGNED (s32_to_s16, int16_t, int32_t, -32768, 32767); + CHECK_SIGNED (s64_to_s32, int32_t, int64_t, -2147483647 - 1, 2147483647); + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-2.c b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-2.c new file mode 100644 index 000000000000..0329f90d6c14 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-2.c @@ -0,0 +1,31 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -mautovec-preference=sve-only" } */ + +typedef __UINT8_TYPE__ uint8_t; +typedef __UINT16_TYPE__ uint16_t; +typedef __UINT32_TYPE__ uint32_t; +typedef __UINT64_TYPE__ uint64_t; + +/* The second store has a wider element than the saturating narrowing, so the + prevailing vector mode makes the narrowing operate on unpacked modes. */ + +#define DEF(NAME, OUT, IN, WIDE, MAX) \ + void __attribute__((noipa)) \ + NAME (OUT *__restrict out, WIDE *__restrict wide, \ + const IN *__restrict in, int n) \ + { \ + for (int i = 0; i < n; ++i) \ + { \ + IN x = in[i]; \ + out[i] = x > (IN) MAX ? (OUT) MAX : (OUT) x; \ + wide[i] = (WIDE) x * 3; \ + } \ + } + +DEF (u16_to_u8_in_s, uint8_t, uint16_t, uint32_t, 255) +DEF (u16_to_u8_in_d, uint8_t, uint16_t, uint64_t, 255) +DEF (u32_to_u16_in_d, uint16_t, uint32_t, uint64_t, 65535) + +/* { dg-final { scan-assembler-times {\tuqxtnb\tz[0-9]+\.b, z[0-9]+\.h\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tuqxtnb\tz[0-9]+\.h, z[0-9]+\.s\n} 1 } } */ +/* { dg-final { scan-assembler-not {\tumin\tz} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-2_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-2_run.c new file mode 100644 index 000000000000..e50d426618da --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-2_run.c @@ -0,0 +1,37 @@ +/* { dg-do run } */ +/* { dg-require-effective-target aarch64_sve2_hw } */ +/* { dg-options "-O3 -mautovec-preference=sve-only" } */ + +#include "vect-sat-trunc-2.c" + +/* Not a multiple of any SVE vector length, so that the loops run a + predicated tail iteration. */ +#define N 257 + +#define CHECK(NAME, OUT, IN, WIDE, MAX) \ + do { \ + IN in[N]; \ + OUT out[N]; \ + WIDE wide[N]; \ + for (int i = 0; i < N; ++i) \ + in[i] = (i & 1) ? (IN) (i - N / 2) \ + : (IN) (i * 0x9e3779b97f4a7c15ULL); \ + NAME (out, wide, in, N); \ + _Pragma ("GCC novector") \ + for (int i = 0; i < N; ++i) \ + { \ + IN x = in[i]; \ + if (out[i] != (OUT) (x > (IN) MAX ? (IN) MAX : x) \ + || wide[i] != (WIDE) x * 3) \ + __builtin_abort (); \ + } \ + } while (0) + +int +main (void) +{ + CHECK (u16_to_u8_in_s, uint8_t, uint16_t, uint32_t, 255); + CHECK (u16_to_u8_in_d, uint8_t, uint16_t, uint64_t, 255); + CHECK (u32_to_u16_in_d, uint16_t, uint32_t, uint64_t, 65535); + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-clip-1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-clip-1_run.c new file mode 100644 index 000000000000..190ec23ed2ef --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-clip-1_run.c @@ -0,0 +1,51 @@ +/* { dg-do run } */ +/* { dg-require-effective-target aarch64_sve2_hw } */ +/* { dg-options "-O3 -mautovec-preference=sve-only" } */ + +typedef __UINT16_TYPE__ uint16_t; +typedef __UINT32_TYPE__ uint32_t; + +/* Not a multiple of any SVE vector length, so that the loop runs a + predicated tail iteration. */ +#define N 257 + +/* Clipping to [0, 65535] written so that the negation cannot overflow. At + INT_MIN the negation is its own inverse, so the shift yields -1 and the + result is 65535 rather than 0. */ + +void __attribute__((noipa)) +clip (uint16_t *__restrict out, const int *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + int x = in[i]; + out[i] = ((uint32_t) x > 65535u + ? (int) (-(uint32_t) x) >> 31 + : x); + } +} + +int +main (void) +{ + int in[N]; + uint16_t out[N]; + + for (int i = 0; i < N; ++i) + in[i] = (i & 3) == 0 ? (-__INT_MAX__ - 1) : i * 12345 - 30000; + + clip (out, in, N); + +#pragma GCC novector + for (int i = 0; i < N; ++i) + { + int x = in[i]; + uint16_t ref = ((uint32_t) x > 65535u + ? (int) (-(uint32_t) x) >> 31 + : x); + if (out[i] != ref) + __builtin_abort (); + } + + return 0; +}
