https://gcc.gnu.org/g:1e1d34b23316f7cc87773edcda5567764af41cc8
commit r17-5179-g1e1d34b23316f7cc87773edcda5567764af41cc8 Author: liuhongt <[email protected]> Date: Fri Oct 9 22:37:16 2026 -0700 i386: Improve vzeroupper placement with IPA-RA [PR127773] The restriction introduced in r10-3477 prevents vzeroupper from clobbering wide values kept in callee-preserved SSE registers. r14-6447 also stopped assuming clean upper state after calls to callees that preserve SSE upper halves. This is too conservative when no preserved SSE register is live across the call. With -fipa-ra, cleanup then goes after a legacy SSE call, leaving the callee to pay the AVX/SSE transition penalties in PR127773. TARGET_MODE_NEEDED now provides LIVE. Use it, excluding registers that die at the call, to retain the correctness protection only when a preserved SSE register is live across the call. Keep the incoming mode after calls with partial clobbers to avoid repeating cleanup afterward. Leave upper state unchanged before calls that clobber no SSE registers, so integer-only callees do not force cleanup in AVX loops. Fixed registers are not counted as clobbered by the callee, e.g. xmm8-xmm15 for ia32 or an SSE register given by -ffixed-xmmN. gcc/ChangeLog: PR target/127773 * config/i386/i386.cc (ix86_avx_u128_mode_needed): Add LIVE argument. Check live preserved SSE registers before requiring AVX_U128_CLEAN. Do not require clean upper state for callees that clobber no SSE registers, ignoring fixed registers. (ix86_mode_needed): Pass LIVE. (ix86_avx_u128_mode_after): Keep MODE after partial-clobber calls. gcc/testsuite/ChangeLog: PR target/127773 * gcc.target/i386/pr127773-1.c: New test. * gcc.target/i386/pr127773-2.c: New test. * gcc.target/i386/pr127773-3.c: New test. * gcc.target/i386/pr127773-4.c: New test. * gcc.target/i386/pr127773-5.c: New test. * gcc.target/i386/pr127773-6.c: New test. Diff: --- gcc/config/i386/i386.cc | 54 +++++++++++++++++---------- gcc/testsuite/gcc.target/i386/pr127773-1.c | 21 +++++++++++ gcc/testsuite/gcc.target/i386/pr127773-2.c | 23 ++++++++++++ gcc/testsuite/gcc.target/i386/pr127773-3.c | 60 ++++++++++++++++++++++++++++++ gcc/testsuite/gcc.target/i386/pr127773-4.c | 27 ++++++++++++++ gcc/testsuite/gcc.target/i386/pr127773-5.c | 8 ++++ gcc/testsuite/gcc.target/i386/pr127773-6.c | 22 +++++++++++ 7 files changed, 196 insertions(+), 19 deletions(-) diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc index 553fa046fa94..4dba04774e86 100644 --- a/gcc/config/i386/i386.cc +++ b/gcc/config/i386/i386.cc @@ -15610,10 +15610,11 @@ ix86_check_avx_upper_stores (rtx dest, const_rtx, void *data) } } -/* Return needed mode for entity in optimize_mode_switching pass. */ +/* Return needed mode for entity in optimize_mode_switching pass. + LIVE is the set of hard registers live before INSN. */ static int -ix86_avx_u128_mode_needed (rtx_insn *insn) +ix86_avx_u128_mode_needed (rtx_insn *insn, HARD_REG_SET live) { if (DEBUG_INSN_P (insn)) return AVX_U128_ANY; @@ -15645,21 +15646,36 @@ ix86_avx_u128_mode_needed (rtx_insn *insn) if (avx_upper_reg_found) return AVX_U128_DIRTY; - /* If the function is known to preserve some SSE registers, - RA and previous passes can legitimately rely on that for - modes wider than 256 bits. It's only safe to issue a - vzeroupper if all SSE registers are clobbered. */ + if (vzeroupper_pattern (PATTERN (insn), VOIDmode)) + return AVX_U128_ANY; + + /* Should be safe to issue a vzeroupper before sibling_call_p. */ + if (SIBLING_CALL_P (insn)) + return AVX_U128_CLEAN; + const function_abi &abi = insn_callee_abi (insn); - if (vzeroupper_pattern (PATTERN (insn), VOIDmode) - /* Should be safe to issue an vzeroupper before sibling_call_p. - Also there not mode_exit for sibling_call, so there could be - missing vzeroupper for that. */ - || !(SIBLING_CALL_P (insn) - || hard_reg_set_subset_p (reg_class_contents[SSE_REGS], - abi.mode_clobbers (V4DImode)))) + + /* IPA-RA conservatively records fixed registers as used by the + callee, e.g. xmm8-xmm15 for ia32, so ignore them. */ + HARD_REG_SET sse_clobbers + = (reg_class_contents[SSE_REGS] & ~fixed_reg_set + & abi.mode_clobbers (V4DImode)); + + /* A callee with no SSE register writes does not need clean upper + state. */ + if (hard_reg_set_empty_p (sse_clobbers)) return AVX_U128_ANY; - return AVX_U128_CLEAN; + /* vzeroupper must not clobber values preserved across the call. */ + live &= reg_class_contents[SSE_REGS] & ~sse_clobbers; + + /* Exclude registers that die at the call. */ + for (link = REG_NOTES (insn); link; link = XEXP (link, 1)) + if (REG_NOTE_KIND (link) == REG_DEAD && REG_P (XEXP (link, 0))) + remove_from_hard_reg_set (&live, GET_MODE (XEXP (link, 0)), + REGNO (XEXP (link, 0))); + + return hard_reg_set_empty_p (live) ? AVX_U128_CLEAN : AVX_U128_ANY; } rtx set = single_set (insn); @@ -15755,14 +15771,14 @@ ix86_i387_mode_needed (int entity, rtx_insn *insn) prior to the execution of insn. */ static int -ix86_mode_needed (int entity, rtx_insn *insn, HARD_REG_SET) +ix86_mode_needed (int entity, rtx_insn *insn, HARD_REG_SET live) { switch (entity) { case X86_DIRFLAG: return ix86_dirflag_mode_needed (insn); case AVX_U128: - return ix86_avx_u128_mode_needed (insn); + return ix86_avx_u128_mode_needed (insn, live); case I387_ROUNDEVEN: case I387_TRUNC: case I387_FLOOR: @@ -15796,13 +15812,13 @@ ix86_avx_u128_mode_after (int mode, rtx_insn *insn) return AVX_U128_DIRTY; /* If the function doesn't clobber any sse registers or only clobber - 128-bit part, Then vzeroupper isn't issued before the function exit. - the status not CLEAN but ANY after the function. */ + 128-bit part, then vzeroupper isn't issued before the function exit, + and the function doesn't change the status. */ const function_abi &abi = insn_callee_abi (insn); if (!(SIBLING_CALL_P (insn) || hard_reg_set_subset_p (reg_class_contents[SSE_REGS], abi.mode_clobbers (V4DImode)))) - return AVX_U128_ANY; + return mode; return AVX_U128_CLEAN; } diff --git a/gcc/testsuite/gcc.target/i386/pr127773-1.c b/gcc/testsuite/gcc.target/i386/pr127773-1.c new file mode 100644 index 000000000000..4265c72aacc6 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr127773-1.c @@ -0,0 +1,21 @@ +/* PR target/127773 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=x86-64" } */ +/* { dg-final { scan-assembler-times "vzeroupper" 1 } } */ +/* { dg-final { scan-assembler "vzeroupper.*call\[ \t\]+_?f\\M" } } */ + +typedef char v32qi __attribute__ ((vector_size (32))); + +static __attribute__ ((noinline)) int +f (void) +{ + __asm__ volatile ("pxor %%xmm0, %%xmm0" ::: "xmm0"); + return 0; +} + +__attribute__ ((target ("avx2"))) int +g (v32qi *p) +{ + *p += *p; + return f () + 1; +} diff --git a/gcc/testsuite/gcc.target/i386/pr127773-2.c b/gcc/testsuite/gcc.target/i386/pr127773-2.c new file mode 100644 index 000000000000..9c87c0c720eb --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr127773-2.c @@ -0,0 +1,23 @@ +/* PR target/127773 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=x86-64 -mfpmath=sse" } */ +/* { dg-final { scan-assembler-times "vzeroupper" 1 } } */ +/* { dg-final { scan-assembler "vzeroupper.*call\[ \t\]+_?f\\M" } } */ + +typedef char v32qi __attribute__ ((vector_size (32))); + +static __attribute__ ((noinline)) double +f (double *a, int n) +{ + double s = 0; + for (int i = 0; i < n; i++) + s += a[i] * a[i]; + return s; +} + +__attribute__ ((target ("avx2"))) double +g (v32qi *p, double *a, int n) +{ + *p += *p; + return f (a, n) + 1; +} diff --git a/gcc/testsuite/gcc.target/i386/pr127773-3.c b/gcc/testsuite/gcc.target/i386/pr127773-3.c new file mode 100644 index 000000000000..6e4272f75258 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr127773-3.c @@ -0,0 +1,60 @@ +/* PR target/127773 */ +/* { dg-do run } */ +/* { dg-require-effective-target avx2 } */ +/* { dg-options "-O2 -mavx2" } */ + +#include "avx2-check.h" + +typedef long long v4di __attribute__ ((vector_size (32))); +typedef double v4df __attribute__ ((vector_size (32))); + +static __attribute__ ((noinline)) int +f (int x) +{ + return x * 3 + 1; +} + +static __attribute__ ((noinline)) double +fd (double x) +{ + return x * 3.0; +} + +/* Check that vzeroupper does not clobber vectors live across calls. */ + +__attribute__ ((noinline)) void +g1 (v4di *p, int x) +{ + v4di t = *p + *p; + int r = f (x); + *p = t + r; +} + +__attribute__ ((noinline)) double +g2 (v4df *p, double d, int c) +{ + v4df t = *p * 2.0; + double s = 0; + for (int i = 0; i < c; i++) + { + s += fd (d + i); + if (i & 1) + t += s; + } + *p = t; + return s; +} + +static void +avx2_test (void) +{ + v4di a = { 1, 2, 3, 4 }; + g1 (&a, 5); + if (a[0] != 18 || a[3] != 24) + abort (); + + v4df b = { 1, 2, 3, 4 }; + double s = g2 (&b, 1.0, 4); + if (s != 3.0 * (1 + 2 + 3 + 4) || b[3] != 8.0 + 9.0 + 30.0) + abort (); +} diff --git a/gcc/testsuite/gcc.target/i386/pr127773-4.c b/gcc/testsuite/gcc.target/i386/pr127773-4.c new file mode 100644 index 000000000000..4ed3f5c6704f --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr127773-4.c @@ -0,0 +1,27 @@ +/* PR target/127773 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=x86-64" } */ +/* f doesn't write any SSE register, so there is no need to issue + vzeroupper before calling it in the loop. */ +/* { dg-final { scan-assembler-times "vzeroupper" 1 } } */ +/* { dg-final { scan-assembler "call\[ \t\]+_?f\\M.*vzeroupper" } } */ + +typedef long long v4di __attribute__ ((vector_size (32))); + +static __attribute__ ((noinline)) int +f (int x) +{ + return x * 3 + 1; +} + +__attribute__ ((target ("avx2"))) int +g (v4di *p, int n) +{ + int s = 0; + for (int i = 0; i < n; i++) + { + p[i] += p[i]; + s += f (i); + } + return s; +} diff --git a/gcc/testsuite/gcc.target/i386/pr127773-5.c b/gcc/testsuite/gcc.target/i386/pr127773-5.c new file mode 100644 index 000000000000..b1be9bf990a7 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr127773-5.c @@ -0,0 +1,8 @@ +/* PR target/127773 */ +/* { dg-do compile { target { ! ia32 } } } */ +/* { dg-options "-O2 -march=x86-64 -ffixed-xmm8" } */ +/* The fixed xmm8 recorded by IPA-RA isn't an SSE register written by f. */ +/* { dg-final { scan-assembler-times "vzeroupper" 1 } } */ +/* { dg-final { scan-assembler "call\[ \t\]+_?f\\M.*vzeroupper" } } */ + +#include "pr127773-4.c" diff --git a/gcc/testsuite/gcc.target/i386/pr127773-6.c b/gcc/testsuite/gcc.target/i386/pr127773-6.c new file mode 100644 index 000000000000..384b84855e12 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr127773-6.c @@ -0,0 +1,22 @@ +/* PR target/127773 */ +/* { dg-do compile { target { ! ia32 } } } */ +/* { dg-options "-O2 -mavx2" } */ +/* The global register variable is live across the call to f, so + vzeroupper must not be issued before the call. */ +/* { dg-final { scan-assembler-not "vzeroupper.*call\[ \t\]+_?f\\M" } } */ + +typedef double v4df __attribute__ ((vector_size (32))); +register v4df gv asm ("xmm8"); /* { dg-warning "call-clobbered register used for global register variable" } */ + +static __attribute__ ((noinline)) double +f (double x) +{ + return x * 3.0; +} + +double +g (v4df *p, double x) +{ + gv += *p; + return f (x) + 1; +}
