https://gcc.gnu.org/g:b3d6786029cc55d7f4dbb43466ec4d18c6bc8334
commit r16-6791-gb3d6786029cc55d7f4dbb43466ec4d18c6bc8334 Author: Kugan Vivekanandarajah <[email protected]> Date: Thu Jan 15 14:37:50 2026 +1100 [AutoFDO] Add hierarchical discriminator for vectorizer Add hierarchical discriminator support for vectorized loop versioning. Assigns copyid discriminators to distinguish vectorized and scalar versions. gcc/ChangeLog: * tree-vect-loop-manip.cc (vect_loop_versioning): Assign copyid discriminators for vectorized and scalar loop versions. * tree-vect-loop.cc (vect_transform_loop): Assign copyid discriminators for vectorized and scalar loop versions. gcc/testsuite/ChangeLog: * gcc.dg/hierarchical-discriminator-vect-version.c: New test. Signed-off-by: Kugan Vivekanandarajah <[email protected]> Diff: --- .../hierarchical-discriminator-vect-version.c | 17 ++++++++++ gcc/tree-vect-loop-manip.cc | 39 ++++++++++++++++++++++ gcc/tree-vect-loop.cc | 18 ++++++++++ 3 files changed, 74 insertions(+) diff --git a/gcc/testsuite/gcc.dg/hierarchical-discriminator-vect-version.c b/gcc/testsuite/gcc.dg/hierarchical-discriminator-vect-version.c new file mode 100644 index 000000000000..5dc945f1274d --- /dev/null +++ b/gcc/testsuite/gcc.dg/hierarchical-discriminator-vect-version.c @@ -0,0 +1,17 @@ +/* Test that vectorization assigns multiplicity and copyid discriminators. + { dg-do compile } + { dg-options "-O3 -g -ftree-vectorize -fno-vect-cost-model" } + { dg-require-effective-target vect_int } + */ + +void +test_vectorize (int *a, int *b, int *c, int n) +{ + int i; + for (i = 0; i < n; i++) + { + a[i] = b[i] + c[i]; /* Main vectorized loop and epilog */ + } +} + +/* { dg-final { scan-assembler "\\.loc 1 13 \[0-9\]+ is_stmt 0 discriminator \[0-9\]+" } } */ diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc index 0719e2cf493d..f28d89ce2d10 100644 --- a/gcc/tree-vect-loop-manip.cc +++ b/gcc/tree-vect-loop-manip.cc @@ -53,6 +53,8 @@ along with GCC; see the file COPYING3. If not see #include "langhooks.h" #include "tree-vector-builder.h" #include "optabs-tree.h" +#include "hierarchical_discriminator.h" + /************************************************************************* Simple Loop Peeling Utilities @@ -3464,6 +3466,15 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, gcc_assert (prolog); prolog->force_vectorize = false; + /* Assign hierarchical discriminators to distinguish prolog loop. */ + gimple *prolog_last = last_nondebug_stmt (prolog->header); + location_t prolog_loc + = prolog_last ? gimple_location (prolog_last) : UNKNOWN_LOCATION; + if (prolog_loc != UNKNOWN_LOCATION) + { + unsigned int prolog_copyid = allocate_copyid_base (prolog_loc, 1); + assign_discriminators_to_loop (prolog, 0, prolog_copyid); + } first_loop = prolog; reset_original_copy_tables (); @@ -3573,6 +3584,22 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, epilog->force_vectorize = false; bb_before_epilog = loop_preheader_edge (epilog)->src; + /* Assign hierarchical discriminators to distinguish epilog loop. + Only assign if it's a scalar epilog. If it will be vectorized + (vect_epilogues), discriminators will be assigned. + Use dynamic copy_id allocation instead of hardcoded constants. */ + if (!vect_epilogues) + { + gimple *epilog_last = last_nondebug_stmt (epilog->header); + location_t epilog_loc + = epilog_last ? gimple_location (epilog_last) : UNKNOWN_LOCATION; + if (epilog_loc != UNKNOWN_LOCATION) + { + unsigned int epilog_copyid = allocate_copyid_base (epilog_loc, 1); + assign_discriminators_to_loop (epilog, 0, epilog_copyid); + } + } + /* Scalar version loop may be preferred. In this case, add guard and skip to epilog. Note this only happens when the number of iterations of loop is unknown at compile time, otherwise this @@ -4501,6 +4528,18 @@ vect_loop_versioning (loop_vec_info loop_vinfo, gcc_assert (nloop); nloop = get_loop_copy (loop); + /* Assign hierarchical discriminators to distinguish loop versions. + Only assign to the scalar version here; the vectorized version will + get discriminators later during transformation/peeling. + Use dynamic copy_id allocation instead of hardcoded constants. */ + gimple *nloop_last = last_nondebug_stmt (nloop->header); + location_t nloop_loc + = nloop_last ? gimple_location (nloop_last) : UNKNOWN_LOCATION; + if (nloop_loc != UNKNOWN_LOCATION) + { + unsigned int nloop_copyid = allocate_copyid_base (nloop_loc, 1); + assign_discriminators_to_loop (nloop, 0, nloop_copyid); + } /* For cycle vectorization with SLP we rely on the PHI arguments appearing in the same order as the SLP node operands which for the loop PHI nodes means the preheader edge dest index needs to remain diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index fdf544fa47b0..9c2345ce955d 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -59,6 +59,7 @@ along with GCC; see the file COPYING3. If not see #include "case-cfn-macros.h" #include "langhooks.h" #include "opts.h" +#include "hierarchical_discriminator.h" /* Loop Vectorization Pass. @@ -11210,6 +11211,23 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call) &step_vector, &niters_vector_mult_vf, th, check_profitability, niters_no_overflow, &advance); + + /* Assign hierarchical discriminators to the vectorized loop. */ + poly_uint64 vf_val = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + unsigned int vf_int = constant_lower_bound (vf_val); + if (vf_int > DISCR_MULTIPLICITY_MAX) + vf_int = DISCR_MULTIPLICITY_MAX; + + /* Assign unique copy_id dynamically instead of using hardcoded constants. + Epilogue and main vectorized loops get different copy_ids. */ + gimple *loop_last = last_nondebug_stmt (loop->header); + location_t loop_loc + = loop_last ? gimple_location (loop_last) : UNKNOWN_LOCATION; + if (loop_loc != UNKNOWN_LOCATION) + { + unsigned int copyid = allocate_copyid_base (loop_loc, 1); + assign_discriminators_to_loop (loop, vf_int, copyid); + } if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo) && LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo).initialized_p ()) {
