Update in V3:
> > + if (SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group
>
> ... I wonder about this, a slp_inst_kind_reduc_group is multiple
> independent reductions (or one) in the same vector, so for that
> we have SLP_TREE_LANES independent fold-left reductions of
> VF lanes each. Esp. when VF == 1 this means the fold-left
> reductions are "free" (this correlates well with the
> heuristic to prefer smaller vectors)
>
> And for slp_inst_kind_reduc_chain we have a single fold-left
> reduction with VF * SLP_TREE_LANES lanes.
>
> I guess for the overall sum this difference doesn't matter, but
> you do not cover slp_inst_kind_reduc_chain at all here? So
> add a || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain?
Added || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain
Bootstrapped and regtested on x86_64-pc-linux-gnu{-m32,}.
Ok for trunk?
-------------------------
Fold-left FP reductions are lowered to scalar operations on x86. In
mixed-width loops the per-statement costs can therefore make a wider
vector mode look better even though it lengthens the serial reduction
chain.
Count the fold-left reduction lanes from reduction groups and chains during
finish_cost and use that as an extra loop-candidate preference. Do not let
emulated sub-SSE modes win on this preference alone. Dump the lane count when
present.
gcc/ChangeLog:
* config/i386/i386.cc (ix86_vector_costs): Add
better_fold_left_reduc_than_p and m_num_fold_left_reduc_lanes.
(ix86_vector_costs::ix86_vector_costs): Initialize it.
(ix86_vector_costs::finish_cost): Count and dump fold-left
reduction lanes.
(ix86_vector_costs::better_fold_left_reduc_than_p): New function.
(ix86_vector_costs::better_main_loop_than_p): Use it.
(ix86_vector_costs::better_epilogue_loop_than_p): Likewise.
gcc/testsuite/ChangeLog:
* gcc.target/i386/fold-left-reduc-cost.c: New test.
* gcc.target/i386/fold-left-reduc-chain-cost.c: New test.
* gcc.target/i386/vect-epilogues-6.c: Update expected vector size.
* gcc.target/i386/vect-epilogues-7.c: Likewise.
* gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c: Likewise.
---
gcc/config/i386/i386.cc | 42 ++++++++++++++++++-
.../costmodel/x86_64/costmodel-vect-epil-1.c | 7 ++--
.../i386/fold-left-reduc-chain-cost.c | 19 +++++++++
.../gcc.target/i386/fold-left-reduc-cost.c | 18 ++++++++
.../gcc.target/i386/vect-epilogues-6.c | 3 +-
.../gcc.target/i386/vect-epilogues-7.c | 4 +-
6 files changed, 84 insertions(+), 9 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
create mode 100644 gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index e66958db7ac..8e22a43fbd1 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -26172,6 +26172,7 @@ public:
private:
+ bool better_fold_left_reduc_than_p (const vector_costs *) const;
/* Estimate register pressure of the vectorized code. */
void ix86_vect_estimate_reg_pressure ();
/* Number of GENERAL_REGS/SSE_REGS used in the vectorizer, it's used for
@@ -26188,6 +26189,8 @@ private:
unsigned m_num_reduc[X86_REDUC_LAST];
/* Don't do unroll if m_prefer_unroll is false, default is true. */
bool m_prefer_unroll;
+ /* Scalar lanes in fold-left reductions. */
+ unsigned int m_num_fold_left_reduc_lanes;
};
ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool costing_for_scalar)
@@ -26197,7 +26200,8 @@ ix86_vector_costs::ix86_vector_costs (vec_info* vinfo,
bool costing_for_scalar)
m_num_avx256_vec_perm (),
m_num_avx512_vec_perm (),
m_num_reduc (),
- m_prefer_unroll (true)
+ m_prefer_unroll (true),
+ m_num_fold_left_reduc_lanes (0)
{}
/* Implement targetm.vectorize.create_costs. */
@@ -26903,6 +26907,20 @@ ix86_vector_costs::finish_cost (const vector_costs
*scalar_costs)
loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
if (loop_vinfo && !m_costing_for_scalar)
{
+ unsigned int vf = vect_vf_for_cost (loop_vinfo);
+ for (auto inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
+ if ((SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group
+ || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain)
+ && (vect_reduc_type (loop_vinfo, SLP_INSTANCE_TREE (inst))
+ == FOLD_LEFT_REDUCTION))
+ m_num_fold_left_reduc_lanes
+ += vf * SLP_TREE_LANES (SLP_INSTANCE_TREE (inst));
+
+ if (m_num_fold_left_reduc_lanes && dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "in-order FP reduction lanes: %u\n",
+ m_num_fold_left_reduc_lanes);
+
/* We are currently not asking the vectorizer to compare costs
between different vector mode sizes. When using predication
that will end up always choosing the preferred mode size even
@@ -27067,6 +27085,21 @@ ix86_vector_costs::finish_cost (const vector_costs
*scalar_costs)
vector_costs::finish_cost (scalar_costs);
}
+/* Return true if THIS has a shorter fold-left reduction chain than OTHER. */
+
+bool
+ix86_vector_costs::better_fold_left_reduc_than_p
+ (const vector_costs *other) const
+{
+ auto other_costs = static_cast<const ix86_vector_costs *> (other);
+ if (m_num_fold_left_reduc_lanes >= other_costs->m_num_fold_left_reduc_lanes)
+ return false;
+
+ /* Do not let emulated sub-SSE modes win on this alone. */
+ loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
+ return known_ge (GET_MODE_SIZE (loop_vinfo->vector_mode), 16);
+}
+
/* Return true if THIS should be preferred over OTHER as main vector loop. */
bool
@@ -27075,6 +27108,9 @@ ix86_vector_costs::better_main_loop_than_p (const
vector_costs *other) const
loop_vec_info this_loop_vinfo = as_a<loop_vec_info> (this->vinfo ());
loop_vec_info other_loop_vinfo = as_a<loop_vec_info> (other->vinfo ());
+ if (better_fold_left_reduc_than_p (other))
+ return true;
+
/* If the other loop is masked it does not need an epilog. Prefer that
if the current loop cannot be vectorized fully with a vector
epilogs with at most one scalar iteration left. */
@@ -27097,6 +27133,10 @@ ix86_vector_costs::better_epilogue_loop_than_p (const
vector_costs *other,
loop_vec_info main_loop) const
{
loop_vec_info this_loop_info = as_a <loop_vec_info> (this->vinfo ());
+
+ if (better_fold_left_reduc_than_p (other))
+ return true;
+
/* The x86 target allows for multiple vector epilogues, if THIS is
the suggested epilog mode of OTHER then keep the latter unless
THIS has a VF of one which means no further epilog needed. */
diff --git a/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
b/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
index abb9f6681c2..370e7ed4d11 100644
--- a/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
+++ b/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
@@ -52,7 +52,6 @@ void test (const unsigned char * __restrict__ pi,
pp_avg_rgb[3] = pp_avg_rgb_3;
}
-/* Even though there's an SLP opportunity in-order reductions should never use
- masked epilogs. */
-/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte
vectors" "vect" } } */
-/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using 32
byte vectors" "vect" } } */
+/* In-order reductions should not use masked epilogs here. */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized using 16 byte
vectors" "vect" } } */
+/* { dg-final { scan-tree-dump-not "optimized: epilogue loop vectorized using
masked" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
b/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
new file mode 100644
index 00000000000..634647287d8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fdump-tree-vect-details" } */
+
+/* The two dependent updates form a fold-left reduction chain. */
+
+float
+foo (float *__restrict__ a, int n)
+{
+ float sum = 0;
+ for (int i = 0; i != n; i++)
+ {
+ sum += a[2 * i];
+ sum += a[2 * i + 1];
+ }
+ return sum;
+}
+
+/* { dg-final { scan-tree-dump "Starting SLP discovery of reduction chain"
"vect" } } */
+/* { dg-final { scan-tree-dump "in-order FP reduction lanes" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
b/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
new file mode 100644
index 00000000000..6babd761d6f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fdump-tree-vect-details" } */
+
+/* The byte loads make V32QI available, but the fold-left reduction should
+ make the smaller SSE loop win. */
+
+float
+foo (char *a, char *b, int n)
+{
+ float sum = 0;
+ for (int i = 0; i != n; i++)
+ sum += a[i] * b[i];
+ return sum;
+}
+
+/* { dg-final { scan-tree-dump "in-order FP reduction lanes" "vect" } } */
+/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors" "vect"
} } */
+/* { dg-final { scan-tree-dump-not "loop vectorized using 32 byte vectors"
"vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
b/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
index 8cd8740c6ec..9f0445390f4 100644
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
@@ -17,5 +17,4 @@ foo (double *a, char *mask, int n)
return sum;
}
-/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte
vectors" "vect" } } */
-/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using 32
byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized using 32 byte
vectors" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
b/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
index 63c29895f9b..be56819fc54 100644
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
@@ -17,5 +17,5 @@ foo (double *a, char *mask, int n)
return sum;
}
-/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte
vectors" "vect" } } */
-/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using
masked 64 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized using 32 byte
vectors" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using
masked 32 byte vectors" "vect" } } */
--
2.34.1
