https://gcc.gnu.org/g:89c443a7e9a0780a52a698fb02d4f5173e025918
commit r14-9335-g89c443a7e9a0780a52a698fb02d4f5173e025918
Author: Richard Biener <rguent...@suse.de>
Date: Wed Mar 6 10:31:02 2024 +0100
tree-optimization/114239 - rework reduction epilogue driving
The following reworks vectorizable_live_operation to pass the
live stmt to vect_create_epilog_for_reduction also for early breaks
and a peeled main exit. This is to be able to figure the scalar
definition to replace. This reverts the PR114192 fix as it is
subsumed by this cleanup.
PR tree-optimization/114239
* tree-vect-loop.cc (vect_get_vect_def): Remove.
(vect_create_epilog_for_reduction): The passed in stmt_info
should now be the live stmt that produces the scalar reduction
result. Revert PR114192 fix. Base reduction info off
info_for_reduction. Remove special handling of
early-break/peeled, restore original vector def gathering.
Make sure to pick the correct exit PHIs.
(vectorizable_live_operation): Pass in the proper stmt_info
for early break exits.
* gcc.dg/vect/vect-early-break_122-pr114239.c: New testcase.
Diff:
---
.../gcc.dg/vect/vect-early-break_122-pr114239.c | 29 ++++++
gcc/tree-vect-loop.cc | 105 +++++----------------
2 files changed, 53 insertions(+), 81 deletions(-)
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_122-pr114239.c
b/gcc/testsuite/gcc.dg/vect/vect-early-break_122-pr114239.c
new file mode 100644
index 00000000000..7bf4db14209
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_122-pr114239.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+/* { dg-add-options vect_early_break } */
+/* { dg-require-effective-target vect_early_break } */
+
+int ip4_getbit_a, ip4_getbit_pos, ip4_clrbit_pos;
+void ip4_clrbit(int *a) { *a &= ip4_clrbit_pos; }
+typedef struct {
+ char pxlen;
+ int prefix;
+} net_addr_ip4;
+void fib_get_chain();
+int trie_match_longest_ip4();
+int trie_match_next_longest_ip4(net_addr_ip4 *n) {
+ int __trans_tmp_1;
+ while (n->pxlen) {
+ n->pxlen--;
+ ip4_clrbit(&n->prefix);
+ __trans_tmp_1 = ip4_getbit_a >> ip4_getbit_pos;
+ if (__trans_tmp_1)
+ return 1;
+ }
+ return 0;
+}
+void net_roa_check_ip4_trie_tab() {
+ net_addr_ip4 px0;
+ for (int _n = trie_match_longest_ip4(&px0); _n;
+ _n = trie_match_next_longest_ip4(&px0))
+ fib_get_chain();
+}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 761cdc67570..20ee0aad932 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -5897,35 +5897,6 @@ vect_create_partial_epilog (tree vec_def, tree vectype,
code_helper code,
return new_temp;
}
-/* Retrieves the definining statement to be used for a reduction.
- For LAST_VAL_REDUC_P we use the current VEC_STMTs which correspond to the
- final value after vectorization and otherwise we look at the reduction
- definitions to get the first. */
-
-tree
-vect_get_vect_def (stmt_vec_info reduc_info, slp_tree slp_node,
- slp_instance slp_node_instance, bool last_val_reduc_p,
- unsigned i, vec <gimple *> &vec_stmts)
-{
- tree def;
-
- if (slp_node)
- {
- if (!last_val_reduc_p)
- slp_node = slp_node_instance->reduc_phis;
- def = vect_get_slp_vect_def (slp_node, i);
- }
- else
- {
- if (!last_val_reduc_p)
- reduc_info = STMT_VINFO_REDUC_DEF (vect_orig_stmt (reduc_info));
- vec_stmts = STMT_VINFO_VEC_STMTS (reduc_info);
- def = gimple_get_lhs (vec_stmts[0]);
- }
-
- return def;
-}
-
/* Function vect_create_epilog_for_reduction
Create code at the loop-epilog to finalize the result of a reduction
@@ -5989,8 +5960,6 @@ vect_create_epilog_for_reduction (loop_vec_info
loop_vinfo,
loop-closed PHI of the inner loop which we remember as
def for the reduction PHI generation. */
bool double_reduc = false;
- bool last_val_reduc_p = LOOP_VINFO_IV_EXIT (loop_vinfo) == loop_exit
- && !LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo);
stmt_vec_info rdef_info = stmt_info;
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
{
@@ -6000,8 +5969,6 @@ vect_create_epilog_for_reduction (loop_vec_info
loop_vinfo,
(stmt_info->stmt, 0));
stmt_info = vect_stmt_to_vectorize (stmt_info);
}
- gphi *reduc_def_stmt
- = as_a <gphi *> (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info))->stmt);
code_helper code = STMT_VINFO_REDUC_CODE (reduc_info);
internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
tree vectype;
@@ -6066,33 +6033,9 @@ vect_create_epilog_for_reduction (loop_vec_info
loop_vinfo,
stmt_vec_info single_live_out_stmt[] = { stmt_info };
array_slice<const stmt_vec_info> live_out_stmts = single_live_out_stmt;
- if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
- && loop_exit != LOOP_VINFO_IV_EXIT (loop_vinfo)
- /* ??? We should fend this off earlier. For conversions we create
- multiple epilogues, one dead. */
- && stmt_info == reduc_info->reduc_def)
- {
- gcc_assert (!slp_node);
- single_live_out_stmt[0] = reduc_info;
- }
- else
- {
- if (slp_reduc)
- /* All statements produce live-out values. */
- live_out_stmts = SLP_TREE_SCALAR_STMTS (slp_node);
- else if (slp_node)
- {
- /* The last statement in the reduction chain produces the live-out
- value. Note SLP optimization can shuffle scalar stmts to
- optimize permutations so we have to search for the last stmt. */
- for (k = 0; k < group_size; ++k)
- if (!REDUC_GROUP_NEXT_ELEMENT (SLP_TREE_SCALAR_STMTS (slp_node)[k]))
- {
- single_live_out_stmt[0] = SLP_TREE_SCALAR_STMTS (slp_node)[k];
- break;
- }
- }
- }
+ if (slp_reduc)
+ /* All statements produce live-out values. */
+ live_out_stmts = SLP_TREE_SCALAR_STMTS (slp_node);
unsigned vec_num;
int ncopies;
@@ -6103,7 +6046,6 @@ vect_create_epilog_for_reduction (loop_vec_info
loop_vinfo,
}
else
{
- stmt_vec_info reduc_info = loop_vinfo->lookup_stmt (reduc_def_stmt);
vec_num = 1;
ncopies = STMT_VINFO_VEC_STMTS (reduc_info).length ();
}
@@ -6247,18 +6189,19 @@ vect_create_epilog_for_reduction (loop_vec_info
loop_vinfo,
exit_bb = loop_exit->dest;
exit_gsi = gsi_after_labels (exit_bb);
reduc_inputs.create (slp_node ? vec_num : ncopies);
- vec <gimple *> vec_stmts = vNULL;
for (unsigned i = 0; i < vec_num; i++)
{
gimple_seq stmts = NULL;
- def = vect_get_vect_def (rdef_info, slp_node, slp_node_instance,
- last_val_reduc_p, i, vec_stmts);
+ if (slp_node)
+ def = vect_get_slp_vect_def (slp_node, i);
+ else
+ def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[0]);
for (j = 0; j < ncopies; j++)
{
tree new_def = copy_ssa_name (def);
phi = create_phi_node (new_def, exit_bb);
if (j)
- def = gimple_get_lhs (vec_stmts[j]);
+ def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[j]);
if (LOOP_VINFO_IV_EXIT (loop_vinfo) == loop_exit)
SET_PHI_ARG_DEF (phi, loop_exit->dest_idx, def);
else
@@ -6963,7 +6906,8 @@ vect_create_epilog_for_reduction (loop_vec_info
loop_vinfo,
{
if (!flow_bb_inside_loop_p (loop, gimple_bb (USE_STMT (use_p))))
{
- if (!is_gimple_debug (USE_STMT (use_p)))
+ if (!is_gimple_debug (USE_STMT (use_p))
+ && gimple_bb (USE_STMT (use_p)) == loop_exit->dest)
phis.safe_push (USE_STMT (use_p));
}
else
@@ -10765,26 +10709,21 @@ vectorizable_live_operation (vec_info *vinfo,
stmt_vec_info stmt_info,
{
if (!vec_stmt_p)
return true;
- if (slp_node)
- {
- /* For reduction chains the meta-info is attached to
- the group leader. */
- if (REDUC_GROUP_FIRST_ELEMENT (stmt_info))
- stmt_info = REDUC_GROUP_FIRST_ELEMENT (stmt_info);
- /* For SLP reductions we vectorize the epilogue for
- all involved stmts together. */
- else if (slp_index != 0)
- return true;
- }
+ /* For SLP reductions we vectorize the epilogue for all involved stmts
+ together. */
+ if (slp_node && !REDUC_GROUP_FIRST_ELEMENT (stmt_info) && slp_index != 0)
+ return true;
stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
gcc_assert (reduc_info->is_reduc_info);
if (STMT_VINFO_REDUC_TYPE (reduc_info) == FOLD_LEFT_REDUCTION
|| STMT_VINFO_REDUC_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION)
return true;
- vect_create_epilog_for_reduction (loop_vinfo, stmt_info, slp_node,
- slp_node_instance,
- LOOP_VINFO_IV_EXIT (loop_vinfo));
+ if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
+ || !LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
+ vect_create_epilog_for_reduction (loop_vinfo, stmt_info, slp_node,
+ slp_node_instance,
+ LOOP_VINFO_IV_EXIT (loop_vinfo));
/* If early break we only have to materialize the reduction on the merge
block, but we have to find an alternate exit first. */
@@ -10793,11 +10732,15 @@ vectorizable_live_operation (vec_info *vinfo,
stmt_vec_info stmt_info,
for (auto exit : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
if (exit != LOOP_VINFO_IV_EXIT (loop_vinfo))
{
- vect_create_epilog_for_reduction (loop_vinfo, stmt_info,
+ vect_create_epilog_for_reduction (loop_vinfo, reduc_info,
slp_node, slp_node_instance,
exit);
break;
}
+ if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
+ vect_create_epilog_for_reduction (loop_vinfo, reduc_info, slp_node,
+ slp_node_instance,
+ LOOP_VINFO_IV_EXIT (loop_vinfo));
}
return true;
