[PATCH] Use STMT_VINFO_REDUC_IDX instead of recomputing it
This is a cleanup. The cond-reduction restriction can go,
the fold-left one stays (it cannot handle more than one stmt in
the cycle - in the future when we get partial loop vectorization
generic code would handle duplicating of scalar code parts, they'd
simply stay single-lane SLP graph parts).
Bootstrap and regtest running on x86_64-unknown-linux-gnu.
Richard.
2019-10-25 Richard Biener
* tree-vect-loop.c (vect_create_epilog_for_reduction): Use
STMT_VINFO_REDUC_IDX from the actual stmt.
(vect_transform_reduction): Likewise.
(vectorizable_reduction): Compute the reduction chain length,
do not recompute the reduction operand index. Remove no longer
necessary restriction for condition reduction chains.
Index: gcc/tree-vect-loop.c
===
--- gcc/tree-vect-loop.c(revision 277441)
+++ gcc/tree-vect-loop.c(working copy)
@@ -4263,9 +4263,9 @@ vect_create_epilog_for_reduction (stmt_v
(CCOMPARE). The then and else values mirror the main VEC_COND_EXPR:
the reduction phi corresponds to NEW_PHI_TREE and the new values
correspond to INDEX_BEFORE_INCR. */
- gcc_assert (STMT_VINFO_REDUC_IDX (reduc_info) >= 1);
+ gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) >= 1);
tree index_cond_expr;
- if (STMT_VINFO_REDUC_IDX (reduc_info) == 2)
+ if (STMT_VINFO_REDUC_IDX (stmt_info) == 2)
index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type,
ccompare, indx_before_incr, new_phi_tree);
else
@@ -5720,19 +5720,21 @@ vectorizable_reduction (stmt_vec_info st
gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
gphi *reduc_def_phi = as_a (phi_info->stmt);
- /* Verify following REDUC_IDX from the latch def leads us back to the PHI.
*/
+ /* Verify following REDUC_IDX from the latch def leads us back to the PHI
+ and compute the reduction chain length. */
tree reduc_def = PHI_ARG_DEF_FROM_EDGE (reduc_def_phi,
loop_latch_edge (loop));
+ unsigned reduc_chain_length = 0;
while (reduc_def != PHI_RESULT (reduc_def_phi))
{
stmt_vec_info def = loop_vinfo->lookup_def (reduc_def);
def = vect_stmt_to_vectorize (def);
gcc_assert (STMT_VINFO_REDUC_IDX (def) != -1);
reduc_def = gimple_op (def->stmt, 1 + STMT_VINFO_REDUC_IDX (def));
+ reduc_chain_length++;
}
reduc_def = PHI_RESULT (reduc_def_phi);
- int reduc_index = -1;
for (i = 0; i < op_type; i++)
{
tree op = gimple_op (stmt, i + 1);
@@ -5753,7 +5755,6 @@ vectorizable_reduction (stmt_vec_info st
if ((dt == vect_reduction_def || dt == vect_nested_cycle)
&& op == reduc_def)
{
- reduc_index = i;
continue;
}
@@ -5792,10 +5793,6 @@ vectorizable_reduction (stmt_vec_info st
if (!vectype_in)
vectype_in = vectype_out;
STMT_VINFO_REDUC_VECTYPE_IN (reduc_info) = vectype_in;
- /* For the SSA cycle we store on each participating stmt the operand index
- where the cycle continues. Store the one relevant for the actual
- operation in the reduction meta. */
- STMT_VINFO_REDUC_IDX (reduc_info) = reduc_index;
enum vect_reduction_type v_reduc_type = STMT_VINFO_REDUC_TYPE (phi_info);
STMT_VINFO_REDUC_TYPE (reduc_info) = v_reduc_type;
@@ -5805,28 +5802,8 @@ vectorizable_reduction (stmt_vec_info st
if (slp_node)
return false;
- /* TODO: We can't yet handle reduction chains, since we need to treat
-each COND_EXPR in the chain specially, not just the last one.
-E.g. for:
-
- x_1 = PHI
- x_2 = a_2 ? ... : x_1;
- x_3 = a_3 ? ... : x_2;
-
-we're interested in the last element in x_3 for which a_2 || a_3
-is true, whereas the current reduction chain handling would
-vectorize x_2 as a normal VEC_COND_EXPR and only treat x_3
-as a reduction operation. */
- if (reduc_index == -1)
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-"conditional reduction chains not supported\n");
- return false;
- }
-
/* When the condition uses the reduction value in the condition, fail.
*/
- if (reduc_index == 0)
+ if (STMT_VINFO_REDUC_IDX (stmt_info) == 0)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -5995,17 +5972,17 @@ vectorizable_reduction (stmt_vec_info st
outer-loop vectorization is safe. */
if (needs_fold_left_reduction_p (scalar_type, orig_code))
{
- STMT_VINFO_REDUC_TYPE (reduc_info)
- = reduction_type = FOLD_LEFT_REDUCTION;
- /* When vectorizing a reduction chain w/o SLP the reduction PHI is not
-directy used in stmt. */
- if
Re: [PATCH] Use STMT_VINFO_REDUC_IDX instead of recomputing it
On Fri, 25 Oct 2019, Richard Biener wrote:
>
> This is a cleanup. The cond-reduction restriction can go,
> the fold-left one stays (it cannot handle more than one stmt in
> the cycle - in the future when we get partial loop vectorization
> generic code would handle duplicating of scalar code parts, they'd
> simply stay single-lane SLP graph parts).
>
> Bootstrap and regtest running on x86_64-unknown-linux-gnu.
The following variant is what I have applied.
Bootstrapped and tested on x86_64-unknown-linux-gnu.
Richard.
2019-10-28 Richard Biener
* tree-vect-loop.c (vect_create_epilog_for_reduction): Use
STMT_VINFO_REDUC_IDX from the actual stmt.
(vect_transform_reduction): Likewise.
(vectorizable_reduction): Compute the reduction chain length,
do not recompute the reduction operand index. Remove no longer
necessary restriction for condition reduction chains.
Index: gcc/tree-vect-loop.c
===
--- gcc/tree-vect-loop.c(revision 277504)
+++ gcc/tree-vect-loop.c(working copy)
@@ -4263,9 +4263,9 @@ vect_create_epilog_for_reduction (stmt_v
(CCOMPARE). The then and else values mirror the main VEC_COND_EXPR:
the reduction phi corresponds to NEW_PHI_TREE and the new values
correspond to INDEX_BEFORE_INCR. */
- gcc_assert (STMT_VINFO_REDUC_IDX (reduc_info) >= 1);
+ gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) >= 1);
tree index_cond_expr;
- if (STMT_VINFO_REDUC_IDX (reduc_info) == 2)
+ if (STMT_VINFO_REDUC_IDX (stmt_info) == 2)
index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type,
ccompare, indx_before_incr, new_phi_tree);
else
@@ -5720,19 +5720,24 @@ vectorizable_reduction (stmt_vec_info st
gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
gphi *reduc_def_phi = as_a (phi_info->stmt);
- /* Verify following REDUC_IDX from the latch def leads us back to the PHI.
*/
+ /* Verify following REDUC_IDX from the latch def leads us back to the PHI
+ and compute the reduction chain length. */
tree reduc_def = PHI_ARG_DEF_FROM_EDGE (reduc_def_phi,
loop_latch_edge (loop));
+ unsigned reduc_chain_length = 0;
+ bool only_slp_reduc_chain = true;
while (reduc_def != PHI_RESULT (reduc_def_phi))
{
stmt_vec_info def = loop_vinfo->lookup_def (reduc_def);
def = vect_stmt_to_vectorize (def);
gcc_assert (STMT_VINFO_REDUC_IDX (def) != -1);
+ if (!REDUC_GROUP_FIRST_ELEMENT (def))
+ only_slp_reduc_chain = false;
reduc_def = gimple_op (def->stmt, 1 + STMT_VINFO_REDUC_IDX (def));
+ reduc_chain_length++;
}
reduc_def = PHI_RESULT (reduc_def_phi);
- int reduc_index = -1;
for (i = 0; i < op_type; i++)
{
tree op = gimple_op (stmt, i + 1);
@@ -5753,7 +5758,6 @@ vectorizable_reduction (stmt_vec_info st
if ((dt == vect_reduction_def || dt == vect_nested_cycle)
&& op == reduc_def)
{
- reduc_index = i;
continue;
}
@@ -5792,10 +5796,6 @@ vectorizable_reduction (stmt_vec_info st
if (!vectype_in)
vectype_in = vectype_out;
STMT_VINFO_REDUC_VECTYPE_IN (reduc_info) = vectype_in;
- /* For the SSA cycle we store on each participating stmt the operand index
- where the cycle continues. Store the one relevant for the actual
- operation in the reduction meta. */
- STMT_VINFO_REDUC_IDX (reduc_info) = reduc_index;
enum vect_reduction_type v_reduc_type = STMT_VINFO_REDUC_TYPE (phi_info);
STMT_VINFO_REDUC_TYPE (reduc_info) = v_reduc_type;
@@ -5805,28 +5805,8 @@ vectorizable_reduction (stmt_vec_info st
if (slp_node)
return false;
- /* TODO: We can't yet handle reduction chains, since we need to treat
-each COND_EXPR in the chain specially, not just the last one.
-E.g. for:
-
- x_1 = PHI
- x_2 = a_2 ? ... : x_1;
- x_3 = a_3 ? ... : x_2;
-
-we're interested in the last element in x_3 for which a_2 || a_3
-is true, whereas the current reduction chain handling would
-vectorize x_2 as a normal VEC_COND_EXPR and only treat x_3
-as a reduction operation. */
- if (reduc_index == -1)
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-"conditional reduction chains not supported\n");
- return false;
- }
-
/* When the condition uses the reduction value in the condition, fail.
*/
- if (reduc_index == 0)
+ if (STMT_VINFO_REDUC_IDX (stmt_info) == 0)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -5995,17 +5975,18 @@ vectorizable_reduction (stmt_vec_info st
outer-loop vectorization is safe. */
if (needs_fold_le
