This patch is the first part of a series to fix to PR85694.
Later patches can make the pattern for a statement S2 reuse the
results of a PATTERN_DEF_SEQ statement attached to an earlier
statement S1. Although vect_mark_stmts_to_be_vectorized handled
this fine, vect_analyze_stmt and vect_transform_loop both skipped the
PATTERN_DEF_SEQ for S1 if S1's main pattern wasn't live or relevant.
I couldn't wrap my head around the flow in vect_transform_loop,
so ended up moving the per-statement handling into a subroutine.
That makes the patch look bigger than it actually is.
Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install?
Richard
2018-06-18 Richard Sandiford <richard.sandif...@arm.com>
gcc/
* tree-vect-stmts.c (vect_analyze_stmt): Move the handling of pattern
definition statements before the early exit for statements that aren't
live or relevant.
* tree-vect-loop.c (vect_transform_loop_stmt): New function,
split out from...
(vect_transform_loop): ...here. Process pattern definition
statements without first checking whether the main pattern
statement is live or relevant.
Index: gcc/tree-vect-stmts.c
===================================================================
*** gcc/tree-vect-stmts.c 2018-06-18 15:50:55.607303415 +0100
--- gcc/tree-vect-stmts.c 2018-06-18 15:50:55.603303451 +0100
*************** vect_analyze_stmt (gimple *stmt, bool *n
*** 9409,9414 ****
--- 9409,9442 ----
return false;
}
+ if (STMT_VINFO_IN_PATTERN_P (stmt_info)
+ && node == NULL
+ && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
+ {
+ gimple_stmt_iterator si;
+
+ for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
+ {
+ gimple *pattern_def_stmt = gsi_stmt (si);
+ if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
+ || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt)))
+ {
+ /* Analyze def stmt of STMT if it's a pattern stmt. */
+ if (dump_enabled_p ())
+ {
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "==> examining pattern def statement: ");
+ dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0);
+ }
+
+ if (!vect_analyze_stmt (pattern_def_stmt,
+ need_to_vectorize, node, node_instance,
+ cost_vec))
+ return false;
+ }
+ }
+ }
+
/* Skip stmts that do not need to be vectorized. In loops this is expected
to include:
- the COND_EXPR which is the loop exit condition
*************** vect_analyze_stmt (gimple *stmt, bool *n
*** 9469,9502 ****
return false;
}
- if (is_pattern_stmt_p (stmt_info)
- && node == NULL
- && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
- {
- gimple_stmt_iterator si;
-
- for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
- {
- gimple *pattern_def_stmt = gsi_stmt (si);
- if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
- || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt)))
- {
- /* Analyze def stmt of STMT if it's a pattern stmt. */
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "==> examining pattern def statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0);
- }
-
- if (!vect_analyze_stmt (pattern_def_stmt,
- need_to_vectorize, node, node_instance,
- cost_vec))
- return false;
- }
- }
- }
-
switch (STMT_VINFO_DEF_TYPE (stmt_info))
{
case vect_internal_def:
--- 9497,9502 ----
Index: gcc/tree-vect-loop.c
===================================================================
*** gcc/tree-vect-loop.c 2018-06-18 15:50:55.607303415 +0100
--- gcc/tree-vect-loop.c 2018-06-18 15:50:55.603303451 +0100
*************** scale_profile_for_vect_loop (struct loop
*** 8306,8311 ****
--- 8306,8381 ----
scale_bbs_frequencies (&loop->latch, 1, exit_l->probability / prob);
}
+ /* Vectorize STMT if relevant, inserting any new instructions before GSI.
+ When vectorizing STMT as a store, set *SEEN_STORE to its stmt_vec_info.
+ *SLP_SCHEDULE is a running record of whether we have called
+ vect_schedule_slp. */
+
+ static void
+ vect_transform_loop_stmt (loop_vec_info loop_vinfo, gimple *stmt,
+ gimple_stmt_iterator *gsi,
+ stmt_vec_info *seen_store, bool *slp_scheduled)
+ {
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ if (!stmt_info)
+ return;
+
+ if (dump_enabled_p ())
+ {
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "------>vectorizing statement: ");
+ dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
+ }
+
+ if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
+ vect_loop_kill_debug_uses (loop, stmt);
+
+ if (!STMT_VINFO_RELEVANT_P (stmt_info)
+ && !STMT_VINFO_LIVE_P (stmt_info))
+ return;
+
+ if (STMT_VINFO_VECTYPE (stmt_info))
+ {
+ poly_uint64 nunits
+ = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
+ if (!STMT_SLP_TYPE (stmt_info)
+ && maybe_ne (nunits, vf)
+ && dump_enabled_p ())
+ /* For SLP VF is set according to unrolling factor, and not
+ to vector size, hence for SLP this print is not valid. */
+ dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n");
+ }
+
+ /* SLP. Schedule all the SLP instances when the first SLP stmt is
+ reached. */
+ if (STMT_SLP_TYPE (stmt_info))
+ {
+ if (!*slp_scheduled)
+ {
+ *slp_scheduled = true;
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "=== scheduling SLP instances ===\n");
+
+ vect_schedule_slp (loop_vinfo);
+ }
+
+ /* Hybrid SLP stmts must be vectorized in addition to SLP. */
+ if (PURE_SLP_STMT (stmt_info))
+ return;
+ }
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n");
+
+ bool grouped_store = false;
+ if (vect_transform_stmt (stmt, gsi, &grouped_store, NULL, NULL))
+ *seen_store = stmt_info;
+ }
+
/* Function vect_transform_loop.
The analysis phase has determined that the loop is vectorizable.
*************** vect_transform_loop (loop_vec_info loop_
*** 8326,8337 ****
tree niters_vector_mult_vf = NULL_TREE;
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
unsigned int lowest_vf = constant_lower_bound (vf);
- bool grouped_store;
bool slp_scheduled = false;
! gimple *stmt, *pattern_stmt;
! gimple_seq pattern_def_seq = NULL;
! gimple_stmt_iterator pattern_def_si = gsi_none ();
! bool transform_pattern_stmt = false;
bool check_profitability = false;
unsigned int th;
--- 8396,8403 ----
tree niters_vector_mult_vf = NULL_TREE;
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
unsigned int lowest_vf = constant_lower_bound (vf);
bool slp_scheduled = false;
! gimple *stmt;
bool check_profitability = false;
unsigned int th;
*************** vect_transform_loop (loop_vec_info loop_
*** 8483,8678 ****
}
}
- pattern_stmt = NULL;
for (gimple_stmt_iterator si = gsi_start_bb (bb);
! !gsi_end_p (si) || transform_pattern_stmt;)
{
! bool is_store;
!
! if (transform_pattern_stmt)
! stmt = pattern_stmt;
! else
! {
! stmt = gsi_stmt (si);
! /* During vectorization remove existing clobber stmts. */
! if (gimple_clobber_p (stmt))
! {
! unlink_stmt_vdef (stmt);
! gsi_remove (&si, true);
! release_defs (stmt);
! continue;
! }
! }
!
! if (dump_enabled_p ())
{
! dump_printf_loc (MSG_NOTE, vect_location,
! "------>vectorizing statement: ");
! dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
}
!
! stmt_info = vinfo_for_stmt (stmt);
!
! /* vector stmts created in the outer-loop during vectorization of
! stmts in an inner-loop may not have a stmt_info, and do not
! need to be vectorized. */
! if (!stmt_info)
{
! gsi_next (&si);
! continue;
! }
! if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
! vect_loop_kill_debug_uses (loop, stmt);
!
! if (!STMT_VINFO_RELEVANT_P (stmt_info)
! && !STMT_VINFO_LIVE_P (stmt_info))
! {
! if (STMT_VINFO_IN_PATTERN_P (stmt_info)
! && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info))
! && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
! || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
! {
! stmt = pattern_stmt;
! stmt_info = vinfo_for_stmt (stmt);
! }
! else
! {
! gsi_next (&si);
! continue;
! }
! }
! else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
! && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info))
! && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
! || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
! transform_pattern_stmt = true;
!
! /* If pattern statement has def stmts, vectorize them too. */
! if (is_pattern_stmt_p (stmt_info))
! {
! if (pattern_def_seq == NULL)
! {
! pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
! pattern_def_si = gsi_start (pattern_def_seq);
! }
! else if (!gsi_end_p (pattern_def_si))
! gsi_next (&pattern_def_si);
! if (pattern_def_seq != NULL)
{
! gimple *pattern_def_stmt = NULL;
! stmt_vec_info pattern_def_stmt_info = NULL;
!
! while (!gsi_end_p (pattern_def_si))
! {
! pattern_def_stmt = gsi_stmt (pattern_def_si);
! pattern_def_stmt_info
! = vinfo_for_stmt (pattern_def_stmt);
! if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
! || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
! break;
! gsi_next (&pattern_def_si);
! }
!
! if (!gsi_end_p (pattern_def_si))
{
! if (dump_enabled_p ())
! {
! dump_printf_loc (MSG_NOTE, vect_location,
! "==> vectorizing pattern def "
! "stmt: ");
! dump_gimple_stmt (MSG_NOTE, TDF_SLIM,
! pattern_def_stmt, 0);
! }
!
! stmt = pattern_def_stmt;
! stmt_info = pattern_def_stmt_info;
}
! else
! {
! pattern_def_si = gsi_none ();
! transform_pattern_stmt = false;
! }
! }
! else
! transform_pattern_stmt = false;
! }
!
! if (STMT_VINFO_VECTYPE (stmt_info))
! {
! poly_uint64 nunits
! = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
! if (!STMT_SLP_TYPE (stmt_info)
! && maybe_ne (nunits, vf)
! && dump_enabled_p ())
! /* For SLP VF is set according to unrolling factor, and not
! to vector size, hence for SLP this print is not valid. */
! dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n");
! }
!
! /* SLP. Schedule all the SLP instances when the first SLP stmt is
! reached. */
! if (STMT_SLP_TYPE (stmt_info))
! {
! if (!slp_scheduled)
! {
! slp_scheduled = true;
!
! if (dump_enabled_p ())
! dump_printf_loc (MSG_NOTE, vect_location,
! "=== scheduling SLP instances ===\n");
!
! vect_schedule_slp (loop_vinfo);
}
!
! /* Hybrid SLP stmts must be vectorized in addition to SLP. */
! if (!vinfo_for_stmt (stmt) || PURE_SLP_STMT (stmt_info))
{
! if (!transform_pattern_stmt && gsi_end_p (pattern_def_si))
{
! pattern_def_seq = NULL;
gsi_next (&si);
}
- continue;
- }
- }
-
- /* -------- vectorize statement ------------ */
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n");
-
- grouped_store = false;
- is_store = vect_transform_stmt (stmt, &si, &grouped_store, NULL,
NULL);
- if (is_store)
- {
- if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
- {
- /* Interleaving. If IS_STORE is TRUE, the vectorization of the
- interleaving chain was completed - free all the stores in
- the chain. */
- gsi_next (&si);
- vect_remove_stores (DR_GROUP_FIRST_ELEMENT (stmt_info));
}
else
! {
! /* Free the attached stmt_vec_info and remove the stmt. */
! gimple *store = gsi_stmt (si);
! free_stmt_vec_info (store);
! unlink_stmt_vdef (store);
! gsi_remove (&si, true);
! release_defs (store);
! }
!
! /* Stores can only appear at the end of pattern statements. */
! gcc_assert (!transform_pattern_stmt);
! pattern_def_seq = NULL;
}
! else if (!transform_pattern_stmt && gsi_end_p (pattern_def_si))
! {
! pattern_def_seq = NULL;
! gsi_next (&si);
! }
! } /* stmts in BB */
/* Stub out scalar statements that must not survive vectorization.
Doing this here helps with grouped statements, or statements that
--- 8549,8615 ----
}
}
for (gimple_stmt_iterator si = gsi_start_bb (bb);
! !gsi_end_p (si);)
{
! stmt = gsi_stmt (si);
! /* During vectorization remove existing clobber stmts. */
! if (gimple_clobber_p (stmt))
{
! unlink_stmt_vdef (stmt);
! gsi_remove (&si, true);
! release_defs (stmt);
}
! else
{
! stmt_info = vinfo_for_stmt (stmt);
! /* vector stmts created in the outer-loop during vectorization of
! stmts in an inner-loop may not have a stmt_info, and do not
! need to be vectorized. */
! stmt_vec_info seen_store = NULL;
! if (stmt_info)
{
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
{
! gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
! for (gimple_stmt_iterator subsi = gsi_start (def_seq);
! !gsi_end_p (subsi); gsi_next (&subsi))
! vect_transform_loop_stmt (loop_vinfo,
! gsi_stmt (subsi), &si,
! &seen_store,
! &slp_scheduled);
! gimple *pat_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
! vect_transform_loop_stmt (loop_vinfo, pat_stmt, &si,
! &seen_store, &slp_scheduled);
}
! vect_transform_loop_stmt (loop_vinfo, stmt, &si,
! &seen_store, &slp_scheduled);
}
! if (seen_store)
{
! if (STMT_VINFO_GROUPED_ACCESS (seen_store))
{
! /* Interleaving. If IS_STORE is TRUE, the
! vectorization of the interleaving chain was
! completed - free all the stores in the chain. */
gsi_next (&si);
+ vect_remove_stores (DR_GROUP_FIRST_ELEMENT (seen_store));
+ }
+ else
+ {
+ /* Free the attached stmt_vec_info and remove the
+ stmt. */
+ free_stmt_vec_info (stmt);
+ unlink_stmt_vdef (stmt);
+ gsi_remove (&si, true);
+ release_defs (stmt);
}
}
else
! gsi_next (&si);
}
! }
/* Stub out scalar statements that must not survive vectorization.
Doing this here helps with grouped statements, or statements that
