The point of this patch is to put pattern statements in the same
vec_basic_block as the statements they replace, with the pattern
statements for S coming between S and S's original predecessor.
This removes the need to handle them specially in various places.
2018-08-28 Richard Sandiford <richard.sandif...@arm.com>
gcc/
* tree-vectorizer.h (vec_basic_block): Expand comment.
(_stmt_vec_info::pattern_def_seq): Delete.
(STMT_VINFO_PATTERN_DEF_SEQ): Likewise.
(is_main_pattern_stmt_p): New function.
* tree-vect-loop.c (vect_determine_vf_for_stmt_1): Rename to...
(vect_determine_vf_for_stmt): ...this, deleting the original
function with this name. Remove vectype_maybe_set_p argument
and test is_pattern_stmt_p instead. Retain the "examining..."
message from the previous vect_determine_vf_for_stmt.
(vect_compute_single_scalar_iteration_cost, vect_update_vf_for_slp)
(vect_analyze_loop_2): Don't treat pattern statements specially.
(vect_transform_loop): Likewise. Use vect_orig_stmt to find the
insertion point.
* tree-vect-slp.c (vect_detect_hybrid_slp): Expect pattern statements
to be in the statement list, without needing to follow
STMT_VINFO_RELATED_STMT. Remove PATTERN_DEF_SEQ handling.
* tree-vect-stmts.c (vect_analyze_stmt): Don't handle pattern
statements specially.
(vect_remove_dead_scalar_stmts): Ignore pattern statements.
* tree-vect-patterns.c (vect_set_pattern_stmt): Insert the pattern
statement into the vec_basic_block immediately before the statement
it replaces.
(append_pattern_def_seq): Likewise. If the original statement is
itself a pattern statement, associate the new one with the original
statement.
(vect_split_statement): Use append_pattern_def_seq to insert the
first pattern statement.
(vect_recog_vector_vector_shift_pattern): Remove mention of
STMT_VINFO_PATTERN_DEF_SEQ.
(adjust_bool_stmts): Get the last pattern statement from the
stmt_vec_info chain.
(vect_mark_pattern_stmts): Rename to...
(vect_replace_stmt_with_pattern): ...this. Remove the
PATTERN_DEF_SEQ handling and process only the pattern statement given.
Use append_pattern_def_seq when replacing a pattern statement with
another pattern statement, and use vec_basic_block::remove instead
of gsi_remove to remove the old one.
(vect_pattern_recog_1): Update accordingly. Remove PATTERN_DEF_SEQ
handling. On failure, remove any half-formed pattern sequence from
the vec_basic_block. Install the vector type in pattern statements
that don't yet have one.
(vect_pattern_recog): Iterate over statements that are added
by previous recognizers, but skipping those that have already
been replaced, or the main pattern statement in such a replacement.
Index: gcc/tree-vectorizer.h
===================================================================
--- gcc/tree-vectorizer.h 2018-08-28 12:05:14.014961439 +0100
+++ gcc/tree-vectorizer.h 2018-08-28 12:05:16.522940287 +0100
@@ -172,7 +172,13 @@ #define SLP_TREE_TWO_OPERATORS(S) (S)-
#define SLP_TREE_DEF_TYPE(S) (S)->def_type
/* Information about the phis and statements in a block that we're trying
- to vectorize, in their original order. */
+ to vectorize. This includes the phis and statements that were in the
+ original scalar code, in their original order. It also includes any
+ pattern statements that the vectorizer has created to replace some
+ of the scalar ones. Such pattern statements come immediately before
+ the statement that they replace; that is, all pattern statements P for
+ which vect_orig_stmt (P) == S form a sequence that comes immediately
+ before S. */
class vec_basic_block
{
public:
@@ -870,11 +876,6 @@ struct _stmt_vec_info {
pattern). */
stmt_vec_info related_stmt;
- /* Used to keep a sequence of def stmts of a pattern stmt if such exists.
- The sequence is attached to the original statement rather than the
- pattern statement. */
- gimple_seq pattern_def_seq;
-
/* List of datarefs that are known to have the same alignment as the dataref
of this stmt. */
vec<dr_p> same_align_refs;
@@ -1048,7 +1049,6 @@ #define STMT_VINFO_DR_INFO(S) \
#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
-#define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
#define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs
#define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info
#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
@@ -1176,6 +1176,17 @@ is_pattern_stmt_p (stmt_vec_info stmt_in
return stmt_info->pattern_stmt_p;
}
+/* Return TRUE if a statement represented by STMT_INFO is the final
+ statement in a pattern. */
+
+static inline bool
+is_main_pattern_stmt_p (stmt_vec_info stmt_info)
+{
+ stmt_vec_info orig_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
+ return (is_pattern_stmt_p (stmt_info)
+ && STMT_VINFO_RELATED_STMT (orig_stmt_info) == stmt_info);
+}
+
/* If STMT_INFO is a pattern statement, return the statement that it
replaces, otherwise return STMT_INFO itself. */
Index: gcc/tree-vect-loop.c
===================================================================
--- gcc/tree-vect-loop.c 2018-08-28 12:05:14.010961472 +0100
+++ gcc/tree-vect-loop.c 2018-08-28 12:05:16.518940320 +0100
@@ -155,18 +155,24 @@ Software Foundation; either version 3, o
static void vect_estimate_min_profitable_iters (loop_vec_info, int *, int *);
-/* Subroutine of vect_determine_vf_for_stmt that handles only one
- statement. VECTYPE_MAYBE_SET_P is true if STMT_VINFO_VECTYPE
- may already be set for general statements (not just data refs). */
+/* Subroutine of vect_determine_vectorization_factor. Set the vector
+ type of STMT_INFO and update the vectorization factor VF accordingly.
+ If the statement produces a mask result whose vector type can only be
+ calculated later, add it to MASK_PRODUCERS. Return true on success
+ or false if something prevented vectorization. */
static bool
-vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info,
- bool vectype_maybe_set_p,
- poly_uint64 *vf,
- vec<stmt_vec_info > *mask_producers)
+vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf,
+ vec<stmt_vec_info > *mask_producers)
{
gimple *stmt = stmt_info->stmt;
+ if (dump_enabled_p ())
+ {
+ dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: ");
+ dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
+ }
+
if ((!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_LIVE_P (stmt_info))
|| gimple_clobber_p (stmt))
@@ -188,7 +194,7 @@ vect_determine_vf_for_stmt_1 (stmt_vec_i
that contain a data ref, or for "pattern-stmts" (stmts generated
by the vectorizer to represent/replace a certain idiom). */
gcc_assert ((STMT_VINFO_DATA_REF (stmt_info)
- || vectype_maybe_set_p)
+ || is_pattern_stmt_p (stmt_info))
&& STMT_VINFO_VECTYPE (stmt_info) == stmt_vectype);
else if (stmt_vectype == boolean_type_node)
mask_producers->safe_push (stmt_info);
@@ -202,62 +208,6 @@ vect_determine_vf_for_stmt_1 (stmt_vec_i
return true;
}
-/* Subroutine of vect_determine_vectorization_factor. Set the vector
- types of STMT_INFO and all attached pattern statements and update
- the vectorization factor VF accordingly. If some of the statements
- produce a mask result whose vector type can only be calculated later,
- add them to MASK_PRODUCERS. Return true on success or false if
- something prevented vectorization. */
-
-static bool
-vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf,
- vec<stmt_vec_info > *mask_producers)
-{
- vec_info *vinfo = stmt_info->vinfo;
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
- }
- if (!vect_determine_vf_for_stmt_1 (stmt_info, false, vf, mask_producers))
- return false;
-
- if (STMT_VINFO_IN_PATTERN_P (stmt_info)
- && STMT_VINFO_RELATED_STMT (stmt_info))
- {
- gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
- stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
-
- /* If a pattern statement has def stmts, analyze them too. */
- for (gimple_stmt_iterator si = gsi_start (pattern_def_seq);
- !gsi_end_p (si); gsi_next (&si))
- {
- stmt_vec_info def_stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "==> examining pattern def stmt: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM,
- def_stmt_info->stmt, 0);
- }
- if (!vect_determine_vf_for_stmt_1 (def_stmt_info, true,
- vf, mask_producers))
- return false;
- }
-
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "==> examining pattern statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
- }
- if (!vect_determine_vf_for_stmt_1 (stmt_info, true, vf, mask_producers))
- return false;
- }
-
- return true;
-}
-
/* Function vect_determine_vectorization_factor
Determine the vectorization factor (VF). VF is the number of data elements
@@ -1113,9 +1063,8 @@ vect_compute_single_scalar_iteration_cos
/* Skip stmts that are not vectorized inside the loop. */
if (stmt_info
&& !STMT_VINFO_RELEVANT_P (stmt_info)
- && (!STMT_VINFO_LIVE_P (stmt_info)
- || !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
- && !STMT_VINFO_IN_PATTERN_P (stmt_info))
+ && (!VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
+ || !STMT_VINFO_LIVE_P (stmt_info)))
continue;
vect_cost_for_stmt kind;
@@ -1429,14 +1378,11 @@ vect_update_vf_for_slp (loop_vec_info lo
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
- {
- stmt_vec_info final_info = vect_stmt_to_vectorize (stmt_info);
- if ((STMT_VINFO_RELEVANT_P (final_info)
- || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (final_info)))
- && !PURE_SLP_STMT (final_info))
- /* STMT needs both SLP and loop-based vectorization. */
- only_slp_in_loop = false;
- }
+ if ((STMT_VINFO_RELEVANT_P (stmt_info)
+ || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
+ && !PURE_SLP_STMT (stmt_info))
+ /* STMT needs both SLP and loop-based vectorization. */
+ only_slp_in_loop = false;
if (only_slp_in_loop)
{
@@ -2199,18 +2145,7 @@ vect_analyze_loop_2 (loop_vec_info loop_
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
- {
- STMT_SLP_TYPE (stmt_info) = loop_vect;
- if (STMT_VINFO_IN_PATTERN_P (stmt_info))
- {
- gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
- STMT_SLP_TYPE (STMT_VINFO_RELATED_STMT (stmt_info)) = loop_vect;
- for (gimple_stmt_iterator pi = gsi_start (pattern_def_seq);
- !gsi_end_p (pi); gsi_next (&pi))
- STMT_SLP_TYPE (loop_vinfo->lookup_stmt (gsi_stmt (pi)))
- = loop_vect;
- }
- }
+ STMT_SLP_TYPE (stmt_info) = loop_vect;
/* Free optimized alias test DDRS. */
LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).truncate (0);
@@ -8409,22 +8344,8 @@ vect_transform_loop (loop_vec_info loop_
loop_vinfo->remove_stmt (stmt_info);
else
{
- gimple_stmt_iterator si = gsi_for_stmt (stmt_info->stmt);
- if (STMT_VINFO_IN_PATTERN_P (stmt_info))
- {
- gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
- for (gimple_stmt_iterator subsi = gsi_start (def_seq);
- !gsi_end_p (subsi); gsi_next (&subsi))
- {
- stmt_vec_info pat_stmt_info
- = loop_vinfo->lookup_stmt (gsi_stmt (subsi));
- vect_transform_loop_stmt (loop_vinfo, pat_stmt_info,
- &si);
- }
- stmt_vec_info pat_stmt_info
- = STMT_VINFO_RELATED_STMT (stmt_info);
- vect_transform_loop_stmt (loop_vinfo, pat_stmt_info, &si);
- }
+ stmt_vec_info place = vect_orig_stmt (stmt_info);
+ gimple_stmt_iterator si = gsi_for_stmt (place->stmt);
vect_transform_loop_stmt (loop_vinfo, stmt_info, &si);
}
}
Index: gcc/tree-vect-slp.c
===================================================================
--- gcc/tree-vect-slp.c 2018-08-28 12:05:14.010961472 +0100
+++ gcc/tree-vect-slp.c 2018-08-28 12:05:16.522940287 +0100
@@ -2362,18 +2362,14 @@ vect_detect_hybrid_slp (loop_vec_info lo
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
- if (STMT_VINFO_IN_PATTERN_P (stmt_info))
+ if (is_pattern_stmt_p (stmt_info))
{
walk_stmt_info wi;
memset (&wi, 0, sizeof (wi));
wi.info = loop_vinfo;
- gimple_stmt_iterator gsi2
- = gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)->stmt);
- walk_gimple_stmt (&gsi2, vect_detect_hybrid_slp_2,
+ gimple_stmt_iterator gsi = gsi_for_stmt (stmt_info->stmt);
+ walk_gimple_stmt (&gsi, vect_detect_hybrid_slp_2,
vect_detect_hybrid_slp_1, &wi);
- walk_gimple_seq (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
- vect_detect_hybrid_slp_2,
- vect_detect_hybrid_slp_1, &wi);
}
/* Then walk the SLP instance trees marking stmts with uses in
Index: gcc/tree-vect-stmts.c
===================================================================
--- gcc/tree-vect-stmts.c 2018-08-28 12:05:14.014961439 +0100
+++ gcc/tree-vect-stmts.c 2018-08-28 12:05:16.522940287 +0100
@@ -9391,11 +9391,9 @@ vect_analyze_stmt (stmt_vec_info stmt_in
slp_tree node, slp_instance node_instance,
stmt_vector_for_cost *cost_vec)
{
- vec_info *vinfo = stmt_info->vinfo;
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
bool ok;
- gimple_seq pattern_def_seq;
if (dump_enabled_p ())
{
@@ -9412,94 +9410,21 @@ vect_analyze_stmt (stmt_vec_info stmt_in
return false;
}
- if (STMT_VINFO_IN_PATTERN_P (stmt_info)
- && node == NULL
- && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
- {
- gimple_stmt_iterator si;
-
- for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
- {
- stmt_vec_info pattern_def_stmt_info
- = vinfo->lookup_stmt (gsi_stmt (si));
- if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
- || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
- {
- /* Analyze def stmt of STMT if it's a pattern stmt. */
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "==> examining pattern def statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM,
- pattern_def_stmt_info->stmt, 0);
- }
-
- if (!vect_analyze_stmt (pattern_def_stmt_info,
- need_to_vectorize, node, node_instance,
- cost_vec))
- return false;
- }
- }
- }
-
/* Skip stmts that do not need to be vectorized. In loops this is expected
to include:
- the COND_EXPR which is the loop exit condition
- any LABEL_EXPRs in the loop
- computations that are used only for array indexing or loop control.
In basic blocks we only analyze statements that are a part of some SLP
- instance, therefore, all the statements are relevant.
-
- Pattern statement needs to be analyzed instead of the original statement
- if the original statement is not relevant. Otherwise, we analyze both
- statements. In basic blocks we are called from some SLP instance
- traversal, don't analyze pattern stmts instead, the pattern stmts
- already will be part of SLP instance. */
+ instance, therefore, all the statements are relevant. */
- stmt_vec_info pattern_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
if (!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_LIVE_P (stmt_info))
{
- if (STMT_VINFO_IN_PATTERN_P (stmt_info)
- && pattern_stmt_info
- && (STMT_VINFO_RELEVANT_P (pattern_stmt_info)
- || STMT_VINFO_LIVE_P (pattern_stmt_info)))
- {
- /* Analyze PATTERN_STMT instead of the original stmt. */
- stmt_info = pattern_stmt_info;
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "==> examining pattern statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
- }
- }
- else
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
-
- return true;
- }
- }
- else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
- && node == NULL
- && pattern_stmt_info
- && (STMT_VINFO_RELEVANT_P (pattern_stmt_info)
- || STMT_VINFO_LIVE_P (pattern_stmt_info)))
- {
- /* Analyze PATTERN_STMT too. */
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "==> examining pattern statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt_info->stmt, 0);
- }
-
- if (!vect_analyze_stmt (pattern_stmt_info, need_to_vectorize, node,
- node_instance, cost_vec))
- return false;
- }
+ dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
+ return true;
+ }
switch (STMT_VINFO_DEF_TYPE (stmt_info))
{
@@ -10924,7 +10849,8 @@ vect_remove_dead_scalar_stmts (vec_info
stmt_info = prev_stmt_info)
{
prev_stmt_info = stmt_info->prev;
- vect_maybe_remove_scalar_stmt (stmt_info);
+ if (!is_pattern_stmt_p (stmt_info))
+ vect_maybe_remove_scalar_stmt (stmt_info);
}
}
}
Index: gcc/tree-vect-patterns.c
===================================================================
--- gcc/tree-vect-patterns.c 2018-08-28 12:05:11.462982962 +0100
+++ gcc/tree-vect-patterns.c 2018-08-28 12:05:16.518940320 +0100
@@ -125,27 +125,26 @@ vect_init_pattern_stmt (gimple *pattern_
vect_set_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info,
tree vectype)
{
- STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
- STMT_VINFO_RELATED_STMT (orig_stmt_info)
+ stmt_vec_info pattern_stmt_info
= vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, vectype);
+ orig_stmt_info->block->add_before (pattern_stmt_info, orig_stmt_info);
+ STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
+ STMT_VINFO_RELATED_STMT (orig_stmt_info) = pattern_stmt_info;
}
-/* Add NEW_STMT to STMT_INFO's pattern definition statements. If VECTYPE
- is nonnull, record that NEW_STMT's vector type is VECTYPE, which might
- be different from the vector type of the final pattern statement. */
+/* Add NEW_STMT to the pattern statements that replace STMT_INFO.
+ If VECTYPE is nonnull, record that NEW_STMT's vector type is VECTYPE,
+ which might be different from the vector type of the final pattern
+ statement. */
static inline void
append_pattern_def_seq (stmt_vec_info stmt_info, gimple *new_stmt,
tree vectype = NULL_TREE)
{
- vec_info *vinfo = stmt_info->vinfo;
- if (vectype)
- {
- stmt_vec_info new_stmt_info = vinfo->add_stmt (new_stmt);
- STMT_VINFO_VECTYPE (new_stmt_info) = vectype;
- }
- gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
- new_stmt);
+ stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
+ stmt_vec_info new_stmt_info
+ = vect_init_pattern_stmt (new_stmt, orig_stmt_info, vectype);
+ stmt_info->block->add_before (new_stmt_info, stmt_info);
}
/* The caller wants to perform new operations on vect_external variable
@@ -633,11 +632,6 @@ vect_split_statement (stmt_vec_info stmt
{
if (is_pattern_stmt_p (stmt2_info))
{
- /* STMT2_INFO is part of a pattern. Get the statement to which
- the pattern is attached. */
- stmt_vec_info orig_stmt2_info = STMT_VINFO_RELATED_STMT (stmt2_info);
- vect_init_pattern_stmt (stmt1, orig_stmt2_info, vectype);
-
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
@@ -645,6 +639,9 @@ vect_split_statement (stmt_vec_info stmt
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt2_info->stmt, 0);
}
+ /* Insert STMT1_INFO before STMT2_INFO. */
+ append_pattern_def_seq (stmt2_info, stmt1, vectype);
+
/* Since STMT2_INFO is a pattern statement, we can change it
in-situ without worrying about changing the code for the
containing block. */
@@ -658,18 +655,6 @@ vect_split_statement (stmt_vec_info stmt
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt2_info->stmt, 0);
}
- gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt2_info);
- if (STMT_VINFO_RELATED_STMT (orig_stmt2_info) == stmt2_info)
- /* STMT2_INFO is the actual pattern statement. Add STMT1
- to the end of the definition sequence. */
- gimple_seq_add_stmt_without_update (def_seq, stmt1);
- else
- {
- /* STMT2_INFO belongs to the definition sequence. Insert STMT1
- before it. */
- gimple_stmt_iterator gsi = gsi_for_stmt (stmt2_info->stmt, def_seq);
- gsi_insert_before_without_update (&gsi, stmt1, GSI_SAME_STMT);
- }
return true;
}
else
@@ -689,10 +674,8 @@ vect_split_statement (stmt_vec_info stmt
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt2_info->stmt, 0);
}
- /* Add STMT1 as a singleton pattern definition sequence. */
- gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt2_info);
- vect_init_pattern_stmt (stmt1, stmt2_info, vectype);
- gimple_seq_add_stmt_without_update (def_seq, stmt1);
+ /* Insert STMT1_INFO before STMT2_INFO. */
+ append_pattern_def_seq (stmt2_info, stmt1, vectype);
/* Build the second of the two pattern statements. */
tree new_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
@@ -2164,7 +2147,7 @@ vect_recog_rotate_pattern (stmt_vec_info
i.e. the shift/rotate stmt. The original stmt (S3) is replaced
with a shift/rotate which has same type on both operands, in the
second case just b_T op c_T, in the first case with added cast
- from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
+ from a_t to c_T beforehand.
Output:
@@ -3518,9 +3501,8 @@ adjust_bool_stmts (hash_set <gimple *> &
adjust_bool_pattern (gimple_assign_lhs (bool_stmts[i]),
out_type, stmt_info, defs);
- /* Pop the last pattern seq stmt and install it as pattern root for STMT. */
- gimple *pattern_stmt
- = gimple_seq_last_stmt (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
+ /* Return the result of the last statement we emitted. */
+ gimple *pattern_stmt = stmt_info->prev->stmt;
return gimple_assign_lhs (pattern_stmt);
}
@@ -4684,14 +4666,14 @@ static vect_recog_func vect_vect_recog_f
const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs);
-/* Mark statements that are involved in a pattern. */
+/* Replace ORIG_STMT_INFO with PATTERN_STMT, using PATTERN_VECTYPE as
+ the vector type for PATTERN_STMT. */
static inline void
-vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
- tree pattern_vectype)
+vect_replace_stmt_with_pattern (stmt_vec_info orig_stmt_info,
+ gimple *pattern_stmt,
+ tree pattern_vectype)
{
- gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
-
gimple *orig_pattern_stmt = NULL;
if (is_pattern_stmt_p (orig_stmt_info))
{
@@ -4718,32 +4700,14 @@ vect_mark_pattern_stmts (stmt_vec_info o
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt, 0);
}
- /* Switch to the statement that ORIG replaces. */
- orig_stmt_info = STMT_VINFO_RELATED_STMT (orig_stmt_info);
-
/* We shouldn't be replacing the main pattern statement. */
- gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info)->stmt
- != orig_pattern_stmt);
- }
+ gcc_assert (!is_main_pattern_stmt_p (orig_stmt_info));
- if (def_seq)
- for (gimple_stmt_iterator si = gsi_start (def_seq);
- !gsi_end_p (si); gsi_next (&si))
- vect_init_pattern_stmt (gsi_stmt (si), orig_stmt_info, pattern_vectype);
-
- if (orig_pattern_stmt)
- {
- vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
-
- /* Insert all the new pattern statements before the original one. */
- gimple_seq *orig_def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
- gimple_stmt_iterator gsi = gsi_for_stmt (orig_pattern_stmt,
- orig_def_seq);
- gsi_insert_seq_before_without_update (&gsi, def_seq, GSI_SAME_STMT);
- gsi_insert_before_without_update (&gsi, pattern_stmt, GSI_SAME_STMT);
+ /* Insert the new pattern statement before the original one. */
+ append_pattern_def_seq (orig_stmt_info, pattern_stmt, pattern_vectype);
/* Remove the pattern statement that this new pattern replaces. */
- gsi_remove (&gsi, false);
+ orig_stmt_info->block->remove (orig_stmt_info);
}
else
vect_set_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
@@ -4770,30 +4734,17 @@ vect_mark_pattern_stmts (stmt_vec_info o
static void
vect_pattern_recog_1 (vect_recog_func *recog_func, stmt_vec_info stmt_info)
{
- vec_info *vinfo = stmt_info->vinfo;
gimple *pattern_stmt;
loop_vec_info loop_vinfo;
tree pattern_vectype;
- /* If this statement has already been replaced with pattern statements,
- leave the original statement alone, since the first match wins.
- Instead try to match against the definition statements that feed
- the main pattern statement. */
- if (STMT_VINFO_IN_PATTERN_P (stmt_info))
- {
- gimple_stmt_iterator gsi;
- for (gsi = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
- !gsi_end_p (gsi); gsi_next (&gsi))
- vect_pattern_recog_1 (recog_func, vinfo->lookup_stmt (gsi_stmt (gsi)));
- return;
- }
-
- gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
+ stmt_vec_info prev_stmt_info = stmt_info->prev;
pattern_stmt = recog_func->fn (stmt_info, &pattern_vectype);
if (!pattern_stmt)
{
- /* Clear any half-formed pattern definition sequence. */
- STMT_VINFO_PATTERN_DEF_SEQ (stmt_info) = NULL;
+ /* Delete any half-formed pattern sequence. */
+ while (stmt_info->prev != prev_stmt_info)
+ stmt_info->block->remove (stmt_info->prev);
return;
}
@@ -4808,8 +4759,15 @@ vect_pattern_recog_1 (vect_recog_func *r
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt, 0);
}
+ /* Install the vector type in pattern definition statements that
+ don't yet have one. */
+ for (stmt_vec_info pat_stmt_info = stmt_info->prev;
+ pat_stmt_info != prev_stmt_info; pat_stmt_info = pat_stmt_info->prev)
+ if (!STMT_VINFO_VECTYPE (pat_stmt_info))
+ STMT_VINFO_VECTYPE (pat_stmt_info) = pattern_vectype;
+
/* Mark the stmts that are involved in the pattern. */
- vect_mark_pattern_stmts (stmt_info, pattern_stmt, pattern_vectype);
+ vect_replace_stmt_with_pattern (stmt_info, pattern_stmt, pattern_vectype);
/* Patterns cannot be vectorized using SLP, because they change the order of
computation. */
@@ -4911,8 +4869,27 @@ vect_pattern_recog (vec_info *vinfo)
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
- if (STMT_VINFO_VECTORIZABLE (stmt_info))
- /* Scan over all generic vect_recog_xxx_pattern functions. */
- for (unsigned int j = 0; j < NUM_PATTERNS; j++)
- vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j], stmt_info);
+ {
+ stmt_vec_info begin_prev = stmt_info->prev;
+ if (STMT_VINFO_VECTORIZABLE (stmt_info))
+ /* Scan over all generic vect_recog_xxx_pattern functions. */
+ for (unsigned int j = 0; j < NUM_PATTERNS; j++)
+ {
+ stmt_vec_info curr_prev;
+ /* Scan over STMT_INFO and any pattern definition statements
+ that were introduced by previous recognizers. */
+ for (stmt_vec_info curr_info = stmt_info;
+ curr_info != begin_prev; curr_info = curr_prev)
+ {
+ curr_prev = curr_info->prev;
+ /* The first match wins, so skip statements that have
+ already been replaced, and the final statement with
+ which they were replaced. */
+ if (!STMT_VINFO_IN_PATTERN_P (curr_info)
+ && !is_main_pattern_stmt_p (curr_info))
+ vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j],
+ curr_info);
+ }
+ }
+ }
}
