The following makes us use the SLP graph and its recorded vector
types to perform alignment analysis even for loops. This avoids
accessing STMT_VINFO_VECTYPE we want to get rid of.
v2 adjusts alignment peeling to ignore DRs that are dead because
of conditional store pattern recognition.
Bootstapped on x86_64-unknown-linux-gnu, testing in progress.
* tree-vect-data-refs.cc (vect_analyze_data_refs_alignment):
Walk over SLP instances and call vect_slp_analyze_instance_alignment.
(vect_slp_analyze_dr_alignment): Split out from ...
(vect_slp_analyze_node_alignment): ... here. Handle
non-grouped access SLP nodes.
(vect_slp_analyze_instance_alignment): Adjust for dropped
return value from the above.
(vect_relevant_for_alignment_p): Dead DRs are not relevant.
(vect_enhance_data_refs_alignment): Ignore DRs not relevant
for alignment when computing how many same-align refs exist.
---
gcc/tree-vect-data-refs.cc | 95 ++++++++++++++++++++------------------
1 file changed, 50 insertions(+), 45 deletions(-)
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index a24ddfbc384..0601d592ca3 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -1754,6 +1754,12 @@ vect_relevant_for_alignment_p (dr_vec_info *dr_info)
&& !STMT_VINFO_GROUPED_ACCESS (stmt_info))
return false;
+ /* When the DR is dead, which can happen for conditional store
+ pattern recognized refs, it is no longer relevant (and we didn't
+ even analyze its alignment). */
+ if (dr_info->misalignment == DR_MISALIGNMENT_UNINITIALIZED)
+ return false;
+
return true;
}
@@ -2391,15 +2397,17 @@ vect_enhance_data_refs_alignment (loop_vec_info
loop_vinfo)
for (unsigned j = i0; j < i; ++j)
{
dr_vec_info *dr_infoj = loop_vinfo->lookup_dr (datarefs[j]);
- for (unsigned k = i0; k < i; ++k)
- {
- if (k == j)
- continue;
- dr_vec_info *dr_infok = loop_vinfo->lookup_dr (datarefs[k]);
- if (vect_dr_aligned_if_related_peeled_dr_is (dr_infok,
- dr_infoj))
- n_same_align_refs[j]++;
- }
+ if (vect_relevant_for_alignment_p (dr_infoj))
+ for (unsigned k = i0; k < i; ++k)
+ {
+ if (k == j)
+ continue;
+ dr_vec_info *dr_infok = loop_vinfo->lookup_dr (datarefs[k]);
+ if (vect_relevant_for_alignment_p (dr_infok)
+ && vect_dr_aligned_if_related_peeled_dr_is (dr_infok,
+ dr_infoj))
+ n_same_align_refs[j]++;
+ }
}
i0 = i;
}
@@ -3083,51 +3091,32 @@ vect_analyze_data_refs_alignment (loop_vec_info
loop_vinfo)
{
DUMP_VECT_SCOPE ("vect_analyze_data_refs_alignment");
- vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
- struct data_reference *dr;
- unsigned int i;
-
vect_record_base_alignments (loop_vinfo);
- FOR_EACH_VEC_ELT (datarefs, i, dr)
- {
- dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
- if (STMT_VINFO_VECTORIZABLE (dr_info->stmt))
- {
- if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt)
- && DR_GROUP_FIRST_ELEMENT (dr_info->stmt) != dr_info->stmt)
- continue;
-
- vect_compute_data_ref_alignment (loop_vinfo, dr_info,
- STMT_VINFO_VECTYPE (dr_info->stmt));
- }
- }
+ for (slp_instance inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
+ vect_slp_analyze_instance_alignment (loop_vinfo, inst);
return opt_result::success ();
}
-/* Analyze alignment of DRs of stmts in NODE. */
+/* Analyze alignment of a DRs. */
-static bool
-vect_slp_analyze_node_alignment (vec_info *vinfo, slp_tree node)
+static void
+vect_slp_analyze_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
+ tree vectype)
{
- /* Alignment is maintained in the first element of the group. */
- stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
- first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
- dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
- tree vectype = SLP_TREE_VECTYPE (node);
poly_uint64 vector_alignment
= exact_div (targetm.vectorize.preferred_vector_alignment (vectype),
BITS_PER_UNIT);
if (dr_info->misalignment == DR_MISALIGNMENT_UNINITIALIZED)
- vect_compute_data_ref_alignment (vinfo, dr_info, SLP_TREE_VECTYPE (node));
+ vect_compute_data_ref_alignment (vinfo, dr_info, vectype);
/* Re-analyze alignment when we're facing a vectorization with a bigger
alignment requirement. */
else if (known_lt (dr_info->target_alignment, vector_alignment))
{
poly_uint64 old_target_alignment = dr_info->target_alignment;
int old_misalignment = dr_info->misalignment;
- vect_compute_data_ref_alignment (vinfo, dr_info, SLP_TREE_VECTYPE
(node));
+ vect_compute_data_ref_alignment (vinfo, dr_info, vectype);
/* But keep knowledge about a smaller alignment. */
if (old_misalignment != DR_MISALIGNMENT_UNKNOWN
&& dr_info->misalignment == DR_MISALIGNMENT_UNKNOWN)
@@ -3136,9 +3125,28 @@ vect_slp_analyze_node_alignment (vec_info *vinfo,
slp_tree node)
dr_info->misalignment = old_misalignment;
}
}
- /* When we ever face unordered target alignments the first one wins in terms
- of analyzing and the other will become unknown in dr_misalignment. */
- return true;
+}
+
+
+/* Analyze alignment of DRs of stmts in NODE. */
+
+static void
+vect_slp_analyze_node_alignment (vec_info *vinfo, slp_tree node)
+{
+ tree vectype = SLP_TREE_VECTYPE (node);
+ for (stmt_vec_info stmt_info : SLP_TREE_SCALAR_STMTS (node))
+ {
+ if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
+ {
+ /* Alignment is maintained in the first element of the group. */
+ stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
+ dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
+ vect_slp_analyze_dr_alignment (vinfo, dr_info, vectype);
+ return;
+ }
+ dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
+ vect_slp_analyze_dr_alignment (vinfo, dr_info, vectype);
+ }
}
/* Function vect_slp_analyze_instance_alignment
@@ -3155,13 +3163,10 @@ vect_slp_analyze_instance_alignment (vec_info *vinfo,
slp_tree node;
unsigned i;
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, node)
- if (! vect_slp_analyze_node_alignment (vinfo, node))
- return false;
+ vect_slp_analyze_node_alignment (vinfo, node);
- if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store
- && ! vect_slp_analyze_node_alignment
- (vinfo, SLP_INSTANCE_TREE (instance)))
- return false;
+ if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store)
+ vect_slp_analyze_node_alignment (vinfo, SLP_INSTANCE_TREE (instance));
return true;
}
--
2.43.0
