This patch is an initial patch to move costing next to the
transform, it still adopts vect_model_store_cost for costing
but moves and duplicates it down according to the handlings
of different vect_memory_access_types or some special
handling need, hope it can make the subsequent patches easy
to review. This patch should not have any functional
changes.
gcc/ChangeLog:
* tree-vect-stmts.cc (vectorizable_store): Move and duplicate the call
to vect_model_store_cost down to some different transform paths
according to the handlings of different vect_memory_access_types
or some special handling need.
---
gcc/tree-vect-stmts.cc | 79 ++++++++++++++++++++++++++++++++----------
1 file changed, 60 insertions(+), 19 deletions(-)
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index a5caaf0bca2..36f7c5b9f4b 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -8372,7 +8372,8 @@ vectorizable_store (vec_info *vinfo,
return false;
}
- if (!vec_stmt) /* transformation not required. */
+ bool costing_p = !vec_stmt;
+ if (costing_p) /* transformation not required. */
{
STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
@@ -8401,11 +8402,6 @@ vectorizable_store (vec_info *vinfo,
"Vectorizing an unaligned access.\n");
STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
- vect_model_store_cost (vinfo, stmt_info, ncopies,
- memory_access_type, &gs_info,
- alignment_support_scheme,
- misalignment, vls_type, slp_node, cost_vec);
- return true;
}
gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
@@ -8415,12 +8411,27 @@ vectorizable_store (vec_info *vinfo,
if (memory_access_type == VMAT_GATHER_SCATTER && gs_info.decl)
{
- vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
- &gs_info, mask);
+ if (costing_p)
+ vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+ &gs_info, alignment_support_scheme, misalignment,
+ vls_type, slp_node, cost_vec);
+ else
+ vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
+ &gs_info, mask);
return true;
}
else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
- return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt, ncopies);
+ {
+ gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
+ if (costing_p)
+ {
+ vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+ &gs_info, alignment_support_scheme,
+ misalignment, vls_type, slp_node, cost_vec);
+ return true;
+ }
+ return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt,
ncopies);
+ }
if (grouped_store)
{
@@ -8449,13 +8460,21 @@ vectorizable_store (vec_info *vinfo,
else
ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "transform store. ncopies = %d\n", ncopies);
+ if (!costing_p && dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "transform store. ncopies =
%d\n",
+ ncopies);
if (memory_access_type == VMAT_ELEMENTWISE
|| memory_access_type == VMAT_STRIDED_SLP)
{
+ if (costing_p)
+ {
+ vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+ &gs_info, alignment_support_scheme,
+ misalignment, vls_type, slp_node, cost_vec);
+ return true;
+ }
+
gimple_stmt_iterator incr_gsi;
bool insert_after;
gimple *incr;
@@ -8718,8 +8737,9 @@ vectorizable_store (vec_info *vinfo,
else if (memory_access_type == VMAT_GATHER_SCATTER)
{
aggr_type = elem_type;
- vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
- &bump, &vec_offset, loop_lens);
+ if (!costing_p)
+ vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
+ &bump, &vec_offset, loop_lens);
}
else
{
@@ -8731,7 +8751,7 @@ vectorizable_store (vec_info *vinfo,
memory_access_type, loop_lens);
}
- if (mask)
+ if (mask && !costing_p)
LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
/* In case the vectorization factor (VF) is bigger than the number
@@ -8782,6 +8802,13 @@ vectorizable_store (vec_info *vinfo,
if (memory_access_type == VMAT_LOAD_STORE_LANES)
{
gcc_assert (!slp && grouped_store);
+ if (costing_p)
+ {
+ vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+ &gs_info, alignment_support_scheme,
+ misalignment, vls_type, slp_node, cost_vec);
+ return true;
+ }
for (j = 0; j < ncopies; j++)
{
gimple *new_stmt;
@@ -8927,6 +8954,13 @@ vectorizable_store (vec_info *vinfo,
if (memory_access_type == VMAT_GATHER_SCATTER)
{
gcc_assert (!slp && !grouped_store);
+ if (costing_p)
+ {
+ vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+ &gs_info, alignment_support_scheme,
+ misalignment, vls_type, slp_node, cost_vec);
+ return true;
+ }
auto_vec<tree> vec_offsets;
for (j = 0; j < ncopies; j++)
{
@@ -9091,7 +9125,7 @@ vectorizable_store (vec_info *vinfo,
for (j = 0; j < ncopies; j++)
{
gimple *new_stmt;
- if (j == 0)
+ if (j == 0 && !costing_p)
{
if (slp)
{
@@ -9158,7 +9192,7 @@ vectorizable_store (vec_info *vinfo,
offset, &dummy, gsi, &ptr_incr,
simd_lane_access_p, bump);
}
- else
+ else if (!costing_p)
{
gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
/* DR_CHAIN is then used as an input to vect_permute_store_chain().
@@ -9179,7 +9213,7 @@ vectorizable_store (vec_info *vinfo,
}
new_stmt = NULL;
- if (grouped_store)
+ if (!costing_p && grouped_store)
/* Permute. */
vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info, gsi,
&result_chain);
@@ -9187,6 +9221,8 @@ vectorizable_store (vec_info *vinfo,
stmt_vec_info next_stmt_info = first_stmt_info;
for (i = 0; i < vec_num; i++)
{
+ if (costing_p)
+ continue;
unsigned misalign;
unsigned HOST_WIDE_INT align;
@@ -9361,7 +9397,7 @@ vectorizable_store (vec_info *vinfo,
if (!next_stmt_info)
break;
}
- if (!slp)
+ if (!slp && !costing_p)
{
if (j == 0)
*vec_stmt = new_stmt;
@@ -9369,6 +9405,11 @@ vectorizable_store (vec_info *vinfo,
}
}
+ if (costing_p)
+ vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+ &gs_info, alignment_support_scheme, misalignment,
+ vls_type, slp_node, cost_vec);
+
return true;
}
--
2.31.1
