Re: [PATCH] vect: Factor out the handling on scatter store having gs_info.decl
On Thu, Aug 17, 2023 at 8:22 AM Kewen.Lin wrote:
>
> Hi,
>
> Similar to the existing function vect_build_gather_load_calls,
> this patch is to factor out the handling on scatter store
> having gs_info.decl to vect_build_scatter_store_calls which
> is a new function. It also does some minor refactoring like
> moving some variables' declarations close to their uses and
> restrict the scope for some of them etc.
>
> It's a pre-patch for upcoming vectorizable_store re-structuring
> for costing.
>
> Bootstrapped and regtested on x86_64-redhat-linux,
> aarch64-linux-gnu and powerpc64{,le}-linux-gnu.
>
> Is it ok for trunk?
OK.
Richard.
> Kewen
> -
>
> gcc/ChangeLog:
>
> * tree-vect-stmts.cc (vect_build_scatter_store_calls): New, factor
> out from ...
> (vectorizable_store): ... here.
> ---
> gcc/tree-vect-stmts.cc | 411 +
> 1 file changed, 212 insertions(+), 199 deletions(-)
>
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
> index cd8e0a76374..f8a904de503 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -2989,6 +2989,216 @@ vect_build_gather_load_calls (vec_info *vinfo,
> stmt_vec_info stmt_info,
>*vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
> }
>
> +/* Build a scatter store call while vectorizing STMT_INFO. Insert new
> + instructions before GSI and add them to VEC_STMT. GS_INFO describes
> + the scatter store operation. If the store is conditional, MASK is the
> + unvectorized condition, otherwise MASK is null. */
> +
> +static void
> +vect_build_scatter_store_calls (vec_info *vinfo, stmt_vec_info stmt_info,
> + gimple_stmt_iterator *gsi, gimple **vec_stmt,
> + gather_scatter_info *gs_info, tree mask)
> +{
> + loop_vec_info loop_vinfo = dyn_cast (vinfo);
> + tree vectype = STMT_VINFO_VECTYPE (stmt_info);
> + poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
> + int ncopies = vect_get_num_copies (loop_vinfo, vectype);
> + enum { NARROW, NONE, WIDEN } modifier;
> + poly_uint64 scatter_off_nunits
> += TYPE_VECTOR_SUBPARTS (gs_info->offset_vectype);
> +
> + tree perm_mask = NULL_TREE, mask_halfvectype = NULL_TREE;
> + if (known_eq (nunits, scatter_off_nunits))
> +modifier = NONE;
> + else if (known_eq (nunits * 2, scatter_off_nunits))
> +{
> + modifier = WIDEN;
> +
> + /* Currently gathers and scatters are only supported for
> +fixed-length vectors. */
> + unsigned int count = scatter_off_nunits.to_constant ();
> + vec_perm_builder sel (count, count, 1);
> + for (unsigned i = 0; i < (unsigned int) count; ++i)
> + sel.quick_push (i | (count / 2));
> +
> + vec_perm_indices indices (sel, 1, count);
> + perm_mask = vect_gen_perm_mask_checked (gs_info->offset_vectype,
> indices);
> + gcc_assert (perm_mask != NULL_TREE);
> +}
> + else if (known_eq (nunits, scatter_off_nunits * 2))
> +{
> + modifier = NARROW;
> +
> + /* Currently gathers and scatters are only supported for
> +fixed-length vectors. */
> + unsigned int count = nunits.to_constant ();
> + vec_perm_builder sel (count, count, 1);
> + for (unsigned i = 0; i < (unsigned int) count; ++i)
> + sel.quick_push (i | (count / 2));
> +
> + vec_perm_indices indices (sel, 2, count);
> + perm_mask = vect_gen_perm_mask_checked (vectype, indices);
> + gcc_assert (perm_mask != NULL_TREE);
> + ncopies *= 2;
> +
> + if (mask)
> + mask_halfvectype = truth_type_for (gs_info->offset_vectype);
> +}
> + else
> +gcc_unreachable ();
> +
> + tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl));
> + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info->decl));
> + tree ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> + tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> + tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> + tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> + tree scaletype = TREE_VALUE (arglist);
> +
> + gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
> + && TREE_CODE (rettype) == VOID_TYPE);
> +
> + tree ptr = fold_convert (ptrtype, gs_info->base);
> + if (!is_gimple_min_invariant (ptr))
> +{
> + gimple_seq seq;
> + ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
> + class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
> + edge pe = loop_preheader_edge (loop);
> + basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
> + gcc_assert (!new_bb);
> +}
> +
> + tree mask_arg = NULL_TREE;
> + if (mask == NULL_TREE)
> +{
> + mask_arg = build_int_cst (masktype, -1);
> + mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype,
> NULL);
> +}
> +
> + tree scale = build_int_cst (scaletype, gs_info->scale);
> +
> + auto_vec v
[PATCH] vect: Factor out the handling on scatter store having gs_info.decl
Hi,
Similar to the existing function vect_build_gather_load_calls,
this patch is to factor out the handling on scatter store
having gs_info.decl to vect_build_scatter_store_calls which
is a new function. It also does some minor refactoring like
moving some variables' declarations close to their uses and
restrict the scope for some of them etc.
It's a pre-patch for upcoming vectorizable_store re-structuring
for costing.
Bootstrapped and regtested on x86_64-redhat-linux,
aarch64-linux-gnu and powerpc64{,le}-linux-gnu.
Is it ok for trunk?
BR,
Kewen
-
gcc/ChangeLog:
* tree-vect-stmts.cc (vect_build_scatter_store_calls): New, factor
out from ...
(vectorizable_store): ... here.
---
gcc/tree-vect-stmts.cc | 411 +
1 file changed, 212 insertions(+), 199 deletions(-)
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index cd8e0a76374..f8a904de503 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -2989,6 +2989,216 @@ vect_build_gather_load_calls (vec_info *vinfo,
stmt_vec_info stmt_info,
*vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
}
+/* Build a scatter store call while vectorizing STMT_INFO. Insert new
+ instructions before GSI and add them to VEC_STMT. GS_INFO describes
+ the scatter store operation. If the store is conditional, MASK is the
+ unvectorized condition, otherwise MASK is null. */
+
+static void
+vect_build_scatter_store_calls (vec_info *vinfo, stmt_vec_info stmt_info,
+ gimple_stmt_iterator *gsi, gimple **vec_stmt,
+ gather_scatter_info *gs_info, tree mask)
+{
+ loop_vec_info loop_vinfo = dyn_cast (vinfo);
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ int ncopies = vect_get_num_copies (loop_vinfo, vectype);
+ enum { NARROW, NONE, WIDEN } modifier;
+ poly_uint64 scatter_off_nunits
+= TYPE_VECTOR_SUBPARTS (gs_info->offset_vectype);
+
+ tree perm_mask = NULL_TREE, mask_halfvectype = NULL_TREE;
+ if (known_eq (nunits, scatter_off_nunits))
+modifier = NONE;
+ else if (known_eq (nunits * 2, scatter_off_nunits))
+{
+ modifier = WIDEN;
+
+ /* Currently gathers and scatters are only supported for
+fixed-length vectors. */
+ unsigned int count = scatter_off_nunits.to_constant ();
+ vec_perm_builder sel (count, count, 1);
+ for (unsigned i = 0; i < (unsigned int) count; ++i)
+ sel.quick_push (i | (count / 2));
+
+ vec_perm_indices indices (sel, 1, count);
+ perm_mask = vect_gen_perm_mask_checked (gs_info->offset_vectype,
indices);
+ gcc_assert (perm_mask != NULL_TREE);
+}
+ else if (known_eq (nunits, scatter_off_nunits * 2))
+{
+ modifier = NARROW;
+
+ /* Currently gathers and scatters are only supported for
+fixed-length vectors. */
+ unsigned int count = nunits.to_constant ();
+ vec_perm_builder sel (count, count, 1);
+ for (unsigned i = 0; i < (unsigned int) count; ++i)
+ sel.quick_push (i | (count / 2));
+
+ vec_perm_indices indices (sel, 2, count);
+ perm_mask = vect_gen_perm_mask_checked (vectype, indices);
+ gcc_assert (perm_mask != NULL_TREE);
+ ncopies *= 2;
+
+ if (mask)
+ mask_halfvectype = truth_type_for (gs_info->offset_vectype);
+}
+ else
+gcc_unreachable ();
+
+ tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl));
+ tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info->decl));
+ tree ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
+ tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
+ tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
+ tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
+ tree scaletype = TREE_VALUE (arglist);
+
+ gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
+ && TREE_CODE (rettype) == VOID_TYPE);
+
+ tree ptr = fold_convert (ptrtype, gs_info->base);
+ if (!is_gimple_min_invariant (ptr))
+{
+ gimple_seq seq;
+ ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ edge pe = loop_preheader_edge (loop);
+ basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
+ gcc_assert (!new_bb);
+}
+
+ tree mask_arg = NULL_TREE;
+ if (mask == NULL_TREE)
+{
+ mask_arg = build_int_cst (masktype, -1);
+ mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype, NULL);
+}
+
+ tree scale = build_int_cst (scaletype, gs_info->scale);
+
+ auto_vec vec_oprnds0;
+ auto_vec vec_oprnds1;
+ auto_vec vec_masks;
+ if (mask)
+{
+ tree mask_vectype = truth_type_for (vectype);
+ vect_get_vec_defs_for_operand (vinfo, stmt_info,
+modifier == NARROW ? ncopies / 2 : ncopies,
+mask, &
