Re: [PATCH] vect: Factor out the handling on scatter store having gs_info.decl
On Thu, Aug 17, 2023 at 8:22 AM Kewen.Lin wrote: > > Hi, > > Similar to the existing function vect_build_gather_load_calls, > this patch is to factor out the handling on scatter store > having gs_info.decl to vect_build_scatter_store_calls which > is a new function. It also does some minor refactoring like > moving some variables' declarations close to their uses and > restrict the scope for some of them etc. > > It's a pre-patch for upcoming vectorizable_store re-structuring > for costing. > > Bootstrapped and regtested on x86_64-redhat-linux, > aarch64-linux-gnu and powerpc64{,le}-linux-gnu. > > Is it ok for trunk? OK. Richard. > Kewen > - > > gcc/ChangeLog: > > * tree-vect-stmts.cc (vect_build_scatter_store_calls): New, factor > out from ... > (vectorizable_store): ... here. > --- > gcc/tree-vect-stmts.cc | 411 + > 1 file changed, 212 insertions(+), 199 deletions(-) > > diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc > index cd8e0a76374..f8a904de503 100644 > --- a/gcc/tree-vect-stmts.cc > +++ b/gcc/tree-vect-stmts.cc > @@ -2989,6 +2989,216 @@ vect_build_gather_load_calls (vec_info *vinfo, > stmt_vec_info stmt_info, >*vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; > } > > +/* Build a scatter store call while vectorizing STMT_INFO. Insert new > + instructions before GSI and add them to VEC_STMT. GS_INFO describes > + the scatter store operation. If the store is conditional, MASK is the > + unvectorized condition, otherwise MASK is null. */ > + > +static void > +vect_build_scatter_store_calls (vec_info *vinfo, stmt_vec_info stmt_info, > + gimple_stmt_iterator *gsi, gimple **vec_stmt, > + gather_scatter_info *gs_info, tree mask) > +{ > + loop_vec_info loop_vinfo = dyn_cast (vinfo); > + tree vectype = STMT_VINFO_VECTYPE (stmt_info); > + poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); > + int ncopies = vect_get_num_copies (loop_vinfo, vectype); > + enum { NARROW, NONE, WIDEN } modifier; > + poly_uint64 scatter_off_nunits > += TYPE_VECTOR_SUBPARTS (gs_info->offset_vectype); > + > + tree perm_mask = NULL_TREE, mask_halfvectype = NULL_TREE; > + if (known_eq (nunits, scatter_off_nunits)) > +modifier = NONE; > + else if (known_eq (nunits * 2, scatter_off_nunits)) > +{ > + modifier = WIDEN; > + > + /* Currently gathers and scatters are only supported for > +fixed-length vectors. */ > + unsigned int count = scatter_off_nunits.to_constant (); > + vec_perm_builder sel (count, count, 1); > + for (unsigned i = 0; i < (unsigned int) count; ++i) > + sel.quick_push (i | (count / 2)); > + > + vec_perm_indices indices (sel, 1, count); > + perm_mask = vect_gen_perm_mask_checked (gs_info->offset_vectype, > indices); > + gcc_assert (perm_mask != NULL_TREE); > +} > + else if (known_eq (nunits, scatter_off_nunits * 2)) > +{ > + modifier = NARROW; > + > + /* Currently gathers and scatters are only supported for > +fixed-length vectors. */ > + unsigned int count = nunits.to_constant (); > + vec_perm_builder sel (count, count, 1); > + for (unsigned i = 0; i < (unsigned int) count; ++i) > + sel.quick_push (i | (count / 2)); > + > + vec_perm_indices indices (sel, 2, count); > + perm_mask = vect_gen_perm_mask_checked (vectype, indices); > + gcc_assert (perm_mask != NULL_TREE); > + ncopies *= 2; > + > + if (mask) > + mask_halfvectype = truth_type_for (gs_info->offset_vectype); > +} > + else > +gcc_unreachable (); > + > + tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl)); > + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info->decl)); > + tree ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > + tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > + tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > + tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > + tree scaletype = TREE_VALUE (arglist); > + > + gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE > + && TREE_CODE (rettype) == VOID_TYPE); > + > + tree ptr = fold_convert (ptrtype, gs_info->base); > + if (!is_gimple_min_invariant (ptr)) > +{ > + gimple_seq seq; > + ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); > + class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); > + edge pe = loop_preheader_edge (loop); > + basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, seq); > + gcc_assert (!new_bb); > +} > + > + tree mask_arg = NULL_TREE; > + if (mask == NULL_TREE) > +{ > + mask_arg = build_int_cst (masktype, -1); > + mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype, > NULL); > +} > + > + tree scale = build_int_cst (scaletype, gs_info->scale); > + > + auto_vec v
[PATCH] vect: Factor out the handling on scatter store having gs_info.decl
Hi, Similar to the existing function vect_build_gather_load_calls, this patch is to factor out the handling on scatter store having gs_info.decl to vect_build_scatter_store_calls which is a new function. It also does some minor refactoring like moving some variables' declarations close to their uses and restrict the scope for some of them etc. It's a pre-patch for upcoming vectorizable_store re-structuring for costing. Bootstrapped and regtested on x86_64-redhat-linux, aarch64-linux-gnu and powerpc64{,le}-linux-gnu. Is it ok for trunk? BR, Kewen - gcc/ChangeLog: * tree-vect-stmts.cc (vect_build_scatter_store_calls): New, factor out from ... (vectorizable_store): ... here. --- gcc/tree-vect-stmts.cc | 411 + 1 file changed, 212 insertions(+), 199 deletions(-) diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index cd8e0a76374..f8a904de503 100644 --- a/gcc/tree-vect-stmts.cc +++ b/gcc/tree-vect-stmts.cc @@ -2989,6 +2989,216 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info, *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; } +/* Build a scatter store call while vectorizing STMT_INFO. Insert new + instructions before GSI and add them to VEC_STMT. GS_INFO describes + the scatter store operation. If the store is conditional, MASK is the + unvectorized condition, otherwise MASK is null. */ + +static void +vect_build_scatter_store_calls (vec_info *vinfo, stmt_vec_info stmt_info, + gimple_stmt_iterator *gsi, gimple **vec_stmt, + gather_scatter_info *gs_info, tree mask) +{ + loop_vec_info loop_vinfo = dyn_cast (vinfo); + tree vectype = STMT_VINFO_VECTYPE (stmt_info); + poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); + int ncopies = vect_get_num_copies (loop_vinfo, vectype); + enum { NARROW, NONE, WIDEN } modifier; + poly_uint64 scatter_off_nunits += TYPE_VECTOR_SUBPARTS (gs_info->offset_vectype); + + tree perm_mask = NULL_TREE, mask_halfvectype = NULL_TREE; + if (known_eq (nunits, scatter_off_nunits)) +modifier = NONE; + else if (known_eq (nunits * 2, scatter_off_nunits)) +{ + modifier = WIDEN; + + /* Currently gathers and scatters are only supported for +fixed-length vectors. */ + unsigned int count = scatter_off_nunits.to_constant (); + vec_perm_builder sel (count, count, 1); + for (unsigned i = 0; i < (unsigned int) count; ++i) + sel.quick_push (i | (count / 2)); + + vec_perm_indices indices (sel, 1, count); + perm_mask = vect_gen_perm_mask_checked (gs_info->offset_vectype, indices); + gcc_assert (perm_mask != NULL_TREE); +} + else if (known_eq (nunits, scatter_off_nunits * 2)) +{ + modifier = NARROW; + + /* Currently gathers and scatters are only supported for +fixed-length vectors. */ + unsigned int count = nunits.to_constant (); + vec_perm_builder sel (count, count, 1); + for (unsigned i = 0; i < (unsigned int) count; ++i) + sel.quick_push (i | (count / 2)); + + vec_perm_indices indices (sel, 2, count); + perm_mask = vect_gen_perm_mask_checked (vectype, indices); + gcc_assert (perm_mask != NULL_TREE); + ncopies *= 2; + + if (mask) + mask_halfvectype = truth_type_for (gs_info->offset_vectype); +} + else +gcc_unreachable (); + + tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl)); + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info->decl)); + tree ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); + tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); + tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); + tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); + tree scaletype = TREE_VALUE (arglist); + + gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE + && TREE_CODE (rettype) == VOID_TYPE); + + tree ptr = fold_convert (ptrtype, gs_info->base); + if (!is_gimple_min_invariant (ptr)) +{ + gimple_seq seq; + ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); + class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); + edge pe = loop_preheader_edge (loop); + basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, seq); + gcc_assert (!new_bb); +} + + tree mask_arg = NULL_TREE; + if (mask == NULL_TREE) +{ + mask_arg = build_int_cst (masktype, -1); + mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype, NULL); +} + + tree scale = build_int_cst (scaletype, gs_info->scale); + + auto_vec vec_oprnds0; + auto_vec vec_oprnds1; + auto_vec vec_masks; + if (mask) +{ + tree mask_vectype = truth_type_for (vectype); + vect_get_vec_defs_for_operand (vinfo, stmt_info, +modifier == NARROW ? ncopies / 2 : ncopies, +mask, &