Richard Biener via Gcc-patches <gcc-patches@gcc.gnu.org> writes:
> This adds a fallback to the masked_ variants for gather_load
> and scatter_store if the latter are not available.
>
> Bootstrap / regtest running on x86_64-unknown-linux-gnu.
LGTM FWIW. I don't know the history behind the TREE_CODE (*mask) != SSA_NAME
check.
I guess we could probably remove the unmasked SVE optabs with this.
Thanks,
Richard
>
> 2021-08-17 Richard Biener <rguent...@suse.de>
>
> * optabs-query.c (supports_vec_gather_load_p): Also check
> for masked optabs.
> (supports_vec_scatter_store_p): Likewise.
> * tree-vect-data-refs.c (vect_gather_scatter_fn_p): Fall
> back to masked variants if non-masked are not supported.
> * tree-vect-patterns.c (vect_recog_gather_scatter_pattern):
> When we need to use masked gather/scatter but do not have
> a mask set up a constant true one.
> * tree-vect-stmts.c (vect_check_scalar_mask): Also allow
> non-SSA_NAME masks.
> ---
> gcc/optabs-query.c | 6 ++++--
> gcc/tree-vect-data-refs.c | 22 +++++++++++++++++++---
> gcc/tree-vect-patterns.c | 7 +++++--
> gcc/tree-vect-stmts.c | 8 --------
> 4 files changed, 28 insertions(+), 15 deletions(-)
>
> diff --git a/gcc/optabs-query.c b/gcc/optabs-query.c
> index 05ee5f517da..a6dd0fed610 100644
> --- a/gcc/optabs-query.c
> +++ b/gcc/optabs-query.c
> @@ -740,7 +740,8 @@ supports_vec_gather_load_p ()
> this_fn_optabs->supports_vec_gather_load_cached = true;
>
> this_fn_optabs->supports_vec_gather_load
> - = supports_vec_convert_optab_p (gather_load_optab);
> + = (supports_vec_convert_optab_p (gather_load_optab)
> + || supports_vec_convert_optab_p (mask_gather_load_optab));
>
> return this_fn_optabs->supports_vec_gather_load;
> }
> @@ -757,7 +758,8 @@ supports_vec_scatter_store_p ()
> this_fn_optabs->supports_vec_scatter_store_cached = true;
>
> this_fn_optabs->supports_vec_scatter_store
> - = supports_vec_convert_optab_p (scatter_store_optab);
> + = (supports_vec_convert_optab_p (scatter_store_optab)
> + || supports_vec_convert_optab_p (mask_scatter_store_optab));
>
> return this_fn_optabs->supports_vec_scatter_store;
> }
> diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
> index c4c14d74065..97745a830a2 100644
> --- a/gcc/tree-vect-data-refs.c
> +++ b/gcc/tree-vect-data-refs.c
> @@ -3735,11 +3735,17 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool
> read_p, bool masked_p,
> return false;
>
> /* Work out which function we need. */
> - internal_fn ifn;
> + internal_fn ifn, alt_ifn;
> if (read_p)
> - ifn = masked_p ? IFN_MASK_GATHER_LOAD : IFN_GATHER_LOAD;
> + {
> + ifn = masked_p ? IFN_MASK_GATHER_LOAD : IFN_GATHER_LOAD;
> + alt_ifn = IFN_MASK_GATHER_LOAD;
> + }
> else
> - ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE;
> + {
> + ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE;
> + alt_ifn = IFN_MASK_SCATTER_STORE;
> + }
>
> for (;;)
> {
> @@ -3755,6 +3761,16 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool
> read_p, bool masked_p,
> *offset_vectype_out = offset_vectype;
> return true;
> }
> + else if (!masked_p
> + && internal_gather_scatter_fn_supported_p (alt_ifn, vectype,
> + memory_type,
> + offset_vectype,
> + scale))
> + {
> + *ifn_out = alt_ifn;
> + *offset_vectype_out = offset_vectype;
> + return true;
> + }
>
> if (TYPE_PRECISION (offset_type) >= POINTER_SIZE
> && TYPE_PRECISION (offset_type) >= element_bits)
> diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
> index 25de97bd9b0..899734005ce 100644
> --- a/gcc/tree-vect-patterns.c
> +++ b/gcc/tree-vect-patterns.c
> @@ -4820,6 +4820,9 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo,
> if (mask)
> mask = vect_convert_mask_for_vectype (mask, gs_vectype, stmt_info,
> loop_vinfo);
> + else if (gs_info.ifn == IFN_MASK_SCATTER_STORE
> + || gs_info.ifn == IFN_MASK_GATHER_LOAD)
> + mask = build_int_cst (TREE_TYPE (truth_type_for (gs_vectype)), -1);
>
> /* Get the invariant base and non-invariant offset, converting the
> latter to the same width as the vector elements. */
> @@ -4847,11 +4850,11 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo,
> {
> tree rhs = vect_get_store_rhs (stmt_info);
> if (mask != NULL)
> - pattern_stmt = gimple_build_call_internal (IFN_MASK_SCATTER_STORE, 5,
> + pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5,
> base, offset, scale, rhs,
> mask);
> else
> - pattern_stmt = gimple_build_call_internal (IFN_SCATTER_STORE, 4,
> + pattern_stmt = gimple_build_call_internal (gs_info.ifn, 4,
> base, offset, scale, rhs);
> }
> gimple_call_set_nothrow (pattern_stmt, true);
> diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
> index e356056be93..59100965d5e 100644
> --- a/gcc/tree-vect-stmts.c
> +++ b/gcc/tree-vect-stmts.c
> @@ -2510,14 +2510,6 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info
> stmt_info,
> return false;
> }
>
> - if (TREE_CODE (*mask) != SSA_NAME)
> - {
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> - "mask argument is not an SSA name.\n");
> - return false;
> - }
> -
> /* If the caller is not prepared for adjusting an external/constant
> SLP mask vector type fail. */
> if (slp_node
