On Mon, 27 Nov 2023, Tamar Christina wrote:
> Hi All,
>
> To make code review of the updates to add multiple exit supports to
> vectorizable_live_operation easier I've extracted the refactoring part to
> its own patch.
>
> This patch is a straight extract of the function with no functional changes.
>
> Bootstrapped Regtested on aarch64-none-linux-gnu and no issues.
>
> Ok for master?
OK.
Richard.
> Thanks,
> Tamar
>
> gcc/ChangeLog:
>
> * tree-vect-loop.cc (vectorizable_live_operation_1): New.
> (vectorizable_live_operation): Extract code to
> vectorizable_live_operation_1.
>
> --- inline copy of patch --
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index
> 8a50380de49bc12105be47ea1d8ee3cf1f2bdab4..df5e1d28fac2ce35e71decdec0d8e31fb75557f5
> 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -10481,6 +10481,95 @@ vectorizable_induction (loop_vec_info loop_vinfo,
> return true;
> }
>
> +
> +/* Function vectorizable_live_operation_1.
> + helper function for vectorizable_live_operation. */
> +tree
> +vectorizable_live_operation_1 (loop_vec_info loop_vinfo,
> + stmt_vec_info stmt_info, edge exit_e,
> + tree vectype, int ncopies, slp_tree slp_node,
> + tree bitsize, tree bitstart, tree vec_lhs,
> + tree lhs_type, gimple_stmt_iterator *exit_gsi)
> +{
> + basic_block exit_bb = exit_e->dest;
> + gcc_assert (single_pred_p (exit_bb) || LOOP_VINFO_EARLY_BREAKS
> (loop_vinfo));
> +
> + tree vec_lhs_phi = copy_ssa_name (vec_lhs);
> + gimple *phi = create_phi_node (vec_lhs_phi, exit_bb);
> + for (unsigned i = 0; i < gimple_phi_num_args (phi); i++)
> + SET_PHI_ARG_DEF (phi, i, vec_lhs);
> +
> + gimple_seq stmts = NULL;
> + tree new_tree;
> + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
> + {
> + /* Emit:
> + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
> + where VEC_LHS is the vectorized live-out result and MASK is
> + the loop mask for the final iteration. */
> + gcc_assert (ncopies == 1 && !slp_node);
> + gimple_seq tem = NULL;
> + gimple_stmt_iterator gsi = gsi_last (tem);
> + tree len = vect_get_loop_len (loop_vinfo, &gsi,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 0, 0);
> + /* BIAS - 1. */
> + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> + tree bias_minus_one
> + = int_const_binop (MINUS_EXPR,
> + build_int_cst (TREE_TYPE (len), biasval),
> + build_one_cst (TREE_TYPE (len)));
> + /* LAST_INDEX = LEN + (BIAS - 1). */
> + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
> + len, bias_minus_one);
> + /* This needs to implement extraction of the first index, but not sure
> + how the LEN stuff works. At the moment we shouldn't get here since
> + there's no LEN support for early breaks. But guard this so there's
> + no incorrect codegen. */
> + gcc_assert (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo));
> +
> + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */
> + tree scalar_res
> + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
> + vec_lhs_phi, last_index);
> + /* Convert the extracted vector element to the scalar type. */
> + new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
> + }
> + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> + {
> + /* Emit:
> + SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK>
> + where VEC_LHS is the vectorized live-out result and MASK is
> + the loop mask for the final iteration. */
> + gcc_assert (!slp_node);
> + tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info));
> + gimple_seq tem = NULL;
> + gimple_stmt_iterator gsi = gsi_last (tem);
> + tree mask = vect_get_loop_mask (loop_vinfo, &gsi,
> + &LOOP_VINFO_MASKS (loop_vinfo),
> + 1, vectype, 0);
> +
> + gimple_seq_add_seq (&stmts, tem);
> + tree scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type,
> + mask, vec_lhs_phi);
> + /* Convert the extracted vector element to the scalar type. */
> + new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
> + }
> + else
> + {
> + tree bftype = TREE_TYPE (vectype);
> + if (VECTOR_BOOLEAN_TYPE_P (vectype))
> + bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
> + new_tree = build3 (BIT_FIELD_REF, bftype, vec_lhs_phi, bitsize,
> bitstart);
> + new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
> + &stmts, true, NULL_TREE);
> + }
> + *exit_gsi = gsi_after_labels (exit_bb);
> + if (stmts)
> + gsi_insert_seq_before (exit_gsi, stmts, GSI_SAME_STMT);
> + return new_tree;
> +}
> +
> /* Function vectorizable_live_operation.
>
> STMT_INFO computes a value that is used outside the loop. Check if
> @@ -10690,79 +10779,13 @@ vectorizable_live_operation (vec_info *vinfo,
> stmt_vec_info stmt_info,
> gimple *phi = create_phi_node (vec_lhs_phi, exit_bb);
> SET_PHI_ARG_DEF (phi, LOOP_VINFO_IV_EXIT (loop_vinfo)->dest_idx,
> vec_lhs);
>
> - gimple_seq stmts = NULL;
> - tree new_tree;
> - if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
> - {
> - /* Emit:
> -
> - SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
> -
> - where VEC_LHS is the vectorized live-out result and MASK is
> - the loop mask for the final iteration. */
> - gcc_assert (ncopies == 1 && !slp_node);
> - gimple_seq tem = NULL;
> - gimple_stmt_iterator gsi = gsi_last (tem);
> - tree len
> - = vect_get_loop_len (loop_vinfo, &gsi,
> - &LOOP_VINFO_LENS (loop_vinfo),
> - 1, vectype, 0, 0);
> -
> - /* BIAS - 1. */
> - signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> - tree bias_minus_one
> - = int_const_binop (MINUS_EXPR,
> - build_int_cst (TREE_TYPE (len), biasval),
> - build_one_cst (TREE_TYPE (len)));
> -
> - /* LAST_INDEX = LEN + (BIAS - 1). */
> - tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
> - len, bias_minus_one);
> -
> - /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */
> - tree scalar_res
> - = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
> - vec_lhs_phi, last_index);
> -
> - /* Convert the extracted vector element to the scalar type. */
> - new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
> - }
> - else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> - {
> - /* Emit:
> -
> - SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK>
> -
> - where VEC_LHS is the vectorized live-out result and MASK is
> - the loop mask for the final iteration. */
> - gcc_assert (ncopies == 1 && !slp_node);
> - tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info));
> - gimple_seq tem = NULL;
> - gimple_stmt_iterator gsi = gsi_last (tem);
> - tree mask = vect_get_loop_mask (loop_vinfo, &gsi,
> - &LOOP_VINFO_MASKS (loop_vinfo),
> - 1, vectype, 0);
> - gimple_seq_add_seq (&stmts, tem);
> - tree scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type,
> - mask, vec_lhs_phi);
> -
> - /* Convert the extracted vector element to the scalar type. */
> - new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
> - }
> - else
> - {
> - tree bftype = TREE_TYPE (vectype);
> - if (VECTOR_BOOLEAN_TYPE_P (vectype))
> - bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
> - new_tree = build3 (BIT_FIELD_REF, bftype,
> - vec_lhs_phi, bitsize, bitstart);
> - new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
> - &stmts, true, NULL_TREE);
> - }
> -
> - gimple_stmt_iterator exit_gsi = gsi_after_labels (exit_bb);
> - if (stmts)
> - gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
> + gimple_stmt_iterator exit_gsi;
> + tree new_tree
> + = vectorizable_live_operation_1 (loop_vinfo, stmt_info,
> + LOOP_VINFO_IV_EXIT (loop_vinfo),
> + vectype, ncopies, slp_node, bitsize,
> + bitstart, vec_lhs, lhs_type,
> + &exit_gsi);
>
> /* Remove existing phis that copy from lhs and create copies
> from new_tree. */
>
>
>
>
>
--
Richard Biener <rguent...@suse.de>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Ivo Totev, Andrew McDonald, Werner Knoblich; (HRB 36809, AG Nuernberg)
