On Mon, Jun 18, 2018 at 4:51 PM Richard Sandiford
<richard.sandif...@arm.com> wrote:
>
> This patch is the first part of a series to fix to PR85694.
> Later patches can make the pattern for a statement S2 reuse the
> results of a PATTERN_DEF_SEQ statement attached to an earlier
> statement S1. Although vect_mark_stmts_to_be_vectorized handled
> this fine, vect_analyze_stmt and vect_transform_loop both skipped the
> PATTERN_DEF_SEQ for S1 if S1's main pattern wasn't live or relevant.
>
> I couldn't wrap my head around the flow in vect_transform_loop,
> so ended up moving the per-statement handling into a subroutine.
> That makes the patch look bigger than it actually is.
>
> Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install?
OK.
Thanks,
Richard.
> Richard
>
>
> 2018-06-18 Richard Sandiford <richard.sandif...@arm.com>
>
> gcc/
> * tree-vect-stmts.c (vect_analyze_stmt): Move the handling of pattern
> definition statements before the early exit for statements that aren't
> live or relevant.
> * tree-vect-loop.c (vect_transform_loop_stmt): New function,
> split out from...
> (vect_transform_loop): ...here. Process pattern definition
> statements without first checking whether the main pattern
> statement is live or relevant.
>
> Index: gcc/tree-vect-stmts.c
> ===================================================================
> *** gcc/tree-vect-stmts.c 2018-06-18 15:50:55.607303415 +0100
> --- gcc/tree-vect-stmts.c 2018-06-18 15:50:55.603303451 +0100
> *************** vect_analyze_stmt (gimple *stmt, bool *n
> *** 9409,9414 ****
> --- 9409,9442 ----
> return false;
> }
>
> + if (STMT_VINFO_IN_PATTERN_P (stmt_info)
> + && node == NULL
> + && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
> + {
> + gimple_stmt_iterator si;
> +
> + for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next
> (&si))
> + {
> + gimple *pattern_def_stmt = gsi_stmt (si);
> + if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
> + || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt)))
> + {
> + /* Analyze def stmt of STMT if it's a pattern stmt. */
> + if (dump_enabled_p ())
> + {
> + dump_printf_loc (MSG_NOTE, vect_location,
> + "==> examining pattern def statement: ");
> + dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0);
> + }
> +
> + if (!vect_analyze_stmt (pattern_def_stmt,
> + need_to_vectorize, node, node_instance,
> + cost_vec))
> + return false;
> + }
> + }
> + }
> +
> /* Skip stmts that do not need to be vectorized. In loops this is expected
> to include:
> - the COND_EXPR which is the loop exit condition
> *************** vect_analyze_stmt (gimple *stmt, bool *n
> *** 9469,9502 ****
> return false;
> }
>
> - if (is_pattern_stmt_p (stmt_info)
> - && node == NULL
> - && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
> - {
> - gimple_stmt_iterator si;
> -
> - for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next
> (&si))
> - {
> - gimple *pattern_def_stmt = gsi_stmt (si);
> - if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
> - || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt)))
> - {
> - /* Analyze def stmt of STMT if it's a pattern stmt. */
> - if (dump_enabled_p ())
> - {
> - dump_printf_loc (MSG_NOTE, vect_location,
> - "==> examining pattern def statement: ");
> - dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0);
> - }
> -
> - if (!vect_analyze_stmt (pattern_def_stmt,
> - need_to_vectorize, node, node_instance,
> - cost_vec))
> - return false;
> - }
> - }
> - }
> -
> switch (STMT_VINFO_DEF_TYPE (stmt_info))
> {
> case vect_internal_def:
> --- 9497,9502 ----
> Index: gcc/tree-vect-loop.c
> ===================================================================
> *** gcc/tree-vect-loop.c 2018-06-18 15:50:55.607303415 +0100
> --- gcc/tree-vect-loop.c 2018-06-18 15:50:55.603303451 +0100
> *************** scale_profile_for_vect_loop (struct loop
> *** 8306,8311 ****
> --- 8306,8381 ----
> scale_bbs_frequencies (&loop->latch, 1, exit_l->probability / prob);
> }
>
> + /* Vectorize STMT if relevant, inserting any new instructions before GSI.
> + When vectorizing STMT as a store, set *SEEN_STORE to its stmt_vec_info.
> + *SLP_SCHEDULE is a running record of whether we have called
> + vect_schedule_slp. */
> +
> + static void
> + vect_transform_loop_stmt (loop_vec_info loop_vinfo, gimple *stmt,
> + gimple_stmt_iterator *gsi,
> + stmt_vec_info *seen_store, bool *slp_scheduled)
> + {
> + struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
> + poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
> + stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
> + if (!stmt_info)
> + return;
> +
> + if (dump_enabled_p ())
> + {
> + dump_printf_loc (MSG_NOTE, vect_location,
> + "------>vectorizing statement: ");
> + dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
> + }
> +
> + if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
> + vect_loop_kill_debug_uses (loop, stmt);
> +
> + if (!STMT_VINFO_RELEVANT_P (stmt_info)
> + && !STMT_VINFO_LIVE_P (stmt_info))
> + return;
> +
> + if (STMT_VINFO_VECTYPE (stmt_info))
> + {
> + poly_uint64 nunits
> + = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
> + if (!STMT_SLP_TYPE (stmt_info)
> + && maybe_ne (nunits, vf)
> + && dump_enabled_p ())
> + /* For SLP VF is set according to unrolling factor, and not
> + to vector size, hence for SLP this print is not valid. */
> + dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n");
> + }
> +
> + /* SLP. Schedule all the SLP instances when the first SLP stmt is
> + reached. */
> + if (STMT_SLP_TYPE (stmt_info))
> + {
> + if (!*slp_scheduled)
> + {
> + *slp_scheduled = true;
> +
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location,
> + "=== scheduling SLP instances ===\n");
> +
> + vect_schedule_slp (loop_vinfo);
> + }
> +
> + /* Hybrid SLP stmts must be vectorized in addition to SLP. */
> + if (PURE_SLP_STMT (stmt_info))
> + return;
> + }
> +
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n");
> +
> + bool grouped_store = false;
> + if (vect_transform_stmt (stmt, gsi, &grouped_store, NULL, NULL))
> + *seen_store = stmt_info;
> + }
> +
> /* Function vect_transform_loop.
>
> The analysis phase has determined that the loop is vectorizable.
> *************** vect_transform_loop (loop_vec_info loop_
> *** 8326,8337 ****
> tree niters_vector_mult_vf = NULL_TREE;
> poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
> unsigned int lowest_vf = constant_lower_bound (vf);
> - bool grouped_store;
> bool slp_scheduled = false;
> ! gimple *stmt, *pattern_stmt;
> ! gimple_seq pattern_def_seq = NULL;
> ! gimple_stmt_iterator pattern_def_si = gsi_none ();
> ! bool transform_pattern_stmt = false;
> bool check_profitability = false;
> unsigned int th;
>
> --- 8396,8403 ----
> tree niters_vector_mult_vf = NULL_TREE;
> poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
> unsigned int lowest_vf = constant_lower_bound (vf);
> bool slp_scheduled = false;
> ! gimple *stmt;
> bool check_profitability = false;
> unsigned int th;
>
> *************** vect_transform_loop (loop_vec_info loop_
> *** 8483,8678 ****
> }
> }
>
> - pattern_stmt = NULL;
> for (gimple_stmt_iterator si = gsi_start_bb (bb);
> ! !gsi_end_p (si) || transform_pattern_stmt;)
> {
> ! bool is_store;
> !
> ! if (transform_pattern_stmt)
> ! stmt = pattern_stmt;
> ! else
> ! {
> ! stmt = gsi_stmt (si);
> ! /* During vectorization remove existing clobber stmts. */
> ! if (gimple_clobber_p (stmt))
> ! {
> ! unlink_stmt_vdef (stmt);
> ! gsi_remove (&si, true);
> ! release_defs (stmt);
> ! continue;
> ! }
> ! }
> !
> ! if (dump_enabled_p ())
> {
> ! dump_printf_loc (MSG_NOTE, vect_location,
> ! "------>vectorizing statement: ");
> ! dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
> }
> !
> ! stmt_info = vinfo_for_stmt (stmt);
> !
> ! /* vector stmts created in the outer-loop during vectorization of
> ! stmts in an inner-loop may not have a stmt_info, and do not
> ! need to be vectorized. */
> ! if (!stmt_info)
> {
> ! gsi_next (&si);
> ! continue;
> ! }
>
> ! if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
> ! vect_loop_kill_debug_uses (loop, stmt);
> !
> ! if (!STMT_VINFO_RELEVANT_P (stmt_info)
> ! && !STMT_VINFO_LIVE_P (stmt_info))
> ! {
> ! if (STMT_VINFO_IN_PATTERN_P (stmt_info)
> ! && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info))
> ! && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
> ! || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
> ! {
> ! stmt = pattern_stmt;
> ! stmt_info = vinfo_for_stmt (stmt);
> ! }
> ! else
> ! {
> ! gsi_next (&si);
> ! continue;
> ! }
> ! }
> ! else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
> ! && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info))
> ! && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
> ! || STMT_VINFO_LIVE_P (vinfo_for_stmt
> (pattern_stmt))))
> ! transform_pattern_stmt = true;
> !
> ! /* If pattern statement has def stmts, vectorize them too. */
> ! if (is_pattern_stmt_p (stmt_info))
> ! {
> ! if (pattern_def_seq == NULL)
> ! {
> ! pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
> ! pattern_def_si = gsi_start (pattern_def_seq);
> ! }
> ! else if (!gsi_end_p (pattern_def_si))
> ! gsi_next (&pattern_def_si);
> ! if (pattern_def_seq != NULL)
> {
> ! gimple *pattern_def_stmt = NULL;
> ! stmt_vec_info pattern_def_stmt_info = NULL;
> !
> ! while (!gsi_end_p (pattern_def_si))
> ! {
> ! pattern_def_stmt = gsi_stmt (pattern_def_si);
> ! pattern_def_stmt_info
> ! = vinfo_for_stmt (pattern_def_stmt);
> ! if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
> ! || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
> ! break;
> ! gsi_next (&pattern_def_si);
> ! }
> !
> ! if (!gsi_end_p (pattern_def_si))
> {
> ! if (dump_enabled_p ())
> ! {
> ! dump_printf_loc (MSG_NOTE, vect_location,
> ! "==> vectorizing pattern def "
> ! "stmt: ");
> ! dump_gimple_stmt (MSG_NOTE, TDF_SLIM,
> ! pattern_def_stmt, 0);
> ! }
> !
> ! stmt = pattern_def_stmt;
> ! stmt_info = pattern_def_stmt_info;
> }
> ! else
> ! {
> ! pattern_def_si = gsi_none ();
> ! transform_pattern_stmt = false;
> ! }
> ! }
> ! else
> ! transform_pattern_stmt = false;
> ! }
> !
> ! if (STMT_VINFO_VECTYPE (stmt_info))
> ! {
> ! poly_uint64 nunits
> ! = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
> ! if (!STMT_SLP_TYPE (stmt_info)
> ! && maybe_ne (nunits, vf)
> ! && dump_enabled_p ())
> ! /* For SLP VF is set according to unrolling factor, and not
> ! to vector size, hence for SLP this print is not valid.
> */
> ! dump_printf_loc (MSG_NOTE, vect_location,
> "multiple-types.\n");
> ! }
> !
> ! /* SLP. Schedule all the SLP instances when the first SLP stmt is
> ! reached. */
> ! if (STMT_SLP_TYPE (stmt_info))
> ! {
> ! if (!slp_scheduled)
> ! {
> ! slp_scheduled = true;
> !
> ! if (dump_enabled_p ())
> ! dump_printf_loc (MSG_NOTE, vect_location,
> ! "=== scheduling SLP instances ===\n");
> !
> ! vect_schedule_slp (loop_vinfo);
> }
> !
> ! /* Hybrid SLP stmts must be vectorized in addition to SLP. */
> ! if (!vinfo_for_stmt (stmt) || PURE_SLP_STMT (stmt_info))
> {
> ! if (!transform_pattern_stmt && gsi_end_p (pattern_def_si))
> {
> ! pattern_def_seq = NULL;
> gsi_next (&si);
> }
> - continue;
> - }
> - }
> -
> - /* -------- vectorize statement ------------ */
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_NOTE, vect_location, "transform
> statement.\n");
> -
> - grouped_store = false;
> - is_store = vect_transform_stmt (stmt, &si, &grouped_store, NULL,
> NULL);
> - if (is_store)
> - {
> - if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
> - {
> - /* Interleaving. If IS_STORE is TRUE, the vectorization of
> the
> - interleaving chain was completed - free all the stores in
> - the chain. */
> - gsi_next (&si);
> - vect_remove_stores (DR_GROUP_FIRST_ELEMENT (stmt_info));
> }
> else
> ! {
> ! /* Free the attached stmt_vec_info and remove the stmt. */
> ! gimple *store = gsi_stmt (si);
> ! free_stmt_vec_info (store);
> ! unlink_stmt_vdef (store);
> ! gsi_remove (&si, true);
> ! release_defs (store);
> ! }
> !
> ! /* Stores can only appear at the end of pattern statements. */
> ! gcc_assert (!transform_pattern_stmt);
> ! pattern_def_seq = NULL;
> }
> ! else if (!transform_pattern_stmt && gsi_end_p (pattern_def_si))
> ! {
> ! pattern_def_seq = NULL;
> ! gsi_next (&si);
> ! }
> ! } /* stmts in BB */
>
> /* Stub out scalar statements that must not survive vectorization.
> Doing this here helps with grouped statements, or statements that
> --- 8549,8615 ----
> }
> }
>
> for (gimple_stmt_iterator si = gsi_start_bb (bb);
> ! !gsi_end_p (si);)
> {
> ! stmt = gsi_stmt (si);
> ! /* During vectorization remove existing clobber stmts. */
> ! if (gimple_clobber_p (stmt))
> {
> ! unlink_stmt_vdef (stmt);
> ! gsi_remove (&si, true);
> ! release_defs (stmt);
> }
> ! else
> {
> ! stmt_info = vinfo_for_stmt (stmt);
>
> ! /* vector stmts created in the outer-loop during vectorization
> of
> ! stmts in an inner-loop may not have a stmt_info, and do not
> ! need to be vectorized. */
> ! stmt_vec_info seen_store = NULL;
> ! if (stmt_info)
> {
> ! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
> {
> ! gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ
> (stmt_info);
> ! for (gimple_stmt_iterator subsi = gsi_start (def_seq);
> ! !gsi_end_p (subsi); gsi_next (&subsi))
> ! vect_transform_loop_stmt (loop_vinfo,
> ! gsi_stmt (subsi), &si,
> ! &seen_store,
> ! &slp_scheduled);
> ! gimple *pat_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
> ! vect_transform_loop_stmt (loop_vinfo, pat_stmt, &si,
> ! &seen_store, &slp_scheduled);
> }
> ! vect_transform_loop_stmt (loop_vinfo, stmt, &si,
> ! &seen_store, &slp_scheduled);
> }
> ! if (seen_store)
> {
> ! if (STMT_VINFO_GROUPED_ACCESS (seen_store))
> {
> ! /* Interleaving. If IS_STORE is TRUE, the
> ! vectorization of the interleaving chain was
> ! completed - free all the stores in the chain. */
> gsi_next (&si);
> + vect_remove_stores (DR_GROUP_FIRST_ELEMENT
> (seen_store));
> + }
> + else
> + {
> + /* Free the attached stmt_vec_info and remove the
> + stmt. */
> + free_stmt_vec_info (stmt);
> + unlink_stmt_vdef (stmt);
> + gsi_remove (&si, true);
> + release_defs (stmt);
> }
> }
> else
> ! gsi_next (&si);
> }
> ! }
>
> /* Stub out scalar statements that must not survive vectorization.
> Doing this here helps with grouped statements, or statements that
