This patch is the first part of a series to fix to PR85694. Later patches can make the pattern for a statement S2 reuse the results of a PATTERN_DEF_SEQ statement attached to an earlier statement S1. Although vect_mark_stmts_to_be_vectorized handled this fine, vect_analyze_stmt and vect_transform_loop both skipped the PATTERN_DEF_SEQ for S1 if S1's main pattern wasn't live or relevant.
I couldn't wrap my head around the flow in vect_transform_loop, so ended up moving the per-statement handling into a subroutine. That makes the patch look bigger than it actually is. Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install? Richard 2018-06-18 Richard Sandiford <richard.sandif...@arm.com> gcc/ * tree-vect-stmts.c (vect_analyze_stmt): Move the handling of pattern definition statements before the early exit for statements that aren't live or relevant. * tree-vect-loop.c (vect_transform_loop_stmt): New function, split out from... (vect_transform_loop): ...here. Process pattern definition statements without first checking whether the main pattern statement is live or relevant. Index: gcc/tree-vect-stmts.c =================================================================== *** gcc/tree-vect-stmts.c 2018-06-18 15:50:55.607303415 +0100 --- gcc/tree-vect-stmts.c 2018-06-18 15:50:55.603303451 +0100 *************** vect_analyze_stmt (gimple *stmt, bool *n *** 9409,9414 **** --- 9409,9442 ---- return false; } + if (STMT_VINFO_IN_PATTERN_P (stmt_info) + && node == NULL + && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info))) + { + gimple_stmt_iterator si; + + for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si)) + { + gimple *pattern_def_stmt = gsi_stmt (si); + if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt)) + || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt))) + { + /* Analyze def stmt of STMT if it's a pattern stmt. */ + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_NOTE, vect_location, + "==> examining pattern def statement: "); + dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0); + } + + if (!vect_analyze_stmt (pattern_def_stmt, + need_to_vectorize, node, node_instance, + cost_vec)) + return false; + } + } + } + /* Skip stmts that do not need to be vectorized. In loops this is expected to include: - the COND_EXPR which is the loop exit condition *************** vect_analyze_stmt (gimple *stmt, bool *n *** 9469,9502 **** return false; } - if (is_pattern_stmt_p (stmt_info) - && node == NULL - && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info))) - { - gimple_stmt_iterator si; - - for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si)) - { - gimple *pattern_def_stmt = gsi_stmt (si); - if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt)) - || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt))) - { - /* Analyze def stmt of STMT if it's a pattern stmt. */ - if (dump_enabled_p ()) - { - dump_printf_loc (MSG_NOTE, vect_location, - "==> examining pattern def statement: "); - dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0); - } - - if (!vect_analyze_stmt (pattern_def_stmt, - need_to_vectorize, node, node_instance, - cost_vec)) - return false; - } - } - } - switch (STMT_VINFO_DEF_TYPE (stmt_info)) { case vect_internal_def: --- 9497,9502 ---- Index: gcc/tree-vect-loop.c =================================================================== *** gcc/tree-vect-loop.c 2018-06-18 15:50:55.607303415 +0100 --- gcc/tree-vect-loop.c 2018-06-18 15:50:55.603303451 +0100 *************** scale_profile_for_vect_loop (struct loop *** 8306,8311 **** --- 8306,8381 ---- scale_bbs_frequencies (&loop->latch, 1, exit_l->probability / prob); } + /* Vectorize STMT if relevant, inserting any new instructions before GSI. + When vectorizing STMT as a store, set *SEEN_STORE to its stmt_vec_info. + *SLP_SCHEDULE is a running record of whether we have called + vect_schedule_slp. */ + + static void + vect_transform_loop_stmt (loop_vec_info loop_vinfo, gimple *stmt, + gimple_stmt_iterator *gsi, + stmt_vec_info *seen_store, bool *slp_scheduled) + { + struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); + poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + if (!stmt_info) + return; + + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_NOTE, vect_location, + "------>vectorizing statement: "); + dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0); + } + + if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info)) + vect_loop_kill_debug_uses (loop, stmt); + + if (!STMT_VINFO_RELEVANT_P (stmt_info) + && !STMT_VINFO_LIVE_P (stmt_info)) + return; + + if (STMT_VINFO_VECTYPE (stmt_info)) + { + poly_uint64 nunits + = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info)); + if (!STMT_SLP_TYPE (stmt_info) + && maybe_ne (nunits, vf) + && dump_enabled_p ()) + /* For SLP VF is set according to unrolling factor, and not + to vector size, hence for SLP this print is not valid. */ + dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n"); + } + + /* SLP. Schedule all the SLP instances when the first SLP stmt is + reached. */ + if (STMT_SLP_TYPE (stmt_info)) + { + if (!*slp_scheduled) + { + *slp_scheduled = true; + + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "=== scheduling SLP instances ===\n"); + + vect_schedule_slp (loop_vinfo); + } + + /* Hybrid SLP stmts must be vectorized in addition to SLP. */ + if (PURE_SLP_STMT (stmt_info)) + return; + } + + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n"); + + bool grouped_store = false; + if (vect_transform_stmt (stmt, gsi, &grouped_store, NULL, NULL)) + *seen_store = stmt_info; + } + /* Function vect_transform_loop. The analysis phase has determined that the loop is vectorizable. *************** vect_transform_loop (loop_vec_info loop_ *** 8326,8337 **** tree niters_vector_mult_vf = NULL_TREE; poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); unsigned int lowest_vf = constant_lower_bound (vf); - bool grouped_store; bool slp_scheduled = false; ! gimple *stmt, *pattern_stmt; ! gimple_seq pattern_def_seq = NULL; ! gimple_stmt_iterator pattern_def_si = gsi_none (); ! bool transform_pattern_stmt = false; bool check_profitability = false; unsigned int th; --- 8396,8403 ---- tree niters_vector_mult_vf = NULL_TREE; poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); unsigned int lowest_vf = constant_lower_bound (vf); bool slp_scheduled = false; ! gimple *stmt; bool check_profitability = false; unsigned int th; *************** vect_transform_loop (loop_vec_info loop_ *** 8483,8678 **** } } - pattern_stmt = NULL; for (gimple_stmt_iterator si = gsi_start_bb (bb); ! !gsi_end_p (si) || transform_pattern_stmt;) { ! bool is_store; ! ! if (transform_pattern_stmt) ! stmt = pattern_stmt; ! else ! { ! stmt = gsi_stmt (si); ! /* During vectorization remove existing clobber stmts. */ ! if (gimple_clobber_p (stmt)) ! { ! unlink_stmt_vdef (stmt); ! gsi_remove (&si, true); ! release_defs (stmt); ! continue; ! } ! } ! ! if (dump_enabled_p ()) { ! dump_printf_loc (MSG_NOTE, vect_location, ! "------>vectorizing statement: "); ! dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0); } ! ! stmt_info = vinfo_for_stmt (stmt); ! ! /* vector stmts created in the outer-loop during vectorization of ! stmts in an inner-loop may not have a stmt_info, and do not ! need to be vectorized. */ ! if (!stmt_info) { ! gsi_next (&si); ! continue; ! } ! if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info)) ! vect_loop_kill_debug_uses (loop, stmt); ! ! if (!STMT_VINFO_RELEVANT_P (stmt_info) ! && !STMT_VINFO_LIVE_P (stmt_info)) ! { ! if (STMT_VINFO_IN_PATTERN_P (stmt_info) ! && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info)) ! && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt)) ! || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt)))) ! { ! stmt = pattern_stmt; ! stmt_info = vinfo_for_stmt (stmt); ! } ! else ! { ! gsi_next (&si); ! continue; ! } ! } ! else if (STMT_VINFO_IN_PATTERN_P (stmt_info) ! && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info)) ! && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt)) ! || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt)))) ! transform_pattern_stmt = true; ! ! /* If pattern statement has def stmts, vectorize them too. */ ! if (is_pattern_stmt_p (stmt_info)) ! { ! if (pattern_def_seq == NULL) ! { ! pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info); ! pattern_def_si = gsi_start (pattern_def_seq); ! } ! else if (!gsi_end_p (pattern_def_si)) ! gsi_next (&pattern_def_si); ! if (pattern_def_seq != NULL) { ! gimple *pattern_def_stmt = NULL; ! stmt_vec_info pattern_def_stmt_info = NULL; ! ! while (!gsi_end_p (pattern_def_si)) ! { ! pattern_def_stmt = gsi_stmt (pattern_def_si); ! pattern_def_stmt_info ! = vinfo_for_stmt (pattern_def_stmt); ! if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info) ! || STMT_VINFO_LIVE_P (pattern_def_stmt_info)) ! break; ! gsi_next (&pattern_def_si); ! } ! ! if (!gsi_end_p (pattern_def_si)) { ! if (dump_enabled_p ()) ! { ! dump_printf_loc (MSG_NOTE, vect_location, ! "==> vectorizing pattern def " ! "stmt: "); ! dump_gimple_stmt (MSG_NOTE, TDF_SLIM, ! pattern_def_stmt, 0); ! } ! ! stmt = pattern_def_stmt; ! stmt_info = pattern_def_stmt_info; } ! else ! { ! pattern_def_si = gsi_none (); ! transform_pattern_stmt = false; ! } ! } ! else ! transform_pattern_stmt = false; ! } ! ! if (STMT_VINFO_VECTYPE (stmt_info)) ! { ! poly_uint64 nunits ! = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info)); ! if (!STMT_SLP_TYPE (stmt_info) ! && maybe_ne (nunits, vf) ! && dump_enabled_p ()) ! /* For SLP VF is set according to unrolling factor, and not ! to vector size, hence for SLP this print is not valid. */ ! dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n"); ! } ! ! /* SLP. Schedule all the SLP instances when the first SLP stmt is ! reached. */ ! if (STMT_SLP_TYPE (stmt_info)) ! { ! if (!slp_scheduled) ! { ! slp_scheduled = true; ! ! if (dump_enabled_p ()) ! dump_printf_loc (MSG_NOTE, vect_location, ! "=== scheduling SLP instances ===\n"); ! ! vect_schedule_slp (loop_vinfo); } ! ! /* Hybrid SLP stmts must be vectorized in addition to SLP. */ ! if (!vinfo_for_stmt (stmt) || PURE_SLP_STMT (stmt_info)) { ! if (!transform_pattern_stmt && gsi_end_p (pattern_def_si)) { ! pattern_def_seq = NULL; gsi_next (&si); } - continue; - } - } - - /* -------- vectorize statement ------------ */ - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n"); - - grouped_store = false; - is_store = vect_transform_stmt (stmt, &si, &grouped_store, NULL, NULL); - if (is_store) - { - if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) - { - /* Interleaving. If IS_STORE is TRUE, the vectorization of the - interleaving chain was completed - free all the stores in - the chain. */ - gsi_next (&si); - vect_remove_stores (DR_GROUP_FIRST_ELEMENT (stmt_info)); } else ! { ! /* Free the attached stmt_vec_info and remove the stmt. */ ! gimple *store = gsi_stmt (si); ! free_stmt_vec_info (store); ! unlink_stmt_vdef (store); ! gsi_remove (&si, true); ! release_defs (store); ! } ! ! /* Stores can only appear at the end of pattern statements. */ ! gcc_assert (!transform_pattern_stmt); ! pattern_def_seq = NULL; } ! else if (!transform_pattern_stmt && gsi_end_p (pattern_def_si)) ! { ! pattern_def_seq = NULL; ! gsi_next (&si); ! } ! } /* stmts in BB */ /* Stub out scalar statements that must not survive vectorization. Doing this here helps with grouped statements, or statements that --- 8549,8615 ---- } } for (gimple_stmt_iterator si = gsi_start_bb (bb); ! !gsi_end_p (si);) { ! stmt = gsi_stmt (si); ! /* During vectorization remove existing clobber stmts. */ ! if (gimple_clobber_p (stmt)) { ! unlink_stmt_vdef (stmt); ! gsi_remove (&si, true); ! release_defs (stmt); } ! else { ! stmt_info = vinfo_for_stmt (stmt); ! /* vector stmts created in the outer-loop during vectorization of ! stmts in an inner-loop may not have a stmt_info, and do not ! need to be vectorized. */ ! stmt_vec_info seen_store = NULL; ! if (stmt_info) { ! if (STMT_VINFO_IN_PATTERN_P (stmt_info)) { ! gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info); ! for (gimple_stmt_iterator subsi = gsi_start (def_seq); ! !gsi_end_p (subsi); gsi_next (&subsi)) ! vect_transform_loop_stmt (loop_vinfo, ! gsi_stmt (subsi), &si, ! &seen_store, ! &slp_scheduled); ! gimple *pat_stmt = STMT_VINFO_RELATED_STMT (stmt_info); ! vect_transform_loop_stmt (loop_vinfo, pat_stmt, &si, ! &seen_store, &slp_scheduled); } ! vect_transform_loop_stmt (loop_vinfo, stmt, &si, ! &seen_store, &slp_scheduled); } ! if (seen_store) { ! if (STMT_VINFO_GROUPED_ACCESS (seen_store)) { ! /* Interleaving. If IS_STORE is TRUE, the ! vectorization of the interleaving chain was ! completed - free all the stores in the chain. */ gsi_next (&si); + vect_remove_stores (DR_GROUP_FIRST_ELEMENT (seen_store)); + } + else + { + /* Free the attached stmt_vec_info and remove the + stmt. */ + free_stmt_vec_info (stmt); + unlink_stmt_vdef (stmt); + gsi_remove (&si, true); + release_defs (stmt); } } else ! gsi_next (&si); } ! } /* Stub out scalar statements that must not survive vectorization. Doing this here helps with grouped statements, or statements that