https://gcc.gnu.org/g:c0f31701556c4162463f28bc0f03007f40a6176e
commit r15-961-gc0f31701556c4162463f28bc0f03007f40a6176e Author: Feng Xue <f...@os.amperecomputing.com> Date: Wed May 29 13:12:12 2024 +0800 vect: Add a function to check lane-reducing code Check if an operation is lane-reducing requires comparison of code against three kinds (DOT_PROD_EXPR/WIDEN_SUM_EXPR/SAD_EXPR). Add an utility function to make source coding for the check handy and concise. 2024-05-29 Feng Xue <f...@os.amperecomputing.com> gcc/ * tree-vectorizer.h (lane_reducing_op_p): New function. * tree-vect-slp.cc (vect_analyze_slp): Use new function lane_reducing_op_p to check statement code. * tree-vect-loop.cc (vect_transform_reduction): Likewise. (vectorizable_reduction): Likewise, and change name of a local variable that holds the result flag. Diff: --- gcc/tree-vect-loop.cc | 29 ++++++++++++----------------- gcc/tree-vect-slp.cc | 4 +--- gcc/tree-vectorizer.h | 6 ++++++ 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 04a9ac64df7..a42d79c7cbf 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -7650,9 +7650,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, gimple_match_op op; if (!gimple_extract_op (stmt_info->stmt, &op)) gcc_unreachable (); - bool lane_reduc_code_p = (op.code == DOT_PROD_EXPR - || op.code == WIDEN_SUM_EXPR - || op.code == SAD_EXPR); + bool lane_reducing = lane_reducing_op_p (op.code); if (!POINTER_TYPE_P (op.type) && !INTEGRAL_TYPE_P (op.type) && !SCALAR_FLOAT_TYPE_P (op.type)) @@ -7664,7 +7662,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, /* For lane-reducing ops we're reducing the number of reduction PHIs which means the only use of that may be in the lane-reducing operation. */ - if (lane_reduc_code_p + if (lane_reducing && reduc_chain_length != 1 && !only_slp_reduc_chain) { @@ -7678,7 +7676,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, since we'll mix lanes belonging to different reductions. But it's OK to use them in a reduction chain or when the reduction group has just one element. */ - if (lane_reduc_code_p + if (lane_reducing && slp_node && !REDUC_GROUP_FIRST_ELEMENT (stmt_info) && SLP_TREE_LANES (slp_node) > 1) @@ -7738,7 +7736,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, /* To properly compute ncopies we are interested in the widest non-reduction input type in case we're looking at a widening accumulation that we later handle in vect_transform_reduction. */ - if (lane_reduc_code_p + if (lane_reducing && vectype_op[i] && (!vectype_in || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in))) @@ -8211,7 +8209,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, && loop_vinfo->suggested_unroll_factor == 1) single_defuse_cycle = true; - if (single_defuse_cycle || lane_reduc_code_p) + if (single_defuse_cycle || lane_reducing) { gcc_assert (op.code != COND_EXPR); @@ -8227,7 +8225,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, mixed-sign dot-products can be implemented using signed dot-products. */ machine_mode vec_mode = TYPE_MODE (vectype_in); - if (!lane_reduc_code_p + if (!lane_reducing && !directly_supported_p (op.code, vectype_in, optab_vector)) { if (dump_enabled_p ()) @@ -8252,7 +8250,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, For the other cases try without the single cycle optimization. */ if (!ok) { - if (lane_reduc_code_p) + if (lane_reducing) return false; else single_defuse_cycle = false; @@ -8263,7 +8261,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, /* If the reduction stmt is one of the patterns that have lane reduction embedded we cannot handle the case of ! single_defuse_cycle. */ if ((ncopies > 1 && ! single_defuse_cycle) - && lane_reduc_code_p) + && lane_reducing) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -8274,7 +8272,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, if (slp_node && !(!single_defuse_cycle - && !lane_reduc_code_p + && !lane_reducing && reduction_type != FOLD_LEFT_REDUCTION)) for (i = 0; i < (int) op.num_ops; i++) if (!vect_maybe_update_slp_op_vectype (slp_op[i], vectype_op[i])) @@ -8295,7 +8293,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, /* Cost the reduction op inside the loop if transformed via vect_transform_reduction. Otherwise this is costed by the separate vectorizable_* routines. */ - if (single_defuse_cycle || lane_reduc_code_p) + if (single_defuse_cycle || lane_reducing) { int factor = 1; if (vect_is_emulated_mixed_dot_prod (loop_vinfo, stmt_info)) @@ -8313,7 +8311,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, /* All but single defuse-cycle optimized, lane-reducing and fold-left reductions go through their own vectorizable_* routines. */ if (!single_defuse_cycle - && !lane_reduc_code_p + && !lane_reducing && reduction_type != FOLD_LEFT_REDUCTION) { stmt_vec_info tem @@ -8555,10 +8553,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo, } bool single_defuse_cycle = STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info); - gcc_assert (single_defuse_cycle - || code == DOT_PROD_EXPR - || code == WIDEN_SUM_EXPR - || code == SAD_EXPR); + gcc_assert (single_defuse_cycle || lane_reducing_op_p (code)); /* Create the destination vector */ tree scalar_dest = gimple_get_lhs (stmt_info->stmt); diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc index bc7a85d6bfc..bf1f467f53f 100644 --- a/gcc/tree-vect-slp.cc +++ b/gcc/tree-vect-slp.cc @@ -3928,9 +3928,7 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size) /* Do not discover SLP reductions for lane-reducing ops, that will fail later. */ && (!(g = dyn_cast <gassign *> (STMT_VINFO_STMT (next_info))) - || (gimple_assign_rhs_code (g) != DOT_PROD_EXPR - && gimple_assign_rhs_code (g) != WIDEN_SUM_EXPR - && gimple_assign_rhs_code (g) != SAD_EXPR))) + || !lane_reducing_op_p (gimple_assign_rhs_code (g)))) scalar_stmts.quick_push (next_info); } if (scalar_stmts.length () > 1) diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 479823412fd..97ec9c341e7 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -2169,6 +2169,12 @@ vect_apply_runtime_profitability_check_p (loop_vec_info loop_vinfo) && th >= vect_vf_for_cost (loop_vinfo)); } +inline bool +lane_reducing_op_p (code_helper code) +{ + return code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code == SAD_EXPR; +} + /* Source location + hotness information. */ extern dump_user_location_t vect_location;