https://gcc.gnu.org/g:d2e1b1248a17d0463bc755b281b6c0ac1e0f8362
commit r17-5047-gd2e1b1248a17d0463bc755b281b6c0ac1e0f8362 Author: Stefan Schulze Frielinghaus <[email protected]> Date: Sat Oct 10 09:51:30 2026 +0200 vect: Defined-at-zero value for .CLZ/.CTZ The defined-at-zero value may differ between scalar and vector modes. Even if the bit precision of a scalar and a vector lane coincide, the defined-at-zero value may differ. Therefore, during vect-patterns, consider the defined-at-zero value for the natural vector mode instead of a scalar mode since this is closer to the final vector mode. However, since in theory an at-zero value could differ for the natural and final vector mode, do a final check during vectorizable_call. gcc/ChangeLog: * config/i386/i386.h (CLZ_DEFINED_VALUE_AT_ZERO): Utilize GET_MODE_UNIT_BITSIZE instead of GET_MODE_BITSIZE in order to deal with vector modes. (CTZ_DEFINED_VALUE_AT_ZERO): Ditto. * config/rs6000/rs6000.h (CLZ_DEFINED_VALUE_AT_ZERO): Ditto. (CTZ_DEFINED_VALUE_AT_ZERO): Ditto. * tree-vect-patterns.cc (vect_recog_ctz_ffs_pattern): Use defined zero value for vector modes. (vect_recog_popcount_clz_ctz_ffs_pattern): Test for a defined zero value for vector modes. * tree-vect-stmts.cc (vectorizable_call): Bail out if natural and final vector mode do not coincide w.r.t. defined-at-zero value. Diff: --- gcc/config/i386/i386.h | 4 ++-- gcc/config/rs6000/rs6000.h | 4 ++-- gcc/tree-vect-patterns.cc | 22 ++++++++++------------ gcc/tree-vect-stmts.cc | 13 +++++++++++++ 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h index 47725c560906..35563a6144ac 100644 --- a/gcc/config/i386/i386.h +++ b/gcc/config/i386/i386.h @@ -3060,9 +3060,9 @@ extern void debug_dispatch_window (int); /* The value at zero is only defined for the BMI instructions LZCNT and TZCNT, not the BSR/BSF insns in the original isa. */ #define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \ - ((VALUE) = GET_MODE_BITSIZE (MODE), TARGET_BMI ? 2 : 0) + ((VALUE) = GET_MODE_UNIT_BITSIZE (MODE), TARGET_BMI ? 2 : 0) #define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \ - ((VALUE) = GET_MODE_BITSIZE (MODE), TARGET_LZCNT ? 2 : 0) + ((VALUE) = GET_MODE_UNIT_BITSIZE (MODE), TARGET_LZCNT ? 2 : 0) /* Flags returned by ix86_get_callcvt (). */ diff --git a/gcc/config/rs6000/rs6000.h b/gcc/config/rs6000/rs6000.h index bcd4c39d24dc..98952f111892 100644 --- a/gcc/config/rs6000/rs6000.h +++ b/gcc/config/rs6000/rs6000.h @@ -1749,14 +1749,14 @@ typedef struct rs6000_args /* The cntlzw and cntlzd instructions return 32 and 64 for input of zero. */ #define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \ - ((VALUE) = GET_MODE_BITSIZE (MODE), 2) + ((VALUE) = GET_MODE_UNIT_BITSIZE (MODE), 2) /* The CTZ patterns that are implemented in terms of CLZ return -1 for input of zero. The hardware instructions added in Power9 and the sequences using popcount return 32 or 64. */ #define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \ (TARGET_CTZ || TARGET_POPCNTD \ - ? ((VALUE) = GET_MODE_BITSIZE (MODE), 2) \ + ? ((VALUE) = GET_MODE_UNIT_BITSIZE (MODE), 2) \ : ((VALUE) = -1, 2)) /* Specify the machine mode that pointers have. diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc index 232c9620bc05..e9d2870feb41 100644 --- a/gcc/tree-vect-patterns.cc +++ b/gcc/tree-vect-patterns.cc @@ -1942,7 +1942,7 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo, { ifnnew = IFN_CTZ; defined_at_zero_new - = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type), + = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type), val_new) == 2; } else if (direct_internal_fn_supported_p (IFN_CLZ, vec_rhs_type, @@ -1950,7 +1950,7 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo, { ifnnew = IFN_CLZ; defined_at_zero_new - = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type), + = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type), val_new) == 2; } if ((ifnnew == IFN_LAST @@ -2243,6 +2243,10 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (lhs_type)) return NULL; + vec_type = get_vectype_for_scalar_type (vinfo, lhs_type); + if (!vec_type || !VECTOR_MODE_P (TYPE_MODE (vec_type))) + return NULL; + /* Also A should be unsigned or same precision as temp_in, otherwise different builtins/internal functions have different behaviors. */ if (TYPE_PRECISION (unprom_diff.type) @@ -2272,8 +2276,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, int val1, val2; val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1)); int d2 - = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type), - val2); + = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2); if (d2 != 2 || val1 != val2 + addend) return NULL; } @@ -2288,8 +2291,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, int val1, val2; val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1)); int d2 - = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type), - val2); + = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2); if (d2 != 2 || val1 != val2) return NULL; } @@ -2301,11 +2303,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, gcc_unreachable (); } - vec_type = get_vectype_for_scalar_type (vinfo, lhs_type); /* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */ - if (!vec_type) - return NULL; - bool supported = direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED); /* We cannot rely on the reduction check in vect_recog_ctz_ffs_pattern @@ -2358,11 +2356,11 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, tree arg2 = NULL_TREE; int val; if (ifn == IFN_CLZ - && CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type), + && CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val) == 2) arg2 = build_int_cst (integer_type_node, val); else if (ifn == IFN_CTZ - && CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type), + && CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val) == 2) arg2 = build_int_cst (integer_type_node, val); if (arg2) diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index 8069ba8a6f25..785dac775385 100644 --- a/gcc/tree-vect-stmts.cc +++ b/gcc/tree-vect-stmts.cc @@ -3555,6 +3555,19 @@ vectorizable_call (vec_info *vinfo, return false; } + /* During vect-patterns we remembered the defined-at-zero value in the + optional second argument for the natural vector mode. Now, once we have + the final vector mode, verify that they coincide. If not, bail out. */ + if (clz_ctz_arg1) + { + int val; + int d = (cfn == CFN_CLZ + ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vectype_in), val) + : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vectype_in), val)); + if (d != 2 || val != tree_to_shwi (clz_ctz_arg1)) + return false; + } + if (VECTOR_BOOLEAN_TYPE_P (vectype_out) != VECTOR_BOOLEAN_TYPE_P (vectype_in)) {
