https://gcc.gnu.org/g:cf4e5c177f59ffefeccab911b4a1a9dc6040fd8e
commit r17-5219-gcf4e5c177f59ffefeccab911b4a1a9dc6040fd8e Author: Richard Sandiford <[email protected]> Date: Thu Oct 15 11:09:58 2026 +0100 aarch64: Move vops handling to gimple_folder::fold Many AArch64 builtins are said to modify memory, either because they genuinely do, or because they modify some other global state. This includes FP exceptions for -ftrapping-math. If we're folding a builtin call while in SSA form, the vdefs for the old call must be copied over to the replacement statements. If the new statements do not modify memory, the vdef must instead be removed and its users updated. gimple_folder::fold_to_stmt_vops was added to cope with this. However, it's easy to forget that even FP arithmetic might need to go down this route. This patch therefore moves the responsibility for handling vops to the main fold routine instead. It also copes with folds that happen before we're in SSA form, where trying to create vops would lead to an ICE. The ACLE folders currently insert any extra statements using the statement iterator. However, gsi_replace_with_seq_vops works best if it sees the full sequence, rather than just the main replacement statement. The patch therefore makes the folders add statements to a new sequence instead, at least when adding before the call. The overall effect seems simpler and should hopefully be more robust. However, the main motivation is that the patch fixes an ICE when vadd_f32 is lowered while in SSA form with -ftrapping-math in effect. In a sense, all the CP_RAISE_EXCEPTIONS stuff is wasted on vadd_f32. We usually just lower to a gimple PLUS_EXPR regardless, before anything really has a chance to act on the CP_RAISE_EXCEPTIONS. That is, we're dropping any pretence that vadd_f32 is a way of accessing architecture behaviour and simply treating vadd_f32 as a long-winded way of writing C +, with C rather than ISA semantics. But that's a different story :) And the situation with svadd_f32 isn't exactly better. gcc/ * config/aarch64/aarch64-acle-builtins.h (gimple_folder::force_vector): Drop the stmts parameter. (gimple_folder::convert_pred): Likewise. (gimple_folder::fold_contiguous_base): Likewise. (gimple_folder::fold_to_stmt_vops): Delete. (gimple_folder::stmts): New member variable. * config/aarch64/aarch64-acle-builtins.cc: Include gimple-ssa.h. (gimple_folder::gimple_folder): Initialize stmts. (gimple_folder::force_vector): Drop the stmts parameter and instead use the member variable of the same name. (gimple_folder::convert_pred): Likewise. (gimple_folder::fold_contiguous_base): Likewise. Update callers to the above. (gimple_folder::convert_and_fold): Add statements to the stmts member variable. Don't insert them into the IL here. (gimple_folder::fold_pfalse): Drop use of fold_to_stmt_vops. (gimple_folder::fold_call_to): Likewise. (gimple_folder::fold_to_stmt_vops): Delete. (gimple_folder::fold): Handle the new stmts member variable. Use gsi_replace_with_seq_vops if we're replacing a call while in SSA form. (gimple_folder::force_val): Add to the gimple sequence instead of inserting directly into the IL. (gimple_folder::assign): Likewise. * config/aarch64/aarch64-neon-builtins-base.cc (gimple_ifn::fold): Likewise. * config/aarch64/aarch64-sve-builtins-base.cc (svdupq_impl::fold): Likewise. Remove direct use of gsi_replace_with_seq. (svld1rq_impl::fold): Likewise gsi_replace_with_seq_vops. (svcmp_impl::fold, svdup_impl::fold, svld234_impl::fold) (svsel_impl::fold, svst1_impl::fold, svst234_impl::fold): Update calls to convert_pred, force_vector, and fold_contiguous_base. Diff: --- gcc/config/aarch64/aarch64-acle-builtins.cc | 76 +++++++++++------------- gcc/config/aarch64/aarch64-acle-builtins.h | 11 ++-- gcc/config/aarch64/aarch64-neon-builtins-base.cc | 2 +- gcc/config/aarch64/aarch64-sve-builtins-base.cc | 61 ++++++------------- 4 files changed, 62 insertions(+), 88 deletions(-) diff --git a/gcc/config/aarch64/aarch64-acle-builtins.cc b/gcc/config/aarch64/aarch64-acle-builtins.cc index 2e4632bcb315..2d52443a68ca 100644 --- a/gcc/config/aarch64/aarch64-acle-builtins.cc +++ b/gcc/config/aarch64/aarch64-acle-builtins.cc @@ -49,6 +49,7 @@ #include "attribs.h" #include "value-range.h" #include "tree-ssanames.h" +#include "gimple-ssa.h" #include "aarch64-acle-builtins.h" #include "aarch64-sve-builtins-base.h" #include "aarch64-sve-builtins-sve2.h" @@ -2803,15 +2804,15 @@ is_pfalse (tree v) gimple_folder::gimple_folder (const function_instance &instance, tree fndecl, gimple_stmt_iterator *gsi_in, gcall *call_in) : function_call_info (gimple_location (call_in), instance, fndecl), - gsi (gsi_in), call (call_in), lhs (gimple_call_lhs (call_in)) + gsi (gsi_in), call (call_in), lhs (gimple_call_lhs (call_in)), + stmts (nullptr) { } /* VALUE might be a vector of type VECTYPE or a single scalar element. - Duplicate it into a vector of type VECTYPE in the latter case, adding any - new statements to STMTS. */ + Duplicate it into a vector of type VECTYPE in the latter case. */ tree -gimple_folder::force_vector (gimple_seq &stmts, tree vectype, tree value) +gimple_folder::force_vector (tree vectype, tree value) { if (!VECTOR_TYPE_P (TREE_TYPE (value))) value = gimple_build_vector_from_val (&stmts, vectype, value); @@ -2819,10 +2820,9 @@ gimple_folder::force_vector (gimple_seq &stmts, tree vectype, tree value) } /* Convert predicate argument ARGNO so that it has the type appropriate for - an operation on VECTYPE. Add any new statements to STMTS. */ + an operation on VECTYPE. */ tree -gimple_folder::convert_pred (gimple_seq &stmts, tree vectype, - unsigned int argno) +gimple_folder::convert_pred (tree vectype, unsigned int argno) { tree pred = gimple_call_arg (call, argno); if (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (pred)), @@ -2834,10 +2834,9 @@ gimple_folder::convert_pred (gimple_seq &stmts, tree vectype, } /* Return a pointer to the address in a contiguous load or store, - given that each memory vector has type VECTYPE. Add any new - statements to STMTS. */ + given that each memory vector has type VECTYPE. */ tree -gimple_folder::fold_contiguous_base (gimple_seq &stmts, tree vectype) +gimple_folder::fold_contiguous_base (tree vectype) { tree base = gimple_call_arg (call, 1); if (mode_suffix_id == MODE_vnum) @@ -2863,11 +2862,9 @@ gimple_folder::fold_contiguous_load (internal_fn ifn_code) tree vectype = vector_type (0); /* Get the predicate and base pointer. */ - gimple_seq stmts = NULL; - tree pred = convert_pred (stmts, vectype, 0); - tree base = fold_contiguous_base (stmts, vectype); + tree pred = convert_pred (vectype, 0); + tree base = fold_contiguous_base (vectype); tree els = build_zero_cst (vectype); - gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); tree cookie = load_store_cookie (TREE_TYPE (vectype)); gcall *new_call = gimple_build_call_internal (ifn_code, 4, @@ -2963,7 +2960,7 @@ gimple_folder::fold_pfalse () && !AGGREGATE_TYPE_P (TREE_TYPE (lhs))) return fold_call_to (build_zero_cst (TREE_TYPE (lhs))); if (flags & (CP_WRITE_MEMORY | CP_PREFETCH_MEMORY)) - return fold_to_stmt_vops (gimple_build_nop ()); + return gimple_build_nop (); } return nullptr; } @@ -2981,7 +2978,6 @@ gimple_folder::convert_and_fold (tree type, gcc_assert (VECTOR_TYPE_P (type) && TYPE_MODE (type) != VNx16BImode); tree old_ty = TREE_TYPE (lhs); - gimple_seq stmts = NULL; bool convert_lhs_p = !useless_type_conversion_p (type, old_ty); tree lhs_conv = convert_lhs_p ? make_ssa_name (type) : lhs; unsigned int num_args = gimple_call_num_args (call); @@ -2999,7 +2995,6 @@ gimple_folder::convert_and_fold (tree type, } gimple *new_stmt = fp (*this, lhs_conv, args_conv); - gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); if (convert_lhs_p) { tree t = build1 (VIEW_CONVERT_EXPR, old_ty, lhs_conv); @@ -3101,10 +3096,8 @@ gimple_folder::fold_active_lanes_to (tree x) if (operand_equal_p (x, vec_inactive, 0)) return gimple_build_assign (lhs, x); - gimple_seq stmts = NULL; - tree pred = convert_pred (stmts, vector_type (0), 0); - x = force_vector (stmts, TREE_TYPE (lhs), x); - gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); + tree pred = convert_pred (vector_type (0), 0); + x = force_vector (TREE_TYPE (lhs), x); return gimple_build_assign (lhs, VEC_COND_EXPR, pred, x, vec_inactive); } @@ -3113,20 +3106,10 @@ gimple * gimple_folder::fold_call_to (tree t) { if (types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (t))) - return fold_to_stmt_vops (gimple_build_assign (lhs, t)); + return gimple_build_assign (lhs, t); tree rhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), t); - return fold_to_stmt_vops (gimple_build_assign (lhs, VIEW_CONVERT_EXPR, rhs)); -} - -/* Fold call to G, incl. adjustments to the virtual operands. */ -gimple * -gimple_folder::fold_to_stmt_vops (gimple *g) -{ - gimple_seq stmts = NULL; - gimple_seq_add_stmt_without_update (&stmts, g); - gsi_replace_with_seq_vops (gsi, stmts); - return g; + return gimple_build_assign (lhs, VIEW_CONVERT_EXPR, rhs); } /* Try to fold the call. Return the new statement on success and null @@ -3146,12 +3129,25 @@ gimple_folder::fold () lhs = create_tmp_var (TREE_TYPE (gimple_call_fntype (call))); /* First try some simplifications that are common to many functions. */ - if (auto *call = redirect_pred_x ()) - return call; - if (auto *call = fold_pfalse ()) - return call; + gimple *res = redirect_pred_x (); + if (!res) + res = fold_pfalse (); + if (!res) + res = base->fold (*this); - return base->fold (*this); + if (res == call) + /* Folds can only reuse the existing call if there are no extra vops + to worry about. */ + gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); + else if (res) + { + gimple_seq_add_stmt_without_update (&stmts, res); + if (gimple_in_ssa_p (cfun)) + gsi_replace_with_seq_vops (gsi, stmts); + else + gsi_replace_with_seq (gsi, stmts, true); + } + return res; } /* Force EXPR to be a GIMPLE value, putting it into a new variable if @@ -3165,7 +3161,7 @@ gimple_folder::force_val (tree expr) auto type = TREE_TYPE (expr); auto var = create_tmp_var (type); auto stmt = gimple_build_assign (var, expr); - gsi_insert_before (this->gsi, stmt, GSI_SAME_STMT); + gimple_seq_add_stmt_without_update (&stmts, stmt); return var; } @@ -3175,7 +3171,7 @@ gassign * gimple_folder::assign (tree lhs, tree rhs) { auto stmt = gimple_build_assign (lhs, rhs); - gsi_insert_before (this->gsi, stmt, GSI_SAME_STMT); + gimple_seq_add_stmt_without_update (&stmts, stmt); return stmt; } diff --git a/gcc/config/aarch64/aarch64-acle-builtins.h b/gcc/config/aarch64/aarch64-acle-builtins.h index ca0f53d70865..1572594872b9 100644 --- a/gcc/config/aarch64/aarch64-acle-builtins.h +++ b/gcc/config/aarch64/aarch64-acle-builtins.h @@ -647,9 +647,9 @@ public: gimple_folder (const function_instance &, tree, gimple_stmt_iterator *, gcall *); - tree force_vector (gimple_seq &, tree, tree); - tree convert_pred (gimple_seq &, tree, unsigned int); - tree fold_contiguous_base (gimple_seq &, tree); + tree force_vector (tree, tree); + tree convert_pred (tree, unsigned int); + tree fold_contiguous_base (tree); gimple *fold_contiguous_load (internal_fn); tree load_store_cookie (tree); @@ -669,7 +669,6 @@ public: gimple *fold_const_binary (enum tree_code); gimple *fold_active_lanes_to (tree); gimple *fold_call_to (tree); - gimple *fold_to_stmt_vops (gimple *); gimple *fold (); @@ -681,6 +680,10 @@ public: /* The result of the call, or null if none. */ tree lhs; + + /* The sequence of statements that should be emitted before the original + call. */ + gimple_seq stmts; }; /* A class for expanding a function call into RTL. */ diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.cc b/gcc/config/aarch64/aarch64-neon-builtins-base.cc index fa056998ae6a..b00aed7c7d77 100644 --- a/gcc/config/aarch64/aarch64-neon-builtins-base.cc +++ b/gcc/config/aarch64/aarch64-neon-builtins-base.cc @@ -553,7 +553,7 @@ public: /* Temporary scalar to hold the result. */ tree res = create_tmp_var (TREE_TYPE (lhs_type)); gimple_call_set_lhs (call, res); - gsi_insert_before (f.gsi, call, GSI_SAME_STMT); + gimple_seq_add_stmt_without_update (&f.stmts, call); /* Build the actual vector that the call's result gets assigned to. */ return gimple_build_assign (f.lhs, build_vec_dup (lhs_type, res)); } diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc b/gcc/config/aarch64/aarch64-sve-builtins-base.cc index 39e7109abd85..ebbd58a9b391 100644 --- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc +++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc @@ -506,9 +506,7 @@ public: && f.type_suffix (0).element_bytes == 1 && f.type_suffix (0).integer_p) { - gimple_seq stmts = NULL; - rhs2 = f.force_vector (stmts, TREE_TYPE (rhs1), rhs2); - gsi_insert_seq_before (f.gsi, stmts, GSI_SAME_STMT); + rhs2 = f.force_vector (TREE_TYPE (rhs1), rhs2); return gimple_build_assign (f.lhs, m_code, rhs1, rhs2); } @@ -1065,10 +1063,8 @@ public: /* svdup_z (pg, x) == VEC_COND_EXPR <pg, VEC_DUPLICATE_EXPR <x>, 0>. */ if (f.pred == PRED_z) { - gimple_seq stmts = NULL; - tree pred = f.convert_pred (stmts, vec_type, 0); - rhs = f.force_vector (stmts, vec_type, rhs); - gsi_insert_seq_before (f.gsi, stmts, GSI_SAME_STMT); + tree pred = f.convert_pred (vec_type, 0); + rhs = f.force_vector (vec_type, rhs); return gimple_build_assign (f.lhs, VEC_COND_EXPR, pred, rhs, build_zero_cst (vec_type)); } @@ -1165,8 +1161,7 @@ private: tree tmp = make_ssa_name_fn (cfun, vq_type, 0); gimple *g = gimple_build_assign (tmp, vec); - gimple_seq stmts = NULL; - gimple_seq_add_stmt_without_update (&stmts, g); + gimple_seq_add_stmt_without_update (&f.stmts, g); poly_uint64 lhs_len = TYPE_VECTOR_SUBPARTS (lhs_type); vec_perm_builder sel (lhs_len, nargs, 1); @@ -1177,10 +1172,7 @@ private: tree mask_type = build_vector_type (ssizetype, lhs_len); tree mask = vec_perm_indices_to_tree (mask_type, indices); - gimple *g2 = gimple_build_assign (lhs, VEC_PERM_EXPR, tmp, tmp, mask); - gimple_seq_add_stmt_without_update (&stmts, g2); - gsi_replace_with_seq (f.gsi, stmts, false); - return g2; + return gimple_build_assign (lhs, VEC_PERM_EXPR, tmp, tmp, mask); } public: @@ -1889,8 +1881,7 @@ public: gimple *mem_ref_stmt = gimple_build_assign (mem_ref_lhs, mem_ref_op); - gimple_seq stmts = NULL; - gimple_seq_add_stmt_without_update (&stmts, mem_ref_stmt); + gimple_seq_add_stmt_without_update (&f.stmts, mem_ref_stmt); int source_nelts = TYPE_VECTOR_SUBPARTS (access_type).to_constant (); @@ -1899,14 +1890,9 @@ public: register and no replication is needed. Just convert the loaded value from the Advanced SIMD type to the SVE type. */ if (known_eq (lhs_len, (unsigned int) source_nelts)) - { - gimple *g - = gimple_build_assign (lhs, build1 (VIEW_CONVERT_EXPR, - lhs_type, mem_ref_lhs)); - gimple_seq_add_stmt_without_update (&stmts, g); - gsi_replace_with_seq_vops (f.gsi, stmts); - return g; - } + return gimple_build_assign (lhs, build1 (VIEW_CONVERT_EXPR, + lhs_type, mem_ref_lhs)); + vec_perm_builder sel (lhs_len, source_nelts, 1); for (int i = 0; i < source_nelts; i++) sel.quick_push (i); @@ -1917,11 +1903,8 @@ public: indices)); tree mask_type = build_vector_type (ssizetype, lhs_len); tree mask = vec_perm_indices_to_tree (mask_type, indices); - gimple *g2 = gimple_build_assign (lhs, VEC_PERM_EXPR, - mem_ref_lhs, mem_ref_lhs, mask); - gimple_seq_add_stmt_without_update (&stmts, g2); - gsi_replace_with_seq_vops (f.gsi, stmts); - return g2; + return gimple_build_assign (lhs, VEC_PERM_EXPR, + mem_ref_lhs, mem_ref_lhs, mask); } return NULL; @@ -1964,11 +1947,9 @@ public: tree vectype = f.vector_type (0); /* Get the predicate and base pointer. */ - gimple_seq stmts = NULL; - tree pred = f.convert_pred (stmts, vectype, 0); + tree pred = f.convert_pred (vectype, 0); tree els = build_zero_cst (vectype); - tree base = f.fold_contiguous_base (stmts, vectype); - gsi_insert_seq_before (f.gsi, stmts, GSI_SAME_STMT); + tree base = f.fold_contiguous_base (vectype); /* Emit two statements: a clobber of the lhs, so that it isn't upwards exposed, and then the load itself. @@ -3001,9 +2982,7 @@ public: return nullptr; /* svsel corresponds exactly to VEC_COND_EXPR. */ - gimple_seq stmts = NULL; - tree pred = f.convert_pred (stmts, f.vector_type (0), 0); - gsi_insert_seq_before (f.gsi, stmts, GSI_SAME_STMT); + tree pred = f.convert_pred (f.vector_type (0), 0); return gimple_build_assign (f.lhs, VEC_COND_EXPR, pred, gimple_call_arg (f.call, 1), gimple_call_arg (f.call, 2)); @@ -3123,10 +3102,8 @@ public: tree vectype = f.vector_type (0); /* Get the predicate and base pointer. */ - gimple_seq stmts = NULL; - tree pred = f.convert_pred (stmts, vectype, 0); - tree base = f.fold_contiguous_base (stmts, vectype); - gsi_insert_seq_before (f.gsi, stmts, GSI_SAME_STMT); + tree pred = f.convert_pred (vectype, 0); + tree base = f.fold_contiguous_base (vectype); tree cookie = f.load_store_cookie (TREE_TYPE (vectype)); tree rhs = gimple_call_arg (f.call, gimple_call_num_args (f.call) - 1); @@ -3222,10 +3199,8 @@ public: tree vectype = f.vector_type (0); /* Get the predicate and base pointer. */ - gimple_seq stmts = NULL; - tree pred = f.convert_pred (stmts, vectype, 0); - tree base = f.fold_contiguous_base (stmts, vectype); - gsi_insert_seq_before (f.gsi, stmts, GSI_SAME_STMT); + tree pred = f.convert_pred (vectype, 0); + tree base = f.fold_contiguous_base (vectype); /* View the stored data as an array of vectors. */ unsigned int num_args = gimple_call_num_args (f.call);
