gcc/ChangeLog: * config/riscv/riscv-vsetvl.cc (vector_infos_manager::vector_infos_manager): Removed. (vector_infos_manager::create_expr): Removed. (class pre_vsetvl): New class. (vector_infos_manager::get_expr_id): Removed. (vector_infos_manager::all_same_ratio_p): Removed. (vector_infos_manager::all_avail_in_compatible_p): Removed. (vector_infos_manager::all_same_avl_p): Removed. (vector_infos_manager::expr_set_num): Removed. (vector_infos_manager::release): Removed. (vector_infos_manager::create_bitmap_vectors): Removed. (vector_infos_manager::free_bitmap_vectors): Removed. (vector_infos_manager::dump): Removed.
--- gcc/config/riscv/riscv-vsetvl.cc | 674 ++++++++++++++----------------- 1 file changed, 307 insertions(+), 367 deletions(-) diff --git a/gcc/config/riscv/riscv-vsetvl.cc b/gcc/config/riscv/riscv-vsetvl.cc index c9f2f653247..c73a84cb6bd 100644 --- a/gcc/config/riscv/riscv-vsetvl.cc +++ b/gcc/config/riscv/riscv-vsetvl.cc @@ -2384,402 +2384,342 @@ public: } }; -vector_infos_manager::vector_infos_manager () -{ - vector_edge_list = nullptr; - vector_kill = nullptr; - vector_del = nullptr; - vector_insert = nullptr; - vector_antic = nullptr; - vector_transp = nullptr; - vector_comp = nullptr; - vector_avin = nullptr; - vector_avout = nullptr; - vector_antin = nullptr; - vector_antout = nullptr; - vector_earliest = nullptr; - vector_insn_infos.safe_grow_cleared (get_max_uid ()); - vector_block_infos.safe_grow_cleared (last_basic_block_for_fn (cfun)); - if (!optimize) - { - basic_block cfg_bb; - rtx_insn *rinsn; - FOR_ALL_BB_FN (cfg_bb, cfun) - { - vector_block_infos[cfg_bb->index].local_dem = vector_insn_info (); - vector_block_infos[cfg_bb->index].reaching_out = vector_insn_info (); - FOR_BB_INSNS (cfg_bb, rinsn) - vector_insn_infos[INSN_UID (rinsn)].parse_insn (rinsn); - } - } - else - { - for (const bb_info *bb : crtl->ssa->bbs ()) - { - vector_block_infos[bb->index ()].local_dem = vector_insn_info (); - vector_block_infos[bb->index ()].reaching_out = vector_insn_info (); - for (insn_info *insn : bb->real_insns ()) - vector_insn_infos[insn->uid ()].parse_insn (insn); - vector_block_infos[bb->index ()].probability = profile_probability (); - } - } -} -void -vector_infos_manager::create_expr (vector_insn_info &info) +class pre_vsetvl { - for (size_t i = 0; i < vector_exprs.length (); i++) - if (*vector_exprs[i] == info) - return; - vector_exprs.safe_push (&info); -} - -size_t -vector_infos_manager::get_expr_id (const vector_insn_info &info) const -{ - for (size_t i = 0; i < vector_exprs.length (); i++) - if (*vector_exprs[i] == info) - return i; - gcc_unreachable (); -} - -auto_vec<size_t> -vector_infos_manager::get_all_available_exprs ( - const vector_insn_info &info) const -{ - auto_vec<size_t> available_list; - for (size_t i = 0; i < vector_exprs.length (); i++) - if (info.available_p (*vector_exprs[i])) - available_list.safe_push (i); - return available_list; -} - -bool -vector_infos_manager::all_same_ratio_p (sbitmap bitdata) const -{ - if (bitmap_empty_p (bitdata)) - return false; +private: + demand_system m_dem; + auto_vec<vsetvl_block_info> m_vector_block_infos; - int ratio = -1; - unsigned int bb_index; - sbitmap_iterator sbi; + /* data for avl reaching defintion. */ + sbitmap m_avl_regs; + sbitmap *m_avl_def_in; + sbitmap *m_avl_def_out; + sbitmap *m_reg_def_loc; + + /* data for vsetvl info reaching defintion. */ + vsetvl_info m_unknow_info; + auto_vec<vsetvl_info *> m_vsetvl_def_exprs; + sbitmap *m_vsetvl_def_in; + sbitmap *m_vsetvl_def_out; + + /* data for lcm */ + auto_vec<vsetvl_info *> m_exprs; + sbitmap *m_avloc; + sbitmap *m_avin; + sbitmap *m_avout; + sbitmap *m_kill; + sbitmap *m_antloc; + sbitmap *m_transp; + sbitmap *m_insert; + sbitmap *m_del; + struct edge_list *m_edges; + + auto_vec<vsetvl_info> m_delete_list; + + vsetvl_block_info &get_block_info (const bb_info *bb) + { + return m_vector_block_infos[bb->index ()]; + } + const vsetvl_block_info &get_block_info (const basic_block bb) const + { + return m_vector_block_infos[bb->index]; + } - EXECUTE_IF_SET_IN_BITMAP (bitdata, 0, bb_index, sbi) - { - if (ratio == -1) - ratio = vector_exprs[bb_index]->get_ratio (); - else if (vector_exprs[bb_index]->get_ratio () != ratio) - return false; - } - return true; -} + vsetvl_block_info &get_block_info (const basic_block bb) + { + return m_vector_block_infos[bb->index]; + } -/* Return TRUE if the incoming vector configuration state - to CFG_BB is compatible with the vector configuration - state in CFG_BB, FALSE otherwise. */ -bool -vector_infos_manager::all_avail_in_compatible_p (const basic_block cfg_bb) const -{ - const auto &info = vector_block_infos[cfg_bb->index].local_dem; - sbitmap avin = vector_avin[cfg_bb->index]; - unsigned int bb_index; - sbitmap_iterator sbi; - EXECUTE_IF_SET_IN_BITMAP (avin, 0, bb_index, sbi) - { - const auto &avin_info - = static_cast<const vl_vtype_info &> (*vector_exprs[bb_index]); - if (!info.compatible_p (avin_info)) - return false; - } - return true; -} + void add_expr (auto_vec<vsetvl_info *> &m_exprs, vsetvl_info &info) + { + for (vsetvl_info *item : m_exprs) + { + if (*item == info) + return; + } + m_exprs.safe_push (&info); + } -bool -vector_infos_manager::all_same_avl_p (const basic_block cfg_bb, - sbitmap bitdata) const -{ - if (bitmap_empty_p (bitdata)) - return false; + unsigned get_expr_index (auto_vec<vsetvl_info *> &m_exprs, + const vsetvl_info &info) + { + for (size_t i = 0; i < m_exprs.length (); i += 1) + { + if (*m_exprs[i] == info) + return i; + } + gcc_unreachable (); + } - const auto &block_info = vector_block_infos[cfg_bb->index]; - if (!block_info.local_dem.demand_p (DEMAND_AVL)) - return true; + bool anticpatable_exp_p (const vsetvl_info &header_info) + { + if (!header_info.has_nonvlmax_reg_avl () && !header_info.has_vl ()) + return true; - avl_info avl = block_info.local_dem.get_avl_info (); - unsigned int bb_index; - sbitmap_iterator sbi; + bb_info *bb = header_info.get_bb (); + insn_info *prev_insn = bb->head_insn (); + insn_info *next_insn = header_info.insn_inside_bb_p () + ? header_info.get_insn () + : header_info.get_bb ()->end_insn (); - EXECUTE_IF_SET_IN_BITMAP (bitdata, 0, bb_index, sbi) - { - if (vector_exprs[bb_index]->get_avl_info () != avl) - return false; - } - return true; -} + return m_dem.avl_vl_unmodified_between_p (prev_insn, next_insn, + header_info); + } -bool -vector_infos_manager::earliest_fusion_worthwhile_p ( - const basic_block cfg_bb) const -{ - edge e; - edge_iterator ei; - profile_probability prob = profile_probability::uninitialized (); - FOR_EACH_EDGE (e, ei, cfg_bb->succs) - { - if (prob == profile_probability::uninitialized ()) - prob = vector_block_infos[e->dest->index].probability; - else if (prob == vector_block_infos[e->dest->index].probability) - continue; - else - /* We pick the highest probability among those incompatible VSETVL - infos. When all incompatible VSTEVL infos have same probability, we - don't pick any of them. */ - return true; - } - return false; -} + bool available_exp_p (const vsetvl_info &prev_info, + const vsetvl_info &next_info) + { + return m_dem.available_p (prev_info, next_info); + } -bool -vector_infos_manager::vsetvl_dominated_by_all_preds_p ( - const basic_block cfg_bb, const vector_insn_info &info) const -{ - edge e; - edge_iterator ei; - FOR_EACH_EDGE (e, ei, cfg_bb->preds) - { - const auto &reaching_out = vector_block_infos[e->src->index].reaching_out; - if (e->src->index == cfg_bb->index && reaching_out.compatible_p (info)) - continue; - if (!vsetvl_dominated_by_p (e->src, info, reaching_out, false)) - return false; - } - return true; -} + void compute_probabilities () + { + edge e; + edge_iterator ei; -size_t -vector_infos_manager::expr_set_num (sbitmap bitdata) const -{ - size_t count = 0; - for (size_t i = 0; i < vector_exprs.length (); i++) - if (bitmap_bit_p (bitdata, i)) - count++; - return count; -} + for (const bb_info *bb : crtl->ssa->bbs ()) + { + basic_block cfg_bb = bb->cfg_bb (); + auto &curr_prob = get_block_info (cfg_bb).probability; + + /* GCC assume entry block (bb 0) are always so + executed so set its probability as "always". */ + if (ENTRY_BLOCK_PTR_FOR_FN (cfun) == cfg_bb) + curr_prob = profile_probability::always (); + /* Exit block (bb 1) is the block we don't need to process. */ + if (EXIT_BLOCK_PTR_FOR_FN (cfun) == cfg_bb) + continue; -void -vector_infos_manager::release (void) -{ - if (!vector_insn_infos.is_empty ()) - vector_insn_infos.release (); - if (!vector_block_infos.is_empty ()) - vector_block_infos.release (); - if (!vector_exprs.is_empty ()) - vector_exprs.release (); - - gcc_assert (to_refine_vsetvls.is_empty ()); - gcc_assert (to_delete_vsetvls.is_empty ()); - if (optimize > 0) - free_bitmap_vectors (); -} + gcc_assert (curr_prob.initialized_p ()); + FOR_EACH_EDGE (e, ei, cfg_bb->succs) + { + auto &new_prob = get_block_info (e->dest).probability; + /* Normally, the edge probability should be initialized. + However, some special testing code which is written in + GIMPLE IR style force the edge probility uninitialized, + we conservatively set it as never so that it will not + affect PRE (Phase 3 && Phse 4). */ + if (!e->probability.initialized_p ()) + new_prob = profile_probability::never (); + else if (!new_prob.initialized_p ()) + new_prob = curr_prob * e->probability; + else if (new_prob == profile_probability::always ()) + continue; + else + new_prob += curr_prob * e->probability; + } + } + } -void -vector_infos_manager::create_bitmap_vectors (void) -{ - /* Create the bitmap vectors. */ - vector_antic = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), - vector_exprs.length ()); - vector_transp = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), - vector_exprs.length ()); - vector_comp = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), - vector_exprs.length ()); - vector_avin = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), - vector_exprs.length ()); - vector_avout = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), - vector_exprs.length ()); - vector_kill = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), - vector_exprs.length ()); - vector_antin = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), - vector_exprs.length ()); - vector_antout = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), - vector_exprs.length ()); - - bitmap_vector_ones (vector_transp, last_basic_block_for_fn (cfun)); - bitmap_vector_clear (vector_antic, last_basic_block_for_fn (cfun)); - bitmap_vector_clear (vector_comp, last_basic_block_for_fn (cfun)); - vector_edge_list = create_edge_list (); - vector_earliest = sbitmap_vector_alloc (NUM_EDGES (vector_edge_list), - vector_exprs.length ()); -} + void insert_vsetvl_insn (enum emit_type emit_type, const vsetvl_info &info) + { + rtx pat = info.get_vsetvl_pat (); + rtx_insn *rinsn = info.get_insn ()->rtl (); -void -vector_infos_manager::free_bitmap_vectors (void) -{ - /* Finished. Free up all the things we've allocated. */ - free_edge_list (vector_edge_list); - if (vector_del) - sbitmap_vector_free (vector_del); - if (vector_insert) - sbitmap_vector_free (vector_insert); - if (vector_kill) - sbitmap_vector_free (vector_kill); - if (vector_antic) - sbitmap_vector_free (vector_antic); - if (vector_transp) - sbitmap_vector_free (vector_transp); - if (vector_comp) - sbitmap_vector_free (vector_comp); - if (vector_avin) - sbitmap_vector_free (vector_avin); - if (vector_avout) - sbitmap_vector_free (vector_avout); - if (vector_antin) - sbitmap_vector_free (vector_antin); - if (vector_antout) - sbitmap_vector_free (vector_antout); - if (vector_earliest) - sbitmap_vector_free (vector_earliest); - - vector_edge_list = nullptr; - vector_kill = nullptr; - vector_del = nullptr; - vector_insert = nullptr; - vector_antic = nullptr; - vector_transp = nullptr; - vector_comp = nullptr; - vector_avin = nullptr; - vector_avout = nullptr; - vector_antin = nullptr; - vector_antout = nullptr; - vector_earliest = nullptr; -} + if (emit_type == EMIT_DIRECT) + { + emit_insn (pat); + if (dump_file) + { + fprintf (dump_file, " Insert vsetvl insn %d:\n", + INSN_UID (get_last_insn ())); + print_rtl_single (dump_file, get_last_insn ()); + } + } + else if (emit_type == EMIT_BEFORE) + { + emit_insn_before (pat, rinsn); + if (dump_file) + { + fprintf (dump_file, " Insert vsetvl insn before insn %d:\n", + INSN_UID (rinsn)); + print_rtl_single (dump_file, PREV_INSN (rinsn)); + } + } + else + { + emit_insn_after (pat, rinsn); + if (dump_file) + { + fprintf (dump_file, " Insert vsetvl insn after insn %d:\n", + INSN_UID (rinsn)); + print_rtl_single (dump_file, NEXT_INSN (rinsn)); + } + } + } -void -vector_infos_manager::dump (FILE *file) const -{ - basic_block cfg_bb; - rtx_insn *rinsn; + void change_vsetvl_insn (const vsetvl_info &info) + { + rtx_insn *rinsn = info.get_insn ()->rtl (); + rtx new_pat = info.get_vsetvl_pat (); - fprintf (file, "\n"); - FOR_ALL_BB_FN (cfg_bb, cfun) - { - fprintf (file, "Local vector info of <bb %d>:\n", cfg_bb->index); - fprintf (file, "<HEADER>="); - vector_block_infos[cfg_bb->index].local_dem.dump (file); - FOR_BB_INSNS (cfg_bb, rinsn) - { - if (!NONDEBUG_INSN_P (rinsn) || !has_vtype_op (rinsn)) - continue; - fprintf (file, "<insn %d>=", INSN_UID (rinsn)); - const auto &info = vector_insn_infos[INSN_UID (rinsn)]; - info.dump (file); - } - fprintf (file, "<FOOTER>="); - vector_block_infos[cfg_bb->index].reaching_out.dump (file); - fprintf (file, "<Probability>="); - vector_block_infos[cfg_bb->index].probability.dump (file); - fprintf (file, "\n\n"); - } + if (dump_file) + { + fprintf (dump_file, " Change insn %d from:\n", INSN_UID (rinsn)); + print_rtl_single (dump_file, rinsn); + } - fprintf (file, "\n"); - FOR_ALL_BB_FN (cfg_bb, cfun) - { - fprintf (file, "Local properties of <bb %d>:\n", cfg_bb->index); + validate_change_or_fail (rinsn, &PATTERN (rinsn), new_pat, false); - fprintf (file, "<ANTLOC>="); - if (vector_antic == nullptr) - fprintf (file, "(nil)\n"); - else - dump_bitmap_file (file, vector_antic[cfg_bb->index]); + if (dump_file) + { + fprintf (dump_file, "\n to:\n"); + print_rtl_single (dump_file, rinsn); + } + } - fprintf (file, "<AVLOC>="); - if (vector_comp == nullptr) - fprintf (file, "(nil)\n"); - else - dump_bitmap_file (file, vector_comp[cfg_bb->index]); + void remove_vsetvl_insn (const vsetvl_info &info) + { + rtx_insn *rinsn = info.get_insn ()->rtl (); + if (dump_file) + { + fprintf (dump_file, " Eliminate insn %d:\n", INSN_UID (rinsn)); + print_rtl_single (dump_file, rinsn); + } + if (in_sequence_p ()) + remove_insn (rinsn); + else + delete_insn (rinsn); + } - fprintf (file, "<TRANSP>="); - if (vector_transp == nullptr) - fprintf (file, "(nil)\n"); - else - dump_bitmap_file (file, vector_transp[cfg_bb->index]); + bool successors_probability_equal_p (const basic_block cfg_bb) const + { + edge e; + edge_iterator ei; + profile_probability prob = profile_probability::uninitialized (); + FOR_EACH_EDGE (e, ei, cfg_bb->succs) + { + if (prob == profile_probability::uninitialized ()) + prob = m_vector_block_infos[e->dest->index].probability; + else if (prob == m_vector_block_infos[e->dest->index].probability) + continue; + else + /* We pick the highest probability among those incompatible VSETVL + infos. When all incompatible VSTEVL infos have same probability, we + don't pick any of them. */ + return false; + } + return true; + } - fprintf (file, "<KILL>="); - if (vector_kill == nullptr) - fprintf (file, "(nil)\n"); - else - dump_bitmap_file (file, vector_kill[cfg_bb->index]); + bool preds_has_same_avl_p (const vsetvl_info &curr_info) + { + gcc_assert ( + !bitmap_empty_p (m_vsetvl_def_in[curr_info.get_bb ()->index ()])); - fprintf (file, "<ANTIN>="); - if (vector_antin == nullptr) - fprintf (file, "(nil)\n"); - else - dump_bitmap_file (file, vector_antin[cfg_bb->index]); + unsigned expr_index; + sbitmap_iterator sbi; + EXECUTE_IF_SET_IN_BITMAP (m_vsetvl_def_in[curr_info.get_bb ()->index ()], 0, + expr_index, sbi) + { + const vsetvl_info &prev_info = *m_vsetvl_def_exprs[expr_index]; + if (!prev_info.valid_p () + || !m_dem.avl_available_p (prev_info, curr_info)) + return false; + } - fprintf (file, "<ANTOUT>="); - if (vector_antout == nullptr) - fprintf (file, "(nil)\n"); - else - dump_bitmap_file (file, vector_antout[cfg_bb->index]); - } + return true; + } - fprintf (file, "\n"); - FOR_ALL_BB_FN (cfg_bb, cfun) - { - fprintf (file, "Global LCM (Lazy code motion) result of <bb %d>:\n", - cfg_bb->index); +public: + pre_vsetvl () + : m_avl_def_in (nullptr), m_avl_def_out (nullptr), + m_vsetvl_def_in (nullptr), m_vsetvl_def_out (nullptr), m_avloc (nullptr), + m_avin (nullptr), m_avout (nullptr), m_kill (nullptr), m_antloc (nullptr), + m_transp (nullptr), m_insert (nullptr), m_del (nullptr), m_edges (nullptr) + { + /* Initialization of RTL_SSA. */ + calculate_dominance_info (CDI_DOMINATORS); + df_analyze (); + crtl->ssa = new function_info (cfun); + m_vector_block_infos.safe_grow_cleared (last_basic_block_for_fn (cfun)); + compute_probabilities (); + m_unknow_info.set_unknown (); + } - fprintf (file, "<AVIN>="); - if (vector_avin == nullptr) - fprintf (file, "(nil)\n"); - else - dump_bitmap_file (file, vector_avin[cfg_bb->index]); + void finish () + { + free_dominance_info (CDI_DOMINATORS); + if (crtl->ssa->perform_pending_updates ()) + cleanup_cfg (0); + delete crtl->ssa; + crtl->ssa = nullptr; + + if (m_avl_regs) + sbitmap_free (m_avl_regs); + if (m_reg_def_loc) + sbitmap_vector_free (m_reg_def_loc); + + if (m_avl_def_in) + sbitmap_vector_free (m_avl_def_in); + if (m_avl_def_out) + sbitmap_vector_free (m_avl_def_out); + + if (m_vsetvl_def_in) + sbitmap_vector_free (m_vsetvl_def_in); + if (m_vsetvl_def_out) + sbitmap_vector_free (m_vsetvl_def_out); + + if (m_avloc) + sbitmap_vector_free (m_avloc); + if (m_kill) + sbitmap_vector_free (m_kill); + if (m_antloc) + sbitmap_vector_free (m_antloc); + if (m_transp) + sbitmap_vector_free (m_transp); + if (m_insert) + sbitmap_vector_free (m_insert); + if (m_del) + sbitmap_vector_free (m_del); + if (m_avin) + sbitmap_vector_free (m_avin); + if (m_avout) + sbitmap_vector_free (m_avout); + + if (m_edges) + free_edge_list (m_edges); + } - fprintf (file, "<AVOUT>="); - if (vector_avout == nullptr) - fprintf (file, "(nil)\n"); - else - dump_bitmap_file (file, vector_avout[cfg_bb->index]); + void compute_avl_def_data (); + void compute_vsetvl_def_data (); + void compute_lcm_local_properties (); - fprintf (file, "<DELETE>="); - if (vector_del == nullptr) - fprintf (file, "(nil)\n"); - else - dump_bitmap_file (file, vector_del[cfg_bb->index]); - } + void fuse_local_vsetvl_info (); + bool earliest_fuse_vsetvl_info (); + void pre_global_vsetvl_info (); + void emit_vsetvl (); + void cleaup (); + void remove_avl_operand (); + void remove_unused_dest_operand (); - for (size_t i = 0; i < vector_exprs.length (); i++) - { - for (int ed = 0; ed < NUM_EDGES (vector_edge_list); ed++) - { - edge eg = INDEX_EDGE (vector_edge_list, ed); - if (vector_insert) - { - if (bitmap_bit_p (vector_insert[ed], i)) - { - fprintf (file, - "\nGlobal LCM (Lazy code motion) INSERT info:\n"); - fprintf (file, - "INSERT edge %d from <bb %d> to <bb %d> for VSETVL " - "expr[%ld]\n", - ed, eg->src->index, eg->dest->index, i); - } - } - else - { - if (bitmap_bit_p (vector_earliest[ed], i)) - { - fprintf (file, - "\nGlobal LCM (Lazy code motion) EARLIEST info:\n"); - fprintf ( - file, - "EARLIEST edge %d from <bb %d> to <bb %d> for VSETVL " - "expr[%ld]\n", - ed, eg->src->index, eg->dest->index, i); - } - } - } - } -} + void dump (FILE *file, const char *title) const + { + fprintf (file, "\nVSETVL infos after %s\n\n", title); + for (const bb_info *bb : crtl->ssa->bbs ()) + { + const auto &block_info = m_vector_block_infos[bb->index ()]; + fprintf (file, " bb %d:\n", bb->index ()); + fprintf (file, " probability: "); + block_info.probability.dump (file); + fprintf (file, "\n"); + if (!block_info.empty_p ()) + { + fprintf (file, " Header vsetvl info:"); + block_info.get_entry_info ().dump (file, " "); + fprintf (file, " Footer vsetvl info:"); + block_info.get_exit_info ().dump (file, " "); + for (const auto &info : block_info.infos) + { + fprintf (file, + " insn %d vsetvl info:", info.get_insn ()->uid ()); + info.dump (file, " "); + } + } + } + } +}; const pass_data pass_data_vsetvl = { RTL_PASS, /* type */ -- 2.36.3