gcc/ChangeLog:
* config/riscv/riscv-vsetvl.cc (incompatible_avl_p): Removed.
(different_sew_p): Removed.
(different_lmul_p): Removed.
(different_ratio_p): Removed.
(different_tail_policy_p): Removed.
(different_mask_policy_p): Removed.
(possible_zero_avl_p): Removed.
(second_ratio_invalid_for_first_sew_p): Removed.
(second_ratio_invalid_for_first_lmul_p): Removed.
(float_insn_valid_sew_p): Removed.
(second_sew_less_than_first_sew_p): Removed.
(first_sew_less_than_second_sew_p): Removed.
(compare_lmul): Removed.
(second_lmul_less_than_first_lmul_p): Removed.
(second_ratio_less_than_first_ratio_p): Removed.
(DEF_INCOMPATIBLE_COND): Removed.
(greatest_sew): Removed.
(first_sew): Removed.
(second_sew): Removed.
(first_vlmul): Removed.
(second_vlmul): Removed.
(first_ratio): Removed.
(second_ratio): Removed.
(vlmul_for_first_sew_second_ratio): Removed.
(vlmul_for_greatest_sew_second_ratio): Removed.
(ratio_for_second_sew_first_vlmul): Removed.
(DEF_SEW_LMUL_FUSE_RULE): Removed.
(always_unavailable): Removed.
(avl_unavailable_p): Removed.
(sew_unavailable_p): Removed.
(lmul_unavailable_p): Removed.
(ge_sew_unavailable_p): Removed.
(ge_sew_lmul_unavailable_p): Removed.
(ge_sew_ratio_unavailable_p): Removed.
(DEF_UNAVAILABLE_COND): Removed.
(same_sew_lmul_demand_p): Removed.
(propagate_avl_across_demands_p): Removed.
(reg_available_p): Removed.
(support_relaxed_compatible_p): Removed.
(count_regno_occurrences): Removed.
(demands_can_be_fused_p): Removed.
(earliest_pred_can_be_fused_p): Removed.
(vsetvl_dominated_by_p): Removed.
(class demand_system): New.
(DEF_SEW_LMUL_RULE): New.
(DEF_POLICY_RULE): New.
(DEF_AVL_RULE): New.
---
gcc/config/riscv/riscv-vsetvl.cc | 1158 +++++++++++++++++-------------
1 file changed, 668 insertions(+), 490 deletions(-)
diff --git a/gcc/config/riscv/riscv-vsetvl.cc b/gcc/config/riscv/riscv-vsetvl.cc
index 8908071dc0d..c9f2f653247 100644
--- a/gcc/config/riscv/riscv-vsetvl.cc
+++ b/gcc/config/riscv/riscv-vsetvl.cc
@@ -1091,496 +1091,6 @@ calculate_vlmul (unsigned int sew, unsigned int ratio)
return LMUL_RESERVED;
}
-static bool
-incompatible_avl_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return !info1.compatible_avl_p (info2) && !info2.compatible_avl_p (info1);
-}
-
-static bool
-different_sew_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return info1.get_sew () != info2.get_sew ();
-}
-
-static bool
-different_lmul_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return info1.get_vlmul () != info2.get_vlmul ();
-}
-
-static bool
-different_ratio_p (const vector_insn_info &info1, const vector_insn_info
&info2)
-{
- return info1.get_ratio () != info2.get_ratio ();
-}
-
-static bool
-different_tail_policy_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info1.get_ta () != info2.get_ta ();
-}
-
-static bool
-different_mask_policy_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info1.get_ma () != info2.get_ma ();
-}
-
-static bool
-possible_zero_avl_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return !info1.has_non_zero_avl () || !info2.has_non_zero_avl ();
-}
-
-static bool
-second_ratio_invalid_for_first_sew_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_vlmul (info1.get_sew (), info2.get_ratio ())
- == LMUL_RESERVED;
-}
-
-static bool
-second_ratio_invalid_for_first_lmul_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_sew (info1.get_vlmul (), info2.get_ratio ()) == 0;
-}
-
-static bool
-float_insn_valid_sew_p (const vector_insn_info &info, unsigned int sew)
-{
- if (info.get_insn () && info.get_insn ()->is_real ()
- && get_attr_type (info.get_insn ()->rtl ()) == TYPE_VFMOVFV)
- {
- if (sew == 16)
- return TARGET_VECTOR_ELEN_FP_16;
- else if (sew == 32)
- return TARGET_VECTOR_ELEN_FP_32;
- else if (sew == 64)
- return TARGET_VECTOR_ELEN_FP_64;
- }
- return true;
-}
-
-static bool
-second_sew_less_than_first_sew_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info2.get_sew () < info1.get_sew ()
- || !float_insn_valid_sew_p (info1, info2.get_sew ());
-}
-
-static bool
-first_sew_less_than_second_sew_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info1.get_sew () < info2.get_sew ()
- || !float_insn_valid_sew_p (info2, info1.get_sew ());
-}
-
-/* return 0 if LMUL1 == LMUL2.
- return -1 if LMUL1 < LMUL2.
- return 1 if LMUL1 > LMUL2. */
-static int
-compare_lmul (vlmul_type vlmul1, vlmul_type vlmul2)
-{
- if (vlmul1 == vlmul2)
- return 0;
-
- switch (vlmul1)
- {
- case LMUL_1:
- if (vlmul2 == LMUL_2 || vlmul2 == LMUL_4 || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_2:
- if (vlmul2 == LMUL_4 || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_4:
- if (vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_8:
- return -1;
- case LMUL_F2:
- if (vlmul2 == LMUL_1 || vlmul2 == LMUL_2 || vlmul2 == LMUL_4
- || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_F4:
- if (vlmul2 == LMUL_F2 || vlmul2 == LMUL_1 || vlmul2 == LMUL_2
- || vlmul2 == LMUL_4 || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_F8:
- return 0;
- default:
- gcc_unreachable ();
- }
-}
-
-static bool
-second_lmul_less_than_first_lmul_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return compare_lmul (info2.get_vlmul (), info1.get_vlmul ()) == -1;
-}
-
-static bool
-second_ratio_less_than_first_ratio_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info2.get_ratio () < info1.get_ratio ();
-}
-
-static CONSTEXPR const demands_cond incompatible_conds[] = {
-#define DEF_INCOMPATIBLE_COND(AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1,
\
- GE_SEW1, TAIL_POLICTY1, MASK_POLICY1, AVL2, \
- SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, \
- TAIL_POLICTY2, MASK_POLICY2, COND) \
- {{{AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1, TAIL_POLICTY1,
\
- MASK_POLICY1},
\
- {AVL2, SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2,
\
- MASK_POLICY2}},
\
- COND},
-#include "riscv-vsetvl.def"
-};
-
-static unsigned
-greatest_sew (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return std::max (info1.get_sew (), info2.get_sew ());
-}
-
-static unsigned
-first_sew (const vector_insn_info &info1, const vector_insn_info &)
-{
- return info1.get_sew ();
-}
-
-static unsigned
-second_sew (const vector_insn_info &, const vector_insn_info &info2)
-{
- return info2.get_sew ();
-}
-
-static vlmul_type
-first_vlmul (const vector_insn_info &info1, const vector_insn_info &)
-{
- return info1.get_vlmul ();
-}
-
-static vlmul_type
-second_vlmul (const vector_insn_info &, const vector_insn_info &info2)
-{
- return info2.get_vlmul ();
-}
-
-static unsigned
-first_ratio (const vector_insn_info &info1, const vector_insn_info &)
-{
- return info1.get_ratio ();
-}
-
-static unsigned
-second_ratio (const vector_insn_info &, const vector_insn_info &info2)
-{
- return info2.get_ratio ();
-}
-
-static vlmul_type
-vlmul_for_first_sew_second_ratio (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_vlmul (info1.get_sew (), info2.get_ratio ());
-}
-
-static vlmul_type
-vlmul_for_greatest_sew_second_ratio (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_vlmul (MAX (info1.get_sew (), info2.get_sew ()),
- info2.get_ratio ());
-}
-
-static unsigned
-ratio_for_second_sew_first_vlmul (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_ratio (info2.get_sew (), info1.get_vlmul ());
-}
-
-static CONSTEXPR const demands_fuse_rule fuse_rules[] = {
-#define DEF_SEW_LMUL_FUSE_RULE(DEMAND_SEW1, DEMAND_LMUL1, DEMAND_RATIO1,
\
- DEMAND_GE_SEW1, DEMAND_SEW2, DEMAND_LMUL2, \
- DEMAND_RATIO2, DEMAND_GE_SEW2, NEW_DEMAND_SEW, \
- NEW_DEMAND_LMUL, NEW_DEMAND_RATIO, \
- NEW_DEMAND_GE_SEW, NEW_SEW, NEW_VLMUL, \
- NEW_RATIO) \
- {{{DEMAND_ANY, DEMAND_SEW1, DEMAND_LMUL1, DEMAND_RATIO1, DEMAND_ANY,
\
- DEMAND_GE_SEW1, DEMAND_ANY, DEMAND_ANY},
\
- {DEMAND_ANY, DEMAND_SEW2, DEMAND_LMUL2, DEMAND_RATIO2, DEMAND_ANY,
\
- DEMAND_GE_SEW2, DEMAND_ANY, DEMAND_ANY}},
\
- NEW_DEMAND_SEW,
\
- NEW_DEMAND_LMUL,
\
- NEW_DEMAND_RATIO,
\
- NEW_DEMAND_GE_SEW,
\
- NEW_SEW,
\
- NEW_VLMUL,
\
- NEW_RATIO},
-#include "riscv-vsetvl.def"
-};
-
-static bool
-always_unavailable (const vector_insn_info &, const vector_insn_info &)
-{
- return true;
-}
-
-static bool
-avl_unavailable_p (const vector_insn_info &info1, const vector_insn_info
&info2)
-{
- return !info2.compatible_avl_p (info1.get_avl_info ());
-}
-
-static bool
-sew_unavailable_p (const vector_insn_info &info1, const vector_insn_info
&info2)
-{
- if (!info2.demand_p (DEMAND_LMUL) && !info2.demand_p (DEMAND_RATIO))
- {
- if (info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- return info1.get_sew () != info2.get_sew ();
- }
- return true;
-}
-
-static bool
-lmul_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.get_vlmul () == info2.get_vlmul () && !info2.demand_p (DEMAND_SEW)
- && !info2.demand_p (DEMAND_RATIO))
- return false;
- return true;
-}
-
-static bool
-ge_sew_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_LMUL) && !info2.demand_p (DEMAND_RATIO)
- && info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- return true;
-}
-
-static bool
-ge_sew_lmul_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_RATIO) && info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- return true;
-}
-
-static bool
-ge_sew_ratio_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_LMUL))
- {
- if (info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- /* Demand GE_SEW should be available for non-demand SEW. */
- else if (!info2.demand_p (DEMAND_SEW))
- return false;
- }
- return true;
-}
-
-static CONSTEXPR const demands_cond unavailable_conds[] = {
-#define DEF_UNAVAILABLE_COND(AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1,
\
- TAIL_POLICTY1, MASK_POLICY1, AVL2, SEW2, LMUL2, \
- RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2, \
- MASK_POLICY2, COND) \
- {{{AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1, TAIL_POLICTY1,
\
- MASK_POLICY1},
\
- {AVL2, SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2,
\
- MASK_POLICY2}},
\
- COND},
-#include "riscv-vsetvl.def"
-};
-
-static bool
-same_sew_lmul_demand_p (const bool *dems1, const bool *dems2)
-{
- return dems1[DEMAND_SEW] == dems2[DEMAND_SEW]
- && dems1[DEMAND_LMUL] == dems2[DEMAND_LMUL]
- && dems1[DEMAND_RATIO] == dems2[DEMAND_RATIO] && !dems1[DEMAND_GE_SEW]
- && !dems2[DEMAND_GE_SEW];
-}
-
-static bool
-propagate_avl_across_demands_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info2.demand_p (DEMAND_AVL))
- {
- if (info2.demand_p (DEMAND_NONZERO_AVL))
- return info1.demand_p (DEMAND_AVL)
- && !info1.demand_p (DEMAND_NONZERO_AVL) && info1.has_avl_reg ();
- }
- else
- return info1.demand_p (DEMAND_AVL) && info1.has_avl_reg ();
- return false;
-}
-
-static bool
-reg_available_p (const insn_info *insn, const vector_insn_info &info)
-{
- if (info.has_avl_reg () && !info.get_avl_source ())
- return false;
- insn_info *def_insn = info.get_avl_source ()->insn ();
- if (def_insn->bb () == insn->bb ())
- return before_p (def_insn, insn);
- else
- return dominated_by_p (CDI_DOMINATORS, insn->bb ()->cfg_bb (),
- def_insn->bb ()->cfg_bb ());
-}
-
-/* Return true if the instruction support relaxed compatible check. */
-static bool
-support_relaxed_compatible_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (fault_first_load_p (info1.get_insn ()->rtl ())
- && info2.demand_p (DEMAND_AVL) && info2.has_avl_reg ()
- && info2.get_avl_source () && info2.get_avl_source ()->insn ()->is_phi
())
- {
- hash_set<set_info *> sets
- = get_all_sets (info2.get_avl_source (), true, false, false);
- for (set_info *set : sets)
- {
- if (read_vl_insn_p (set->insn ()->rtl ()))
- {
- const insn_info *insn
- = get_backward_fault_first_load_insn (set->insn ());
- if (insn == info1.get_insn ())
- return info2.compatible_vtype_p (info1);
- }
- }
- }
- return false;
-}
-
-/* Count the number of REGNO in RINSN. */
-static int
-count_regno_occurrences (rtx_insn *rinsn, unsigned int regno)
-{
- int count = 0;
- extract_insn (rinsn);
- for (int i = 0; i < recog_data.n_operands; i++)
- if (refers_to_regno_p (regno, recog_data.operand[i]))
- count++;
- return count;
-}
-
-/* Return TRUE if the demands can be fused. */
-static bool
-demands_can_be_fused_p (const vector_insn_info &be_fused,
- const vector_insn_info &to_fuse)
-{
- return be_fused.compatible_p (to_fuse) && !be_fused.available_p (to_fuse);
-}
-
-/* Return true if we can fuse VSETVL demand info into predecessor of earliest
- * edge. */
-static bool
-earliest_pred_can_be_fused_p (const bb_info *earliest_pred,
- const vector_insn_info &earliest_info,
- const vector_insn_info &expr, rtx *vlmax_vl)
-{
- /* Backward VLMAX VL:
- bb 3:
- vsetivli zero, 1 ... -> vsetvli t1, zero
- vmv.s.x
- bb 5:
- vsetvli t1, zero ... -> to be elided.
- vlse16.v
-
- We should forward "t1". */
- if (!earliest_info.has_avl_reg () && expr.has_avl_reg ())
- {
- rtx avl_or_vl_reg = expr.get_avl_or_vl_reg ();
- gcc_assert (avl_or_vl_reg);
- const insn_info *last_insn = earliest_info.get_insn ();
- /* To fuse demand on earlest edge, we make sure AVL/VL
- didn't change from the consume insn to the predecessor
- of the edge. */
- for (insn_info *i = earliest_pred->end_insn ()->prev_nondebug_insn ();
- real_insn_and_same_bb_p (i, earliest_pred)
- && after_or_same_p (i, last_insn);
- i = i->prev_nondebug_insn ())
- {
- if (find_access (i->defs (), REGNO (avl_or_vl_reg)))
- return false;
- if (find_access (i->uses (), REGNO (avl_or_vl_reg)))
- return false;
- }
- if (vlmax_vl && vlmax_avl_p (expr.get_avl ()))
- *vlmax_vl = avl_or_vl_reg;
- }
-
- return true;
-}
-
-/* Return true if the current VSETVL 1 is dominated by preceding VSETVL 2.
-
- VSETVL 2 dominates VSETVL 1 should satisfy this following check:
-
- - VSETVL 2 should have the RATIO (SEW/LMUL) with VSETVL 1.
- - VSETVL 2 is user vsetvl (vsetvl VL, AVL)
- - VSETVL 2 "VL" result is the "AVL" of VSETL1. */
-static bool
-vsetvl_dominated_by_p (const basic_block cfg_bb,
- const vector_insn_info &vsetvl1,
- const vector_insn_info &vsetvl2, bool fuse_p)
-{
- if (!vsetvl1.valid_or_dirty_p () || !vsetvl2.valid_or_dirty_p ())
- return false;
- if (!has_vl_op (vsetvl1.get_insn ()->rtl ())
- || !vsetvl_insn_p (vsetvl2.get_insn ()->rtl ()))
- return false;
-
- hash_set<set_info *> sets
- = get_all_sets (vsetvl1.get_avl_source (), true, false, false);
- set_info *set = get_same_bb_set (sets, cfg_bb);
-
- if (!vsetvl1.has_avl_reg () || vlmax_avl_p (vsetvl1.get_avl ())
- || !vsetvl2.same_vlmax_p (vsetvl1) || !set
- || set->insn () != vsetvl2.get_insn ())
- return false;
-
- if (fuse_p && vsetvl2.same_vtype_p (vsetvl1))
- return false;
- else if (!fuse_p && !vsetvl2.same_vtype_p (vsetvl1))
- return false;
- return true;
-}
-
/* This flags indicates the minimum demand of the vl and vtype values by the
RVV instruction. For example, DEMAND_RATIO_P indicates that this RVV
instruction only needs the SEW/LMUL ratio to remain the same, and does not
@@ -2206,6 +1716,674 @@ public:
void set_empty_info () { m_info.set_empty (); }
};
+
+/* Demand system is the RVV-based VSETVL info analysis tools wrapper.
+ It defines compatible rules for SEW/LMUL, POLICY and AVL.
+ Also, it provides 3 iterfaces avaiable_p, compatible_p and
+ merge for the VSETVL PASS analysis and optimization.
+
+ - avaiable_p: Determine whether the next info can get the
+ avaiable VSETVL status from previous info.
+ e.g. bb 2 (demand SEW = 32, LMUL = M2) -> bb 3 (demand RATIO = 16).
+ Since bb 2 demand info (SEW/LMUL = 32/2 = 16) satisfies the bb 3
+ demand, the VSETVL instruction in bb 3 can be elided.
+ avaiable_p (previous, next) is true in such situation.
+ - compatible_p: Determine whether prev_info is compatible with next_info
+ so that we can have a new merged info that is avaiable to both of them.
+ - merge: Merge the stricter demand information from
+ next_info into prev_info so that prev_info becomes available to
+ next_info. */
+class demand_system
+{
+private:
+ sbitmap *m_avl_def_in;
+ sbitmap *m_avl_def_out;
+
+ /* predictors. */
+
+ inline bool always_true (const vsetvl_info &prev ATTRIBUTE_UNUSED,
+ const vsetvl_info &next ATTRIBUTE_UNUSED)
+ {
+ return true;
+ }
+ inline bool always_false (const vsetvl_info &prev ATTRIBUTE_UNUSED,
+ const vsetvl_info &next ATTRIBUTE_UNUSED)
+ {
+ return false;
+ }
+
+ /* predictors for sew and lmul */
+
+ inline bool lmul_eq_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_vlmul () == next.get_vlmul ();
+ }
+ inline bool sew_eq_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ();
+ }
+ inline bool sew_lmul_eq_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return lmul_eq_p (prev, next) && sew_eq_p (prev, next);
+ }
+ inline bool sew_ge_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ()
+ || (next.get_ta () && prev.get_sew () > next.get_sew ());
+ }
+ inline bool sew_le_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ()
+ || (prev.get_ta () && prev.get_sew () < next.get_sew ());
+ }
+ inline bool prev_sew_le_next_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_sew () <= next.get_max_sew ();
+ }
+ inline bool next_sew_le_prev_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return next.get_sew () <= prev.get_max_sew ();
+ }
+ inline bool max_sew_overlap_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return !(prev.get_sew () > next.get_max_sew ()
+ || next.get_sew () > prev.get_max_sew ());
+ }
+ inline bool ratio_eq_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.has_same_ratio (next);
+ }
+ inline bool prev_ratio_valid_for_next_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ratio () >= (next.get_sew () / 8);
+ }
+ inline bool next_ratio_valid_for_prev_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return next.get_ratio () >= (prev.get_sew () / 8);
+ }
+
+ inline bool sew_ge_and_ratio_eq_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return sew_ge_p (prev, next) && ratio_eq_p (prev, next);
+ }
+ inline bool sew_ge_and_prev_sew_le_next_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return sew_ge_p (prev, next) && prev_sew_le_next_max_sew_p (prev, next);
+ }
+ inline bool
+ sew_ge_and_prev_sew_le_next_max_sew_and_next_ratio_valid_for_prev_sew_p (
+ const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return sew_ge_p (prev, next) && prev_sew_le_next_max_sew_p (prev, next)
+ && next_ratio_valid_for_prev_sew_p (prev, next);
+ }
+ inline bool sew_le_and_next_sew_le_prev_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return sew_le_p (prev, next) && next_sew_le_prev_max_sew_p (prev, next);
+ }
+ inline bool
+ max_sew_overlap_and_next_ratio_valid_for_prev_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return next_ratio_valid_for_prev_sew_p (prev, next)
+ && max_sew_overlap_p (prev, next);
+ }
+ inline bool
+ sew_le_and_next_sew_le_prev_max_sew_and_ratio_eq_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return sew_le_p (prev, next) && ratio_eq_p (prev, next)
+ && next_sew_le_prev_max_sew_p (prev, next);
+ }
+ inline bool
+ max_sew_overlap_and_prev_ratio_valid_for_next_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev_ratio_valid_for_next_sew_p (prev, next)
+ && max_sew_overlap_p (prev, next);
+ }
+ inline bool
+ sew_le_and_next_sew_le_prev_max_sew_and_prev_ratio_valid_for_next_sew_p (
+ const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return sew_le_p (prev, next) && prev_ratio_valid_for_next_sew_p (prev,
next)
+ && next_sew_le_prev_max_sew_p (prev, next);
+ }
+ inline bool max_sew_overlap_and_ratio_eq_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ratio_eq_p (prev, next) && max_sew_overlap_p (prev, next);
+ }
+
+ /* predictors for tail and mask policy */
+
+ inline bool tail_policy_eq_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ta () == next.get_ta ();
+ }
+ inline bool mask_policy_eq_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ma () == next.get_ma ();
+ }
+ inline bool tail_mask_policy_eq_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return tail_policy_eq_p (prev, next) && mask_policy_eq_p (prev, next);
+ }
+
+ /* predictors for avl */
+
+ inline bool modify_or_use_vl_p (insn_info *i, const vsetvl_info &info)
+ {
+ return info.has_vl ()
+ && (find_access (i->uses (), REGNO (info.get_vl ()))
+ || find_access (i->defs (), REGNO (info.get_vl ())));
+ }
+ inline bool modify_avl_p (insn_info *i, const vsetvl_info &info)
+ {
+ return info.has_nonvlmax_reg_avl ()
+ && find_access (i->defs (), REGNO (info.get_avl ()));
+ }
+
+ inline bool modify_reg_between_p (insn_info *prev_insn, insn_info *curr_insn,
+ unsigned regno)
+ {
+ gcc_assert (prev_insn->compare_with (curr_insn) < 0);
+ for (insn_info *i = curr_insn->prev_nondebug_insn (); i != prev_insn;
+ i = i->prev_nondebug_insn ())
+ {
+ // no def of regno
+ if (find_access (i->defs (), regno))
+ return true;
+ }
+ return false;
+ }
+
+ inline bool reg_avl_equal_p (const vsetvl_info &prev, const vsetvl_info
&next)
+ {
+ if (!prev.has_nonvlmax_reg_avl () || !next.has_nonvlmax_reg_avl ())
+ return false;
+
+ if (same_equiv_note_p (prev.get_avl_def (), next.get_avl_def ()))
+ return true;
+
+ if (REGNO (prev.get_avl ()) != REGNO (next.get_avl ()))
+ return false;
+
+ insn_info *prev_insn = prev.get_insn ();
+ if (prev.get_bb () != prev_insn->bb ())
+ prev_insn = prev.get_bb ()->end_insn ();
+
+ insn_info *next_insn = next.get_insn ();
+ if (next.get_bb () != next_insn->bb ())
+ next_insn = next.get_bb ()->end_insn ();
+
+ return avl_vl_unmodified_between_p (prev_insn, next_insn, next, false);
+ }
+
+ inline bool avl_equal_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+
+ if (prev.get_ratio () != next.get_ratio ())
+ return false;
+
+ if (next.has_vl () && next.vl_use_by_non_rvv_insn_p ())
+ return false;
+
+ if (vector_config_insn_p (prev.get_insn ()->rtl ()) && next.get_avl_def ()
+ && next.get_avl_def ()->insn () == prev.get_insn ())
+ return true;
+
+ if (prev.get_read_vl_insn ())
+ {
+ if (!next.has_nonvlmax_reg_avl () || !next.get_avl_def ())
+ return false;
+ insn_info *avl_def_insn = extract_single_source (next.get_avl_def ());
+ return avl_def_insn == prev.get_read_vl_insn ();
+ }
+
+ if (prev == next && prev.has_nonvlmax_reg_avl ())
+ {
+ insn_info *insn = prev.get_insn ();
+ bb_info *bb = insn->bb ();
+ for (insn_info *i = insn; real_insn_and_same_bb_p (i, bb);
+ i = i->next_nondebug_insn ())
+ if (find_access (i->defs (), REGNO (prev.get_avl ())))
+ return false;
+ }
+
+ if (prev.has_vlmax_avl () && next.has_vlmax_avl ())
+ return true;
+ else if (prev.has_imm_avl () && next.has_imm_avl ())
+ return INTVAL (prev.get_avl ()) == INTVAL (next.get_avl ());
+ else if (prev.has_vl () && next.has_nonvlmax_reg_avl ()
+ && REGNO (prev.get_vl ()) == REGNO (next.get_avl ()))
+ {
+ insn_info *prev_insn = prev.insn_inside_bb_p ()
+ ? prev.get_insn ()
+ : prev.get_bb ()->end_insn ();
+
+ insn_info *next_insn = next.insn_inside_bb_p ()
+ ? next.get_insn ()
+ : next.get_bb ()->end_insn ();
+ return avl_vl_unmodified_between_p (prev_insn, next_insn, next, false);
+ }
+ else if (prev.has_nonvlmax_reg_avl () && next.has_nonvlmax_reg_avl ())
+ return reg_avl_equal_p (prev, next);
+
+ return false;
+ }
+ inline bool avl_equal_or_prev_avl_non_zero_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return avl_equal_p (prev, next) || prev.has_non_zero_avl ();
+ }
+
+ inline bool can_use_next_avl_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ if (!next.has_nonvlmax_reg_avl () && !next.has_vl ())
+ return true;
+
+ insn_info *prev_insn = prev.get_insn ();
+ if (prev.get_bb () != prev_insn->bb ())
+ prev_insn = prev.get_bb ()->end_insn ();
+
+ insn_info *next_insn = next.get_insn ();
+ if (next.get_bb () != next_insn->bb ())
+ next_insn = next.get_bb ()->end_insn ();
+
+ return avl_vl_unmodified_between_p (prev_insn, next_insn, next);
+ }
+
+ inline bool avl_equal_or_next_avl_non_zero_and_can_use_next_avl_p (
+ const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return avl_equal_p (prev, next)
+ || (next.has_non_zero_avl () && can_use_next_avl_p (prev, next));
+ }
+
+ /* modifiers */
+
+ inline void nop (const vsetvl_info &prev ATTRIBUTE_UNUSED,
+ const vsetvl_info &next ATTRIBUTE_UNUSED)
+ {}
+
+ /* modifiers for sew and lmul */
+
+ inline void use_min_of_max_sew (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ prev.set_max_sew (MIN (prev.get_max_sew (), next.get_max_sew ()));
+ }
+ inline void use_next_sew (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ prev.set_sew (next.get_sew ());
+ use_min_of_max_sew (prev, next);
+ }
+ inline void use_max_sew (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ auto max_sew = std::max (prev.get_sew (), next.get_sew ());
+ prev.set_sew (max_sew);
+ use_min_of_max_sew (prev, next);
+ }
+ inline void use_next_sew_lmul (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ use_next_sew (prev, next);
+ prev.set_vlmul (next.get_vlmul ());
+ prev.set_ratio (next.get_ratio ());
+ }
+ inline void use_next_sew_with_prev_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ use_next_sew (prev, next);
+ prev.set_vlmul (calculate_vlmul (next.get_sew (), prev.get_ratio ()));
+ }
+ inline void modify_lmul_with_next_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ prev.set_vlmul (calculate_vlmul (prev.get_sew (), next.get_ratio ()));
+ prev.set_ratio (next.get_ratio ());
+ }
+
+ inline void use_max_sew_and_lmul_with_next_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ prev.set_vlmul (calculate_vlmul (prev.get_sew (), next.get_ratio ()));
+ use_max_sew (prev, next);
+ prev.set_ratio (next.get_ratio ());
+ }
+
+ inline void use_max_sew_and_lmul_with_prev_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ auto max_sew = std::max (prev.get_sew (), next.get_sew ());
+ prev.set_vlmul (calculate_vlmul (max_sew, prev.get_ratio ()));
+ prev.set_sew (max_sew);
+ }
+
+ /* modifiers for tail and mask policy */
+
+ inline void use_tail_policy (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ if (!next.get_ta ())
+ prev.set_ta (next.get_ta ());
+ }
+ inline void use_mask_policy (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ if (!next.get_ma ())
+ prev.set_ma (next.get_ma ());
+ }
+ inline void use_tail_mask_policy (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ use_tail_policy (prev, next);
+ use_mask_policy (prev, next);
+ }
+
+ /* modifiers for avl */
+
+ inline void use_next_avl (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (can_use_next_avl_p (prev, next));
+ prev.update_avl (next);
+ }
+
+ inline void use_next_avl_when_not_equal (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ if (avl_equal_p (prev, next))
+ return;
+ gcc_assert (next.has_non_zero_avl ());
+ use_next_avl (prev, next);
+ }
+
+public:
+ demand_system () : m_avl_def_in (nullptr), m_avl_def_out (nullptr) {}
+
+ void set_avl_in_out_data (sbitmap *m_avl_def_in, sbitmap *m_avl_def_out)
+ {
+ m_avl_def_in = m_avl_def_in;
+ m_avl_def_out = m_avl_def_out;
+ }
+
+ /* Can we move vsetvl info between prev_insn and next_insn safe? */
+ bool avl_vl_unmodified_between_p (insn_info *prev_insn, insn_info *next_insn,
+ const vsetvl_info &info,
+ bool ignore_vl = false)
+ {
+ gcc_assert ((ignore_vl && info.has_nonvlmax_reg_avl ())
+ || (info.has_nonvlmax_reg_avl () || info.has_vl ()));
+
+ gcc_assert (!prev_insn->is_debug_insn () && !next_insn->is_debug_insn ());
+ if (prev_insn->bb () == next_insn->bb ()
+ && prev_insn->compare_with (next_insn) < 0)
+ {
+ for (insn_info *i = next_insn->prev_nondebug_insn (); i != prev_insn;
+ i = i->prev_nondebug_insn ())
+ {
+ // no def amd use of vl
+ if (!ignore_vl && modify_or_use_vl_p (i, info))
+ return false;
+
+ // no def of avl
+ if (modify_avl_p (i, info))
+ return false;
+ }
+ return true;
+ }
+ else
+ {
+ if (!ignore_vl && info.has_vl ())
+ {
+ bitmap live_out = df_get_live_out (prev_insn->bb ()->cfg_bb ());
+ if (bitmap_bit_p (live_out, REGNO (info.get_vl ())))
+ return false;
+ }
+
+ if (info.has_nonvlmax_reg_avl () && m_avl_def_in && m_avl_def_out)
+ {
+ bool has_avl_out = false;
+ unsigned regno = REGNO (info.get_avl ());
+ unsigned expr_id;
+ sbitmap_iterator sbi;
+ EXECUTE_IF_SET_IN_BITMAP (m_avl_def_out[prev_insn->bb ()->index ()],
+ 0, expr_id, sbi)
+ {
+ if (get_regno (expr_id, last_basic_block_for_fn (cfun))
+ != regno)
+ continue;
+ has_avl_out = true;
+ if (!bitmap_bit_p (m_avl_def_in[next_insn->bb ()->index ()],
+ expr_id))
+ return false;
+ }
+ if (!has_avl_out)
+ return false;
+ }
+
+ for (insn_info *i = next_insn; i != next_insn->bb ()->head_insn ();
+ i = i->prev_nondebug_insn ())
+ {
+ // no def amd use of vl
+ if (!ignore_vl && modify_or_use_vl_p (i, info))
+ return false;
+
+ // no def of avl
+ if (modify_avl_p (i, info))
+ return false;
+ }
+
+ for (insn_info *i = prev_insn->bb ()->end_insn (); i != prev_insn;
+ i = i->prev_nondebug_insn ())
+ {
+ // no def amd use of vl
+ if (!ignore_vl && modify_or_use_vl_p (i, info))
+ return false;
+
+ // no def of avl
+ if (modify_avl_p (i, info))
+ return false;
+ }
+ }
+ return true;
+ }
+
+ bool sew_lmul_compatible_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ sew_lmul_demand_type prev_flags = prev.get_sew_lmul_demand ();
+ sew_lmul_demand_type next_flags = next.get_sew_lmul_demand ();
+#define DEF_SEW_LMUL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P,
\
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == sew_lmul_demand_type::PREV_FLAGS
\
+ && next_flags == sew_lmul_demand_type::NEXT_FLAGS)
\
+ return COMPATIBLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool sew_lmul_available_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ sew_lmul_demand_type prev_flags = prev.get_sew_lmul_demand ();
+ sew_lmul_demand_type next_flags = next.get_sew_lmul_demand ();
+#define DEF_SEW_LMUL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P,
\
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == sew_lmul_demand_type::PREV_FLAGS
\
+ && next_flags == sew_lmul_demand_type::NEXT_FLAGS)
\
+ return AVAILABLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ void merge_sew_lmul (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ sew_lmul_demand_type prev_flags = prev.get_sew_lmul_demand ();
+ sew_lmul_demand_type next_flags = next.get_sew_lmul_demand ();
+#define DEF_SEW_LMUL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P,
\
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == sew_lmul_demand_type::PREV_FLAGS
\
+ && next_flags == sew_lmul_demand_type::NEXT_FLAGS)
\
+ {
\
+ gcc_assert (COMPATIBLE_P (prev, next));
\
+ FUSE (prev, next);
\
+ prev.set_sew_lmul_demand (sew_lmul_demand_type::NEW_FLAGS);
\
+ return;
\
+ }
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool policy_compatible_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ policy_demand_type prev_flags = prev.get_policy_demand ();
+ policy_demand_type next_flags = next.get_policy_demand ();
+#define DEF_POLICY_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P,
\
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == policy_demand_type::PREV_FLAGS
\
+ && next_flags == policy_demand_type::NEXT_FLAGS)
\
+ return COMPATIBLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool policy_available_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ policy_demand_type prev_flags = prev.get_policy_demand ();
+ policy_demand_type next_flags = next.get_policy_demand ();
+#define DEF_POLICY_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P,
\
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == policy_demand_type::PREV_FLAGS
\
+ && next_flags == policy_demand_type::NEXT_FLAGS)
\
+ return AVAILABLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ void merge_policy (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ policy_demand_type prev_flags = prev.get_policy_demand ();
+ policy_demand_type next_flags = next.get_policy_demand ();
+#define DEF_POLICY_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P,
\
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == policy_demand_type::PREV_FLAGS
\
+ && next_flags == policy_demand_type::NEXT_FLAGS)
\
+ {
\
+ gcc_assert (COMPATIBLE_P (prev, next));
\
+ FUSE (prev, next);
\
+ prev.set_policy_demand (policy_demand_type::NEW_FLAGS);
\
+ return;
\
+ }
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool avl_compatible_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ avl_demand_type prev_flags = prev.get_avl_demand ();
+ avl_demand_type next_flags = next.get_avl_demand ();
+#define DEF_AVL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P,
\
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == avl_demand_type::PREV_FLAGS
\
+ && next_flags == avl_demand_type::NEXT_FLAGS)
\
+ return COMPATIBLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool avl_available_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ avl_demand_type prev_flags = prev.get_avl_demand ();
+ avl_demand_type next_flags = next.get_avl_demand ();
+#define DEF_AVL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P,
\
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == avl_demand_type::PREV_FLAGS
\
+ && next_flags == avl_demand_type::NEXT_FLAGS)
\
+ return AVAILABLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ void merge_avl (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ avl_demand_type prev_flags = prev.get_avl_demand ();
+ avl_demand_type next_flags = next.get_avl_demand ();
+#define DEF_AVL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P,
\
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == avl_demand_type::PREV_FLAGS
\
+ && next_flags == avl_demand_type::NEXT_FLAGS)
\
+ {
\
+ gcc_assert (COMPATIBLE_P (prev, next));
\
+ FUSE (prev, next);
\
+ prev.set_avl_demand (avl_demand_type::NEW_FLAGS);
\
+ return;
\
+ }
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool compatible_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ bool compatible_p = sew_lmul_compatible_p (prev, next)
+ && policy_compatible_p (prev, next)
+ && avl_compatible_p (prev, next);
+ return compatible_p;
+ }
+
+ bool available_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ bool available_p = sew_lmul_available_p (prev, next)
+ && policy_available_p (prev, next)
+ && avl_available_p (prev, next);
+ gcc_assert (!available_p || compatible_p (prev, next));
+ return available_p;
+ }
+
+ void merge (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (compatible_p (prev, next));
+ merge_sew_lmul (prev, next);
+ merge_policy (prev, next);
+ merge_avl (prev, next);
+ gcc_assert (available_p (prev, next));
+ }
+};
+
vector_infos_manager::vector_infos_manager ()
{
vector_edge_list = nullptr;
--
2.36.3
