From: Gregory Etelson <getel...@nvidia.com>
The MLX5 PMD expands flow rule with the RSS action in the
non-template environment.
The patch adds RSS flow rule expansion for legacy flow rules in
the template setup.
Signed-off-by: Gregory Etelson <getel...@nvidia.com>
Acked-by: Dariusz Sosnowski <dsosnow...@nvidia.com>
---
drivers/net/mlx5/hws/mlx5dr_definer.c | 2 +
drivers/net/mlx5/meson.build | 1 +
drivers/net/mlx5/mlx5.c | 4 +
drivers/net/mlx5/mlx5.h | 6 +-
drivers/net/mlx5/mlx5_flow.h | 36 +-
drivers/net/mlx5/mlx5_flow_hw.c | 65 +--
drivers/net/mlx5/mlx5_nta_rss.c | 564 ++++++++++++++++++++++++++
7 files changed, 646 insertions(+), 32 deletions(-)
create mode 100644 drivers/net/mlx5/mlx5_nta_rss.c
diff --git a/drivers/net/mlx5/hws/mlx5dr_definer.c
b/drivers/net/mlx5/hws/mlx5dr_definer.c
index 4be9012e25f..9ebda9267da 100644
--- a/drivers/net/mlx5/hws/mlx5dr_definer.c
+++ b/drivers/net/mlx5/hws/mlx5dr_definer.c
@@ -391,6 +391,8 @@ mlx5dr_definer_ptype_l4_set(struct mlx5dr_definer_fc *fc,
l4_type = STE_UDP;
else if (packet_type == (inner ? RTE_PTYPE_INNER_L4_ICMP :
RTE_PTYPE_L4_ICMP))
l4_type = STE_ICMP;
+ else if (packet_type == RTE_PTYPE_TUNNEL_ESP)
+ l4_type = STE_ESP;
DR_SET(tag, l4_type, fc->byte_off, fc->bit_off, fc->bit_mask);
}
diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index d705fe21bbc..b279ddf47c9 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -42,6 +42,7 @@ sources = files(
'mlx5_vlan.c',
'mlx5_utils.c',
'mlx5_devx.c',
+ 'mlx5_nta_rss.c',
)
if is_linux
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index e41b1e82d7f..997b02c5717 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -2365,6 +2365,10 @@ mlx5_dev_close(struct rte_eth_dev *dev)
claim_zero(mlx5_geneve_tlv_options_destroy(priv->tlv_options,
sh->phdev));
priv->tlv_options = NULL;
}
+ if (priv->ptype_rss_groups) {
+ mlx5_ipool_destroy(priv->ptype_rss_groups);
+ priv->ptype_rss_groups = NULL;
+ }
#endif
if (priv->rxq_privs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 159ea10018e..c9a3837bd2b 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1204,6 +1204,10 @@ struct mlx5_flow_tbl_resource {
#define MLX5_MAX_TABLES_EXTERNAL MLX5_FLOW_TABLE_LEVEL_POLICY
#define MLX5_FLOW_TABLE_HWS_POLICY (MLX5_MAX_TABLES - 10)
#define MLX5_MAX_TABLES_FDB UINT16_MAX
+#define MLX5_FLOW_TABLE_PTYPE_RSS_NUM 1024
+#define MLX5_FLOW_TABLE_PTYPE_RSS_LAST (MLX5_MAX_TABLES - 11)
+#define MLX5_FLOW_TABLE_PTYPE_RSS_BASE \
+(1 + MLX5_FLOW_TABLE_PTYPE_RSS_LAST - MLX5_FLOW_TABLE_PTYPE_RSS_NUM)
#define MLX5_FLOW_TABLE_FACTOR 10
/* ID generation structure. */
@@ -2040,7 +2044,7 @@ struct mlx5_priv {
* Todo: consider to add *_MAX macro.
*/
struct mlx5dr_action *action_nat64[MLX5DR_TABLE_TYPE_MAX][2];
-
+ struct mlx5_indexed_pool *ptype_rss_groups;
#endif
struct rte_eth_dev *shared_host; /* Host device for HW steering. */
RTE_ATOMIC(uint16_t) shared_refcnt; /* HW steering host reference
counter. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 582d5754cd5..27b746629f3 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -489,6 +489,9 @@ enum mlx5_feature_name {
RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_NONFRAG_IPV4_UDP | \
RTE_ETH_RSS_NONFRAG_IPV4_OTHER)
+/* Valid L4 RSS types */
+#define MLX5_L4_RSS_TYPES (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)
+
/* IBV hash source bits for IPV4. */
#define MLX5_IPV4_IBV_RX_HASH (IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4)
@@ -1318,6 +1321,8 @@ enum {
#define MLX5_DR_RULE_SIZE 72
+SLIST_HEAD(mlx5_nta_rss_flow_head, rte_flow_hw);
+
/** HWS non template flow data. */
struct rte_flow_nt2hws {
/** BWC rule pointer. */
@@ -1330,7 +1335,10 @@ struct rte_flow_nt2hws {
struct mlx5_flow_dv_modify_hdr_resource *modify_hdr;
/** Encap/decap index. */
uint32_t rix_encap_decap;
-};
+ uint8_t chaned_flow;
+ /** Chain NTA flows. */
+ SLIST_ENTRY(rte_flow_hw) next;
+} __rte_packed;
/** HWS flow struct. */
struct rte_flow_hw {
@@ -3574,7 +3582,6 @@ flow_hw_get_ipv6_route_ext_mod_id_from_ctx(void *dr_ctx,
uint8_t idx)
#endif
return 0;
}
-
void
mlx5_indirect_list_handles_release(struct rte_eth_dev *dev);
#ifdef HAVE_MLX5_HWS_SUPPORT
@@ -3587,6 +3594,31 @@ mlx5_destroy_legacy_indirect(struct rte_eth_dev *dev,
void
mlx5_hw_decap_encap_destroy(struct rte_eth_dev *dev,
struct mlx5_indirect_list *reformat);
+int
+flow_hw_create_flow(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ uint64_t item_flags, uint64_t action_flags, bool external,
+ struct rte_flow_hw **flow, struct rte_flow_error *error);
+void
+flow_hw_destroy(struct rte_eth_dev *dev, struct rte_flow_hw *flow);
+void
+flow_hw_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ uintptr_t flow_idx);
+const struct rte_flow_action_rss *
+flow_nta_locate_rss(struct rte_eth_dev *dev,
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error);
+struct rte_flow_hw *
+flow_nta_handle_rss(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ const struct rte_flow_action_rss *rss_conf,
+ uint64_t item_flags, uint64_t action_flags,
+ bool external, enum mlx5_flow_type flow_type,
+ struct rte_flow_error *error);
extern const struct rte_flow_action_raw_decap empty_decap;
extern const struct rte_flow_item_ipv6 nic_ipv6_mask;
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index dd7d1fe8a28..19c7f810285 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -507,7 +507,7 @@ flow_hw_hashfields_set(struct mlx5_flow_rss_desc *rss_desc,
fields |= IBV_RX_HASH_IPSEC_SPI;
if (rss_inner)
fields |= IBV_RX_HASH_INNER;
- *hash_fields = fields;
+ *hash_fields |= fields;
}
/**
@@ -770,9 +770,7 @@ flow_hw_jump_release(struct rte_eth_dev *dev, struct
mlx5_hw_jump_action *jump)
static inline struct mlx5_hrxq*
flow_hw_tir_action_register(struct rte_eth_dev *dev,
uint32_t hws_flags,
- const struct rte_flow_action *action,
- uint64_t item_flags,
- bool is_template)
+ const struct rte_flow_action *action)
{
struct mlx5_flow_rss_desc rss_desc = {
.hws_flags = hws_flags,
@@ -795,10 +793,7 @@ flow_hw_tir_action_register(struct rte_eth_dev *dev,
rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
rss_desc.types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
rss_desc.symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func);
- if (is_template)
- flow_hw_hashfields_set(&rss_desc,
&rss_desc.hash_fields);
- else
- flow_dv_hashfields_set(item_flags, &rss_desc,
&rss_desc.hash_fields);
+ flow_hw_hashfields_set(&rss_desc, &rss_desc.hash_fields);
flow_dv_action_rss_l34_hash_adjust(rss->types,
&rss_desc.hash_fields);
if (rss->level > 1) {
@@ -2541,9 +2536,8 @@ __flow_hw_translate_actions_template(struct rte_eth_dev
*dev,
((const struct rte_flow_action_queue *)
masks->conf)->index) {
acts->tir = flow_hw_tir_action_register
- (dev,
- mlx5_hw_act_flag[!!attr->group][type],
- actions, 0, true);
+ (dev, mlx5_hw_act_flag[!!attr->group][type],
+ actions);
if (!acts->tir)
goto err;
acts->rule_acts[dr_pos].action =
@@ -2557,9 +2551,8 @@ __flow_hw_translate_actions_template(struct rte_eth_dev
*dev,
case RTE_FLOW_ACTION_TYPE_RSS:
if (actions->conf && masks->conf) {
acts->tir = flow_hw_tir_action_register
- (dev,
- mlx5_hw_act_flag[!!attr->group][type],
- actions, 0, true);
+ (dev, mlx5_hw_act_flag[!!attr->group][type],
+ actions);
if (!acts->tir)
goto err;
acts->rule_acts[dr_pos].action =
@@ -3449,11 +3442,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
break;
case RTE_FLOW_ACTION_TYPE_RSS:
case RTE_FLOW_ACTION_TYPE_QUEUE:
- hrxq = flow_hw_tir_action_register(dev,
- ft_flag,
- action,
- item_flags,
- !flow->nt_rule);
+ hrxq = flow_hw_tir_action_register(dev, ft_flag,
action);
if (!hrxq)
goto error;
rule_acts[act_data->action_dst].action = hrxq->action;
@@ -13400,7 +13389,7 @@ static int flow_hw_apply(struct rte_eth_dev *dev
__rte_unused,
* @return
* 0 on success, negative errno value otherwise and rte_errno set.
*/
-static int
+int
flow_hw_create_flow(struct rte_eth_dev *dev, enum mlx5_flow_type type,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@@ -13513,7 +13502,7 @@ flow_hw_create_flow(struct rte_eth_dev *dev, enum
mlx5_flow_type type,
}
#endif
-static void
+void
flow_hw_destroy(struct rte_eth_dev *dev, struct rte_flow_hw *flow)
{
int ret;
@@ -13568,18 +13557,23 @@ flow_hw_destroy(struct rte_eth_dev *dev, struct
rte_flow_hw *flow)
* @param[in] flow_addr
* Address of flow to destroy.
*/
-static void flow_hw_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type
type,
- uintptr_t flow_addr)
+void
+flow_hw_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ uintptr_t flow_addr)
{
struct mlx5_priv *priv = dev->data->dev_private;
- /* Get flow via idx */
struct rte_flow_hw *flow = (struct rte_flow_hw *)flow_addr;
+ struct mlx5_nta_rss_flow_head head = { .slh_first = flow };
- if (!flow)
+ if (flow->nt2hws->chaned_flow)
return;
- flow_hw_destroy(dev, flow);
- /* Release flow memory by idx */
- mlx5_ipool_free(priv->flows[type], flow->idx);
+ while (!SLIST_EMPTY(&head)) {
+ flow = SLIST_FIRST(&head);
+ SLIST_REMOVE_HEAD(&head, nt2hws->next);
+ flow_hw_destroy(dev, flow);
+ /* Release flow memory by idx */
+ mlx5_ipool_free(priv->flows[type], flow->idx);
+ }
}
#endif
@@ -13617,6 +13611,19 @@ static uintptr_t flow_hw_list_create(struct
rte_eth_dev *dev,
uint64_t item_flags = flow_hw_matching_item_flags_get(items);
uint64_t action_flags = flow_hw_action_flags_get(actions, error);
+
+ if (action_flags & MLX5_FLOW_ACTION_RSS) {
+ const struct rte_flow_action_rss
+ *rss_conf = flow_nta_locate_rss(dev, actions, error);
+ flow = flow_nta_handle_rss(dev, attr, items, actions, rss_conf,
+ item_flags, action_flags, external,
+ type, error);
+ if (flow)
+ return (uintptr_t)flow;
+ if (error->type != RTE_FLOW_ERROR_TYPE_NONE)
+ return 0;
+ /* Fall Through to non-expanded RSS flow */
+ }
/*TODO: Handle split/expand to num_flows. */
/* Create single flow. */
@@ -13776,7 +13783,7 @@ mirror_format_tir(struct rte_eth_dev *dev,
table_type = get_mlx5dr_table_type(&table_cfg->attr.flow_attr);
hws_flags = mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_NONE_ROOT][table_type];
- tir_ctx = flow_hw_tir_action_register(dev, hws_flags, action, 0, true);
+ tir_ctx = flow_hw_tir_action_register(dev, hws_flags, action);
if (!tir_ctx)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
diff --git a/drivers/net/mlx5/mlx5_nta_rss.c b/drivers/net/mlx5/mlx5_nta_rss.c
new file mode 100644
index 00000000000..1f0085ff069
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_nta_rss.c
@@ -0,0 +1,564 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2024 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_flow.h>
+
+#include <mlx5_malloc.h>
+#include "mlx5.h"
+#include "mlx5_defs.h"
+#include "mlx5_flow.h"
+#include "mlx5_rx.h"
+#include "rte_common.h"
+
+#ifdef HAVE_MLX5_HWS_SUPPORT
+
+struct mlx5_nta_rss_ctx {
+ struct rte_eth_dev *dev;
+ struct rte_flow_attr *attr;
+ struct rte_flow_item *pattern;
+ struct rte_flow_action *actions;
+ const struct rte_flow_action_rss *rss_conf;
+ struct rte_flow_error *error;
+ struct mlx5_nta_rss_flow_head *head;
+ uint64_t pattern_flags;
+ enum mlx5_flow_type flow_type;
+ bool external;
+};
+
+#define MLX5_RSS_PTYPE_ITEM_INDEX 0
+#ifdef MLX5_RSS_PTYPE_DEBUG
+#define MLX5_RSS_PTYPE_ACTION_INDEX 1
+#else
+#define MLX5_RSS_PTYPE_ACTION_INDEX 0
+#endif
+
+#define MLX5_RSS_PTYPE_ITEMS_NUM (MLX5_RSS_PTYPE_ITEM_INDEX + 2)
+#define MLX5_RSS_PTYPE_ACTIONS_NUM (MLX5_RSS_PTYPE_ACTION_INDEX + 2)
+
+static int
+mlx5_nta_ptype_rss_flow_create(struct mlx5_nta_rss_ctx *ctx,
+ uint32_t ptype, uint64_t rss_type)
+{
+ int ret;
+ struct rte_flow_hw *flow;
+ struct rte_flow_item_ptype *ptype_spec = (void *)(uintptr_t)
+
ctx->pattern[MLX5_RSS_PTYPE_ITEM_INDEX].spec;
+ struct rte_flow_action_rss *rss_conf = (void *)(uintptr_t)
+
ctx->actions[MLX5_RSS_PTYPE_ACTION_INDEX].conf;
+ bool dbg_log = rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG);
+ uint32_t mark_id = 0;
+#ifdef MLX5_RSS_PTYPE_DEBUG
+ struct rte_flow_action_mark *mark = (void *)(uintptr_t)
+ ctx->actions[MLX5_RSS_PTYPE_ACTION_INDEX -
1].conf;
+
+ /*
+ * Inner L3 and L4 ptype values are too large for 24bit mark
+ */
+ mark->id =
+ ((ptype & (RTE_PTYPE_INNER_L3_MASK | RTE_PTYPE_INNER_L4_MASK))
== ptype) ?
+ ptype >> 20 : ptype;
+ mark_id = mark->id;
+ dbg_log = true;
+#endif
+ ptype_spec->packet_type = ptype;
+ rss_conf->types = rss_type;
+ ret = flow_hw_create_flow(ctx->dev, MLX5_FLOW_TYPE_GEN, ctx->attr,
+ ctx->pattern, ctx->actions,
+ MLX5_FLOW_ITEM_PTYPE, MLX5_FLOW_ACTION_RSS,
+ ctx->external, &flow, ctx->error);
+ if (flow) {
+ SLIST_INSERT_HEAD(ctx->head, flow, nt2hws->next);
+ if (dbg_log) {
+ DRV_LOG(NOTICE,
+ "PTYPE RSS: group %u ptype spec %#x rss types
%#lx mark %#x\n",
+ ctx->attr->group, ptype_spec->packet_type,
+ (unsigned long)rss_conf->types, mark_id);
+ }
+ }
+ return ret;
+}
+
+/*
+ * Call conditions:
+ * * Flow pattern did not include outer L3 and L4 items.
+ * * RSS configuration had L3 hash types.
+ */
+static struct rte_flow_hw *
+mlx5_hw_rss_expand_l3(struct mlx5_nta_rss_ctx *rss_ctx)
+{
+ int ret;
+ int ptype_ip4, ptype_ip6;
+ uint64_t rss_types = rte_eth_rss_hf_refine(rss_ctx->rss_conf->types);
+
+ if (rss_ctx->rss_conf->level < 2) {
+ ptype_ip4 = RTE_PTYPE_L3_IPV4;
+ ptype_ip6 = RTE_PTYPE_L3_IPV6;
+ } else {
+ ptype_ip4 = RTE_PTYPE_INNER_L3_IPV4;
+ ptype_ip6 = RTE_PTYPE_INNER_L3_IPV6;
+ }
+ if (rss_types & MLX5_IPV4_LAYER_TYPES) {
+ ret = mlx5_nta_ptype_rss_flow_create
+ (rss_ctx, ptype_ip4, (rss_types &
~MLX5_IPV6_LAYER_TYPES));
+ if (ret)
+ goto error;
+ }
+ if (rss_types & MLX5_IPV6_LAYER_TYPES) {
+ ret = mlx5_nta_ptype_rss_flow_create
+ (rss_ctx, ptype_ip6, rss_types &
~MLX5_IPV4_LAYER_TYPES);
+ if (ret)
+ goto error;
+ }
+ return SLIST_FIRST(rss_ctx->head);
+
+error:
+ flow_hw_list_destroy(rss_ctx->dev, rss_ctx->flow_type,
+ (uintptr_t)SLIST_FIRST(rss_ctx->head));
+ return NULL;
+}
+
+static void
+mlx5_nta_rss_expand_l3_l4(struct mlx5_nta_rss_ctx *rss_ctx,
+ uint64_t rss_types, uint64_t rss_l3_types)
+{
+ int ret;
+ int ptype_l3, ptype_l4_udp, ptype_l4_tcp, ptype_l4_esp = 0;
+ uint64_t rss = rss_types &
+ ~(rss_l3_types == MLX5_IPV4_LAYER_TYPES ?
+ MLX5_IPV6_LAYER_TYPES : MLX5_IPV4_LAYER_TYPES);
+
+
+ if (rss_ctx->rss_conf->level < 2) {
+ ptype_l3 = rss_l3_types == MLX5_IPV4_LAYER_TYPES ?
+ RTE_PTYPE_L3_IPV4 : RTE_PTYPE_L3_IPV6;
+ ptype_l4_esp = RTE_PTYPE_TUNNEL_ESP;
+ ptype_l4_udp = RTE_PTYPE_L4_UDP;
+ ptype_l4_tcp = RTE_PTYPE_L4_TCP;
+ } else {
+ ptype_l3 = rss_l3_types == MLX5_IPV4_LAYER_TYPES ?
+ RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_INNER_L3_IPV6;
+ ptype_l4_udp = RTE_PTYPE_INNER_L4_UDP;
+ ptype_l4_tcp = RTE_PTYPE_INNER_L4_TCP;
+ }
+ if (rss_types & RTE_ETH_RSS_ESP) {
+ ret = mlx5_nta_ptype_rss_flow_create
+ (rss_ctx, ptype_l3 | ptype_l4_esp,
+ rss & ~(RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP));
+ if (ret)
+ goto error;
+ }
+ if (rss_types & RTE_ETH_RSS_UDP) {
+ ret = mlx5_nta_ptype_rss_flow_create(rss_ctx,
+ ptype_l3 | ptype_l4_udp,
+ rss & ~(RTE_ETH_RSS_ESP | RTE_ETH_RSS_TCP));
+ if (ret)
+ goto error;
+ }
+ if (rss_types & RTE_ETH_RSS_TCP) {
+ ret = mlx5_nta_ptype_rss_flow_create(rss_ctx,
+ ptype_l3 | ptype_l4_tcp,
+ rss & ~(RTE_ETH_RSS_ESP | RTE_ETH_RSS_UDP));
+ if (ret)
+ goto error;
+ }
+ return;
+error:
+ flow_hw_list_destroy(rss_ctx->dev, rss_ctx->flow_type,
+ (uintptr_t)SLIST_FIRST(rss_ctx->head));
+}
+
+/*
+ * Call conditions:
+ * * Flow pattern did not include L4 item.
+ * * RSS configuration had L4 hash types.
+ */
+static struct rte_flow_hw *
+mlx5_hw_rss_expand_l4(struct mlx5_nta_rss_ctx *rss_ctx)
+{
+ uint64_t rss_types = rte_eth_rss_hf_refine(rss_ctx->rss_conf->types);
+ uint64_t l3_item = rss_ctx->pattern_flags &
+ (rss_ctx->rss_conf->level < 2 ?
+ MLX5_FLOW_LAYER_OUTER_L3 :
MLX5_FLOW_LAYER_INNER_L3);
+
+ if (l3_item) {
+ /*
+ * Outer L3 header was present in the original pattern.
+ * Expand L4 level only.
+ */
+ if (l3_item & MLX5_FLOW_LAYER_L3_IPV4)
+ mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types,
MLX5_IPV4_LAYER_TYPES);
+ else
+ mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types,
MLX5_IPV6_LAYER_TYPES);
+ } else {
+ if (rss_types & (MLX5_IPV4_LAYER_TYPES |
MLX5_IPV6_LAYER_TYPES)) {
+ mlx5_hw_rss_expand_l3(rss_ctx);
+ /*
+ * No outer L3 item in application flow pattern.
+ * RSS hash types are L3 and L4.
+ * ** Expand L3 according to RSS configuration and L4.
+ */
+ if (rss_types & MLX5_IPV4_LAYER_TYPES)
+ mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types,
+
MLX5_IPV4_LAYER_TYPES);
+ if (rss_types & MLX5_IPV6_LAYER_TYPES)
+ mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types,
+
MLX5_IPV6_LAYER_TYPES);
+ } else {
+ /*
+ * No outer L3 item in application flow pattern,
+ * RSS hash type is L4 only.
+ */
+ mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types,
+ MLX5_IPV4_LAYER_TYPES);
+ mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types,
+ MLX5_IPV6_LAYER_TYPES);
+ }
+ }
+ return SLIST_EMPTY(rss_ctx->head) ? NULL : SLIST_FIRST(rss_ctx->head);
+}
+
+static struct mlx5_indexed_pool *
+mlx5_nta_ptype_ipool_create(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_indexed_pool_config ipool_cfg = {
+ .size = 1,
+ .trunk_size = 32,
+ .grow_trunk = 5,
+ .grow_shift = 1,
+ .need_lock = 1,
+ .release_mem_en = !!priv->sh->config.reclaim_mode,
+ .malloc = mlx5_malloc,
+ .max_idx = MLX5_FLOW_TABLE_PTYPE_RSS_NUM,
+ .free = mlx5_free,
+ .type = "mlx5_nta_ptype_rss"
+ };
+ return mlx5_ipool_create(&ipool_cfg);
+}
+
+static void
+mlx5_hw_release_rss_ptype_group(struct rte_eth_dev *dev, uint32_t group)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ if (!priv->ptype_rss_groups)
+ return;
+ mlx5_ipool_free(priv->ptype_rss_groups, group);
+}
+
+static uint32_t
+mlx5_hw_get_rss_ptype_group(struct rte_eth_dev *dev)
+{
+ void *obj;
+ uint32_t idx = 0;
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ if (!priv->ptype_rss_groups) {
+ priv->ptype_rss_groups = mlx5_nta_ptype_ipool_create(dev);
+ if (!priv->ptype_rss_groups) {
+ DRV_LOG(DEBUG, "PTYPE RSS: failed to allocate groups
pool");
+ return 0;
+ }
+ }
+ obj = mlx5_ipool_malloc(priv->ptype_rss_groups, &idx);
+ if (!obj) {
+ DRV_LOG(DEBUG, "PTYPE RSS: failed to fetch ptype group from the
pool");
+ return 0;
+ }
+ return idx + MLX5_FLOW_TABLE_PTYPE_RSS_BASE;
+}
+
+static struct rte_flow_hw *
+mlx5_hw_rss_ptype_create_miss_flow(struct rte_eth_dev *dev,
+ const struct rte_flow_action_rss *rss_conf,
+ uint32_t ptype_group, bool external,
+ struct rte_flow_error *error)
+{
+ struct rte_flow_hw *flow = NULL;
+ const struct rte_flow_attr miss_attr = {
+ .ingress = 1,
+ .group = ptype_group,
+ .priority = 3
+ };
+ const struct rte_flow_item miss_pattern[2] = {
+ [0] = { .type = RTE_FLOW_ITEM_TYPE_ETH },
+ [1] = { .type = RTE_FLOW_ITEM_TYPE_END }
+ };
+ struct rte_flow_action miss_actions[] = {
+#ifdef MLX5_RSS_PTYPE_DEBUG
+ [MLX5_RSS_PTYPE_ACTION_INDEX - 1] = {
+ .type = RTE_FLOW_ACTION_TYPE_MARK,
+ .conf = &(const struct rte_flow_action_mark){.id =
0xfac}
+ },
+#endif
+ [MLX5_RSS_PTYPE_ACTION_INDEX] = {
+ .type = RTE_FLOW_ACTION_TYPE_RSS,
+ .conf = rss_conf
+ },
+ [MLX5_RSS_PTYPE_ACTION_INDEX + 1] = { .type =
RTE_FLOW_ACTION_TYPE_END }
+ };
+
+ flow_hw_create_flow(dev, MLX5_FLOW_TYPE_GEN, &miss_attr,
+ miss_pattern, miss_actions, 0, MLX5_FLOW_ACTION_RSS,
+ external, &flow, error);
+ return flow;
+}
+
+static struct rte_flow_hw *
+mlx5_hw_rss_ptype_create_base_flow(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item pattern[],
+ const struct rte_flow_action orig_actions[],
+ uint32_t ptype_group, uint64_t item_flags,
+ uint64_t action_flags, bool external,
+ enum mlx5_flow_type flow_type,
+ struct rte_flow_error *error)
+{
+ int i = 0;
+ struct rte_flow_hw *flow = NULL;
+ struct rte_flow_action actions[MLX5_HW_MAX_ACTS];
+ enum mlx5_indirect_type indirect_type;
+
+ do {
+ switch (orig_actions[i].type) {
+ case RTE_FLOW_ACTION_TYPE_INDIRECT:
+ indirect_type = (typeof(indirect_type))
+ MLX5_INDIRECT_ACTION_TYPE_GET
+ (orig_actions[i].conf);
+ if (indirect_type != MLX5_INDIRECT_ACTION_TYPE_RSS) {
+ actions[i] = orig_actions[i];
+ break;
+ }
+ /* Fall through */
+ case RTE_FLOW_ACTION_TYPE_RSS:
+ actions[i].type = RTE_FLOW_ACTION_TYPE_JUMP;
+ actions[i].conf = &(const struct rte_flow_action_jump) {
+ .group = ptype_group
+ };
+ break;
+ default:
+ actions[i] = orig_actions[i];
+ }
+
+ } while (actions[i++].type != RTE_FLOW_ACTION_TYPE_END);
+ action_flags &= ~MLX5_FLOW_ACTION_RSS;
+ action_flags |= MLX5_FLOW_ACTION_JUMP;
+ flow_hw_create_flow(dev, flow_type, attr, pattern, actions,
+ item_flags, action_flags, external, &flow, error);
+ return flow;
+}
+
+const struct rte_flow_action_rss *
+flow_nta_locate_rss(struct rte_eth_dev *dev,
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error)
+{
+ const struct rte_flow_action *a;
+ const struct rte_flow_action_rss *rss_conf = NULL;
+
+ for (a = actions; a->type != RTE_FLOW_ACTION_TYPE_END; a++) {
+ if (a->type == RTE_FLOW_ACTION_TYPE_RSS) {
+ rss_conf = a->conf;
+ break;
+ }
+ if (a->type == RTE_FLOW_ACTION_TYPE_INDIRECT &&
+ MLX5_INDIRECT_ACTION_TYPE_GET(a->conf) ==
+ MLX5_INDIRECT_ACTION_TYPE_RSS) {
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_shared_action_rss *shared_rss;
+ uint32_t handle = (uint32_t)(uintptr_t)a->conf;
+
+ shared_rss = mlx5_ipool_get
+ (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
+ MLX5_INDIRECT_ACTION_IDX_GET(handle));
+ if (!shared_rss) {
+ rte_flow_error_set(error, EINVAL,
+
RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+ a->conf, "invalid shared RSS
handle");
+ return NULL;
+ }
+ rss_conf = &shared_rss->origin;
+ break;
+ }
+ }
+ if (a->type == RTE_FLOW_ACTION_TYPE_END) {
+ rte_flow_error_set(error, 0, RTE_FLOW_ERROR_TYPE_NONE, NULL,
NULL);
+ return NULL;
+ }
+ return rss_conf;
+}
+
+static __rte_always_inline void
+mlx5_nta_rss_init_ptype_ctx(struct mlx5_nta_rss_ctx *rss_ctx,
+ struct rte_eth_dev *dev,
+ struct rte_flow_attr *ptype_attr,
+ struct rte_flow_item *ptype_pattern,
+ struct rte_flow_action *ptype_actions,
+ const struct rte_flow_action_rss *rss_conf,
+ struct mlx5_nta_rss_flow_head *head,
+ struct rte_flow_error *error,
+ uint64_t item_flags,
+ enum mlx5_flow_type flow_type, bool external)
+{
+ rss_ctx->dev = dev;
+ rss_ctx->attr = ptype_attr;
+ rss_ctx->pattern = ptype_pattern;
+ rss_ctx->actions = ptype_actions;
+ rss_ctx->rss_conf = rss_conf;
+ rss_ctx->error = error;
+ rss_ctx->head = head;
+ rss_ctx->pattern_flags = item_flags;
+ rss_ctx->flow_type = flow_type;
+ rss_ctx->external = external;
+}
+
+/*
+ * MLX5 HW hashes IPv4 and IPv6 L3 headers and UDP, TCP, ESP L4 headers.
+ * RSS expansion is required when RSS action was configured to hash
+ * network protocol that was not mentioned in flow pattern.
+ *
+ */
+#define MLX5_PTYPE_RSS_OUTER_MASK (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV6 | \
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP | \
+ RTE_PTYPE_TUNNEL_ESP)
+#define MLX5_PTYPE_RSS_INNER_MASK (RTE_PTYPE_INNER_L3_IPV4 |
RTE_PTYPE_INNER_L3_IPV6 | \
+ RTE_PTYPE_INNER_L4_TCP |
RTE_PTYPE_INNER_L4_UDP)
+
+struct rte_flow_hw *
+flow_nta_handle_rss(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ const struct rte_flow_action_rss *rss_conf,
+ uint64_t item_flags, uint64_t action_flags,
+ bool external, enum mlx5_flow_type flow_type,
+ struct rte_flow_error *error)
+{
+ struct rte_flow_hw *rss_base = NULL, *rss_next = NULL, *rss_miss = NULL;
+ struct rte_flow_action_rss ptype_rss_conf;
+ struct mlx5_nta_rss_ctx rss_ctx;
+ uint64_t rss_types = rte_eth_rss_hf_refine(rss_conf->types);
+ bool inner_rss = rss_conf->level > 1;
+ bool outer_rss = !inner_rss;
+ bool l3_item = (outer_rss && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
+ (inner_rss && (item_flags & MLX5_FLOW_LAYER_INNER_L3));
+ bool l4_item = (outer_rss && (item_flags & MLX5_FLOW_LAYER_OUTER_L4)) ||
+ (inner_rss && (item_flags & MLX5_FLOW_LAYER_INNER_L4));
+ bool l3_hash = rss_types & (MLX5_IPV4_LAYER_TYPES |
MLX5_IPV6_LAYER_TYPES);
+ bool l4_hash = rss_types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP |
RTE_ETH_RSS_ESP);
+ struct mlx5_nta_rss_flow_head expansion_head =
SLIST_HEAD_INITIALIZER(0);
+ struct rte_flow_attr ptype_attr = {
+ .ingress = 1
+ };
+ struct rte_flow_item_ptype ptype_spec = { .packet_type = 0 };
+ const struct rte_flow_item_ptype ptype_mask = {
+ .packet_type = outer_rss ?
+ MLX5_PTYPE_RSS_OUTER_MASK : MLX5_PTYPE_RSS_INNER_MASK
+ };
+ struct rte_flow_item ptype_pattern[MLX5_RSS_PTYPE_ITEMS_NUM] = {
+ [MLX5_RSS_PTYPE_ITEM_INDEX] = {
+ .type = RTE_FLOW_ITEM_TYPE_PTYPE,
+ .spec = &ptype_spec,
+ .mask = &ptype_mask
+ },
+ [MLX5_RSS_PTYPE_ITEM_INDEX + 1] = { .type =
RTE_FLOW_ITEM_TYPE_END }
+ };
+ struct rte_flow_action ptype_actions[MLX5_RSS_PTYPE_ACTIONS_NUM] = {
+#ifdef MLX5_RSS_PTYPE_DEBUG
+ [MLX5_RSS_PTYPE_ACTION_INDEX - 1] = {
+ .type = RTE_FLOW_ACTION_TYPE_MARK,
+ .conf = &(const struct rte_flow_action_mark) {.id = 101}
+ },
+#endif
+ [MLX5_RSS_PTYPE_ACTION_INDEX] = {
+ .type = RTE_FLOW_ACTION_TYPE_RSS,
+ .conf = &ptype_rss_conf
+ },
+ [MLX5_RSS_PTYPE_ACTION_INDEX + 1] = { .type =
RTE_FLOW_ACTION_TYPE_END }
+ };
+
+ if (l4_item) {
+ /*
+ * Original flow pattern extended up to L4 level.
+ * L4 is the maximal expansion level.
+ * Original pattern does not need expansion.
+ */
+ rte_flow_error_set(error, 0, RTE_FLOW_ERROR_TYPE_NONE, NULL,
NULL);
+ return NULL;
+ }
+ if (!l4_hash) {
+ if (!l3_hash) {
+ /*
+ * RSS action was not configured to hash L3 or L4.
+ * No expansion needed.
+ */
+ rte_flow_error_set(error, 0, RTE_FLOW_ERROR_TYPE_NONE,
NULL, NULL);
+ return NULL;
+ }
+ if (l3_item) {
+ /*
+ * Original flow pattern extended up to L3 level.
+ * RSS action was not set for L4 hash.
+ * Original pattern does not need expansion.
+ */
+ rte_flow_error_set(error, 0, RTE_FLOW_ERROR_TYPE_NONE,
NULL, NULL);
+ return NULL;
+ }
+ }
+ /* Create RSS expansions in dedicated PTYPE flow group */
+ ptype_attr.group = mlx5_hw_get_rss_ptype_group(dev);
+ if (!ptype_attr.group) {
+ rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+ NULL, "cannot get RSS PTYPE group");
+ return NULL;
+ }
+ ptype_rss_conf = *rss_conf;
+ mlx5_nta_rss_init_ptype_ctx(&rss_ctx, dev, &ptype_attr, ptype_pattern,
+ ptype_actions, rss_conf, &expansion_head,
+ error, item_flags, flow_type, external);
+ rss_miss = mlx5_hw_rss_ptype_create_miss_flow(dev, rss_conf,
ptype_attr.group,
+ external, error);
+ if (!rss_miss)
+ goto error;
+ if (l4_hash) {
+ rss_next = mlx5_hw_rss_expand_l4(&rss_ctx);
+ if (!rss_next)
+ goto error;
+ } else if (l3_hash) {
+ rss_next = mlx5_hw_rss_expand_l3(&rss_ctx);
+ if (!rss_next)
+ goto error;
+ }
+ rss_base = mlx5_hw_rss_ptype_create_base_flow(dev, attr, items, actions,
+ ptype_attr.group,
item_flags,
+ action_flags, external,
+ flow_type, error);
+ if (!rss_base)
+ goto error;
+ SLIST_INSERT_HEAD(&expansion_head, rss_miss, nt2hws->next);
+ SLIST_INSERT_HEAD(&expansion_head, rss_base, nt2hws->next);
+ /**
+ * PMD must return to application a reference to the base flow.
+ * This way RSS expansion could work with counter, meter and other
+ * flow actions.
+ */
+ MLX5_ASSERT(rss_base == SLIST_FIRST(&expansion_head));
+ rss_next = SLIST_NEXT(rss_base, nt2hws->next);
+ while (rss_next) {
+ rss_next->nt2hws->chaned_flow = 1;
+ rss_next = SLIST_NEXT(rss_next, nt2hws->next);
+ }
+ return SLIST_FIRST(&expansion_head);
+
+error:
+ if (rss_miss)
+ flow_hw_list_destroy(dev, flow_type, (uintptr_t)rss_miss);
+ if (rss_next)
+ flow_hw_list_destroy(dev, flow_type, (uintptr_t)rss_next);
+ mlx5_hw_release_rss_ptype_group(dev, ptype_attr.group);
+ return NULL;
+}
+
+#endif
+
--
2.21.0
