From: Shai Brandes <shaib...@amazon.com>
ENA-express is powered by AWS scalable reliable datagram (SRD)
technology. SRD is a high performance network transport protocol
that uses dynamic routing to increase throughput and minimize
tail latency.
The driver expose the following ENA-express metrics via xstats:
* ena_srd_mode – Describes which ENA-express features are enabled
* ena_srd_eligible_tx_pkts – The number of network packets sent
within a given time period that meet SRD requirements for
eligibility
* ena_srd_tx_pkts – The number of SRD packets transmitted within
a given time period.
* ena_srd_rx_pkts – The number of SRD packets received within
a given time period.
* ena_srd_resource_utilization – The percentage of the maximum
allowed memory utilization for concurrent SRD connections
that the instance has consumed.
Probing the ENA Express metrics is performed via an admin command.
Thus, a multi-process proxy handler was added.
Signed-off-by: Shai Brandes <shaib...@amazon.com>
Reviewed-by: Amit Bernstein <amitb...@amazon.com>
---
doc/guides/rel_notes/release_23_11.rst | 1 +
drivers/net/ena/ena_ethdev.c | 103 ++++++++++++++++++++++++-
drivers/net/ena/ena_ethdev.h | 18 +++++
3 files changed, 121 insertions(+), 1 deletion(-)
diff --git a/doc/guides/rel_notes/release_23_11.rst
b/doc/guides/rel_notes/release_23_11.rst
index 2666371ccd..10482a643d 100644
--- a/doc/guides/rel_notes/release_23_11.rst
+++ b/doc/guides/rel_notes/release_23_11.rst
@@ -127,6 +127,7 @@ New Features
* Upgraded ENA HAL to latest version.
* Added support for connection tracking allowance utilization metric.
* Added support for reporting rx overrun errors in xstats.
+ * Added support for ENA-express metrics.
* **Updated Intel cpfl driver.**
diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c
index bd9d7c6249..009ee425fe 100644
--- a/drivers/net/ena/ena_ethdev.c
+++ b/drivers/net/ena/ena_ethdev.c
@@ -66,6 +66,9 @@ struct ena_stats {
#define ENA_STAT_GLOBAL_ENTRY(stat) \
ENA_STAT_ENTRY(stat, dev)
+#define ENA_STAT_ENA_SRD_ENTRY(stat) \
+ ENA_STAT_ENTRY(stat, srd)
+
/* Device arguments */
#define ENA_DEVARG_LARGE_LLQ_HDR "large_llq_hdr"
/* Timeout in seconds after which a single uncompleted Tx packet should be
@@ -106,6 +109,14 @@ static struct ena_stats ena_stats_metrics_strings[] = {
ENA_STAT_METRICS_ENTRY(conntrack_allowance_available),
};
+static const struct ena_stats ena_stats_srd_strings[] = {
+ ENA_STAT_ENA_SRD_ENTRY(ena_srd_mode),
+ ENA_STAT_ENA_SRD_ENTRY(ena_srd_tx_pkts),
+ ENA_STAT_ENA_SRD_ENTRY(ena_srd_eligible_tx_pkts),
+ ENA_STAT_ENA_SRD_ENTRY(ena_srd_rx_pkts),
+ ENA_STAT_ENA_SRD_ENTRY(ena_srd_resource_utilization),
+};
+
static const struct ena_stats ena_stats_tx_strings[] = {
ENA_STAT_TX_ENTRY(cnt),
ENA_STAT_TX_ENTRY(bytes),
@@ -132,6 +143,7 @@ static const struct ena_stats ena_stats_rx_strings[] = {
#define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings)
#define ENA_STATS_ARRAY_METRICS ARRAY_SIZE(ena_stats_metrics_strings)
#define ENA_STATS_ARRAY_METRICS_LEGACY (ENA_STATS_ARRAY_METRICS - 1)
+#define ENA_STATS_ARRAY_ENA_SRD ARRAY_SIZE(ena_stats_srd_strings)
#define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings)
#define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings)
@@ -272,6 +284,8 @@ static int ena_parse_devargs(struct ena_adapter *adapter,
static void ena_copy_customer_metrics(struct ena_adapter *adapter,
uint64_t *buf,
size_t buf_size);
+static void ena_copy_ena_srd_info(struct ena_adapter *adapter,
+ struct ena_stats_srd *srd_info);
static int ena_setup_rx_intr(struct rte_eth_dev *dev);
static int ena_rx_queue_intr_enable(struct rte_eth_dev *dev,
uint16_t queue_id);
@@ -324,6 +338,7 @@ enum ena_mp_req {
ENA_MP_IND_TBL_GET,
ENA_MP_IND_TBL_SET,
ENA_MP_CUSTOMER_METRICS_GET,
+ ENA_MP_SRD_STATS_GET,
};
/** Proxy message body. Shared between requests and responses. */
@@ -581,6 +596,22 @@ ENA_PROXY_DESC(ena_com_get_customer_metrics,
ENA_MP_CUSTOMER_METRICS_GET,
}),
struct ena_com_dev *ena_dev, char *buf, size_t buf_size);
+ENA_PROXY_DESC(ena_com_get_ena_srd_info, ENA_MP_SRD_STATS_GET,
+({
+ ENA_TOUCH(adapter);
+ ENA_TOUCH(req);
+ ENA_TOUCH(ena_dev);
+ ENA_TOUCH(info);
+}),
+({
+ ENA_TOUCH(rsp);
+ ENA_TOUCH(ena_dev);
+ if ((struct ena_stats_srd *)info != &adapter->srd_stats)
+ rte_memcpy((struct ena_stats_srd *)info,
+ &adapter->srd_stats,
+ sizeof(struct ena_stats_srd));
+}),
+ struct ena_com_dev *ena_dev, struct ena_admin_ena_srd_info *info);
static inline void ena_trigger_reset(struct ena_adapter *adapter,
enum ena_regs_reset_reason_types reason)
@@ -787,6 +818,7 @@ static unsigned int ena_xstats_calc_num(struct
rte_eth_dev_data *data)
return ENA_STATS_ARRAY_GLOBAL +
adapter->metrics_num +
+ ENA_STATS_ARRAY_ENA_SRD +
(data->nb_tx_queues * ENA_STATS_ARRAY_TX) +
(data->nb_rx_queues * ENA_STATS_ARRAY_RX);
}
@@ -3250,6 +3282,27 @@ static void ena_copy_customer_metrics(struct ena_adapter
*adapter, uint64_t *buf
}
}
+static void ena_copy_ena_srd_info(struct ena_adapter *adapter,
+ struct ena_stats_srd *srd_info)
+{
+ int rc;
+
+ if (!ena_com_get_cap(&adapter->ena_dev, ENA_ADMIN_ENA_SRD_INFO))
+ return;
+
+ rte_spinlock_lock(&adapter->admin_lock);
+ rc = ENA_PROXY(adapter,
+ ena_com_get_ena_srd_info,
+ &adapter->ena_dev,
+ (struct ena_admin_ena_srd_info *)srd_info);
+ rte_spinlock_unlock(&adapter->admin_lock);
+ if (rc != ENA_COM_OK && rc != ENA_COM_UNSUPPORTED) {
+ PMD_DRV_LOG(WARNING,
+ "Failed to get ENA express srd info, rc: %d\n",
rc);
+ return;
+ }
+}
+
/**
* DPDK callback to retrieve names of extended device statistics
*
@@ -3282,6 +3335,10 @@ static int ena_xstats_get_names(struct rte_eth_dev *dev,
rte_strscpy(xstats_names[count].name,
ena_stats_metrics_strings[stat].name,
RTE_ETH_XSTATS_NAME_SIZE);
+ for (stat = 0; stat < ENA_STATS_ARRAY_ENA_SRD; stat++, count++)
+ rte_strscpy(xstats_names[count].name,
+ ena_stats_srd_strings[stat].name,
+ RTE_ETH_XSTATS_NAME_SIZE);
for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++)
for (i = 0; i < dev->data->nb_rx_queues; i++, count++)
@@ -3353,6 +3410,15 @@ static int ena_xstats_get_names_by_id(struct rte_eth_dev
*dev,
}
id -= adapter->metrics_num;
+
+ if (id < ENA_STATS_ARRAY_ENA_SRD) {
+ rte_strscpy(xstats_names[i].name,
+ ena_stats_srd_strings[id].name,
+ RTE_ETH_XSTATS_NAME_SIZE);
+ continue;
+ }
+ id -= ENA_STATS_ARRAY_ENA_SRD;
+
if (id < ENA_STATS_ARRAY_RX) {
qid = id / dev->data->nb_rx_queues;
id %= dev->data->nb_rx_queues;
@@ -3404,6 +3470,7 @@ static int ena_xstats_get(struct rte_eth_dev *dev,
int stat_offset;
void *stats_begin;
uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS];
+ struct ena_stats_srd srd_info = {0};
if (n < xstats_count)
return xstats_count;
@@ -3430,6 +3497,15 @@ static int ena_xstats_get(struct rte_eth_dev *dev,
((char *)stats_begin + stat_offset));
}
+ ena_copy_ena_srd_info(adapter, &srd_info);
+ stats_begin = &srd_info;
+ for (stat = 0; stat < ENA_STATS_ARRAY_ENA_SRD; stat++, count++) {
+ stat_offset = ena_stats_srd_strings[stat].stat_offset;
+ xstats[count].id = count;
+ xstats[count].value = *((uint64_t *)
+ ((char *)stats_begin + stat_offset));
+ }
+
for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++) {
for (i = 0; i < dev->data->nb_rx_queues; i++, count++) {
stat_offset = ena_stats_rx_strings[stat].stat_offset;
@@ -3467,7 +3543,9 @@ static int ena_xstats_get_by_id(struct rte_eth_dev *dev,
int qid;
int valid = 0;
bool were_metrics_copied = false;
+ bool was_srd_info_copied = false;
uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS];
+ struct ena_stats_srd srd_info = {0};
for (i = 0; i < n; ++i) {
id = ids[i];
@@ -3497,8 +3575,27 @@ static int ena_xstats_get_by_id(struct rte_eth_dev *dev,
continue;
}
- /* Check if id belongs to rx queue statistics */
+ /* Check if id belongs to SRD info statistics */
id -= adapter->metrics_num;
+
+ if (id < ENA_STATS_ARRAY_ENA_SRD) {
+ /*
+ * Avoid reading srd info multiple times in a single
+ * function call, as it requires communication with the
+ * admin queue.
+ */
+ if (!was_srd_info_copied) {
+ was_srd_info_copied = true;
+ ena_copy_ena_srd_info(adapter, &srd_info);
+ }
+ values[i] = *((uint64_t *)&adapter->srd_stats + id);
+ ++valid;
+ continue;
+ }
+
+ /* Check if id belongs to rx queue statistics */
+ id -= ENA_STATS_ARRAY_ENA_SRD;
+
rx_entries = ENA_STATS_ARRAY_RX * dev->data->nb_rx_queues;
if (id < rx_entries) {
qid = id % dev->data->nb_rx_queues;
@@ -3993,6 +4090,10 @@ ena_mp_primary_handle(const struct rte_mp_msg *mp_msg,
const void *peer)
(char *)adapter->metrics_stats,
sizeof(uint64_t) * adapter->metrics_num);
break;
+ case ENA_MP_SRD_STATS_GET:
+ res = ena_com_get_ena_srd_info(ena_dev,
+ (struct ena_admin_ena_srd_info
*)&adapter->srd_stats);
+ break;
default:
PMD_DRV_LOG(ERR, "Unknown request type %d\n", req->type);
res = -EINVAL;
diff --git a/drivers/net/ena/ena_ethdev.h b/drivers/net/ena/ena_ethdev.h
index 3f29764ca6..4988fbffb5 100644
--- a/drivers/net/ena/ena_ethdev.h
+++ b/drivers/net/ena/ena_ethdev.h
@@ -252,6 +252,23 @@ struct ena_stats_metrics {
uint64_t conntrack_allowance_available;
};
+struct ena_stats_srd {
+ /* Describes which ENA Express features are enabled */
+ uint64_t ena_srd_mode;
+
+ /* Number of packets transmitted over ENA SRD */
+ uint64_t ena_srd_tx_pkts;
+
+ /* Number of packets transmitted or could have been transmitted over
ENA SRD */
+ uint64_t ena_srd_eligible_tx_pkts;
+
+ /* Number of packets received over ENA SRD */
+ uint64_t ena_srd_rx_pkts;
+
+ /* Percentage of the ENA SRD resources that is in use */
+ uint64_t ena_srd_resource_utilization;
+};
+
struct ena_offloads {
uint32_t tx_offloads;
uint32_t rx_offloads;
@@ -329,6 +346,7 @@ struct ena_adapter {
*/
uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS] __rte_cache_aligned;
uint16_t metrics_num;
+ struct ena_stats_srd srd_stats __rte_cache_aligned;
};
int ena_mp_indirect_table_set(struct ena_adapter *adapter);
--
2.17.1
