From: Andy Gospodarek <go...@broadcom.com>
This implements the changes needed for the bnxt_en driver to add support
for adaptive interrupt moderation per ring.
This does add additional counters in the receive path, but testing shows
that any additional instructions are offset by throughput gain when the
default configuration is for low latency.
Signed-off-by: Andy Gospodarek <go...@broadcom.com>
---
drivers/net/ethernet/broadcom/bnxt/Makefile | 2 +-
drivers/net/ethernet/broadcom/bnxt/bnxt.c | 51 +++++++++++++++++++++++
drivers/net/ethernet/broadcom/bnxt/bnxt.h | 34 ++++++++++-----
drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 7 ++++
drivers/net/ethernet/broadcom/bnxt/bnxt_rx_am.c | 32 ++++++++++++++
5 files changed, 114 insertions(+), 12 deletions(-)
create mode 100644 drivers/net/ethernet/broadcom/bnxt/bnxt_rx_am.c
diff --git a/drivers/net/ethernet/broadcom/bnxt/Makefile
b/drivers/net/ethernet/broadcom/bnxt/Makefile
index 59c8ec9..1b0c78c 100644
--- a/drivers/net/ethernet/broadcom/bnxt/Makefile
+++ b/drivers/net/ethernet/broadcom/bnxt/Makefile
@@ -1,4 +1,4 @@
obj-$(CONFIG_BNXT) += bnxt_en.o
-bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o
bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o
+bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o
bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o bnxt_rx_am.o
bnxt_en-$(CONFIG_BNXT_FLOWER_OFFLOAD) += bnxt_tc.o
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index 4e3d569..e1110d9 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -1482,6 +1482,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi
*bnapi, u32 *raw_cons,
u32 tmp_raw_cons = *raw_cons;
u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
struct bnxt_sw_rx_bd *rx_buf;
+ unsigned int pkts = 0;
unsigned int len;
u8 *data_ptr, agg_bufs, cmp_type;
dma_addr_t dma_addr;
@@ -1522,6 +1523,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi
*bnapi, u32 *raw_cons,
rc = -ENOMEM;
if (likely(skb)) {
+ struct skb_shared_info *shinfo = skb_shinfo(skb);
+ pkts = shinfo->nr_frags;
bnxt_deliver_skb(bp, bnapi, skb);
rc = 1;
}
@@ -1645,6 +1648,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi
*bnapi, u32 *raw_cons,
rxr->rx_next_cons = NEXT_RX(cons);
next_rx_no_prod:
+ cpr->rx_packets += pkts ? : 1;
+ cpr->rx_bytes += len;
*raw_cons = tmp_raw_cons;
return rc;
@@ -1798,6 +1803,7 @@ static irqreturn_t bnxt_msix(int irq, void *dev_instance)
struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
u32 cons = RING_CMP(cpr->cp_raw_cons);
+ cpr->event_ctr++;
prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
napi_schedule(&bnapi->napi);
return IRQ_HANDLED;
@@ -2021,6 +2027,11 @@ static int bnxt_poll(struct napi_struct *napi, int
budget)
break;
}
}
+ if (bp->flags & BNXT_FLAG_RX_AM)
+ net_rx_am(&cpr->am,
+ cpr->event_ctr,
+ cpr->rx_packets,
+ cpr->rx_bytes);
mmiowb();
return work_done;
}
@@ -2606,6 +2617,8 @@ static void bnxt_init_cp_rings(struct bnxt *bp)
struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
ring->fw_ring_id = INVALID_HW_RING_ID;
+ cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
+ cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
}
}
@@ -4579,6 +4592,38 @@ static void bnxt_hwrm_set_coal_params(struct bnxt_coal
*hw_coal,
req->flags = cpu_to_le16(flags);
}
+int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
+{
+ struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
+ struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
+ struct bnxt_coal coal;
+ int rc = 0;
+
+ /* Tick values in micro seconds.
+ * 1 coal_buf x bufs_per_record = 1 completion record.
+ */
+ memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
+
+ coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
+ coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
+
+ if (!bnapi->rx_ring)
+ return -ENODEV;
+
+ bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
+ HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
+
+ bnxt_hwrm_set_coal_params(&coal, &req_rx);
+
+ mutex_lock(&bp->hwrm_cmd_lock);
+ req_rx.ring_id = cpr->cp_ring_struct.fw_ring_id;
+
+ rc = _hwrm_send_message(bp, &req_rx, sizeof(req_rx),
+ HWRM_CMD_TIMEOUT);
+ mutex_unlock(&bp->hwrm_cmd_lock);
+ return rc;
+}
+
int bnxt_hwrm_set_coal(struct bnxt *bp)
{
int i, rc = 0;
@@ -5706,7 +5751,11 @@ static void bnxt_enable_napi(struct bnxt *bp)
int i;
for (i = 0; i < bp->cp_nr_rings; i++) {
+ struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
bp->bnapi[i]->in_reset = false;
+
+ INIT_WORK(&cpr->am.work, bnxt_rx_am_work);
+ cpr->am.mode = NET_CQ_PERIOD_MODE_START_FROM_EQE;
napi_enable(&bp->bnapi[i]->napi);
}
}
@@ -8124,6 +8173,8 @@ static int bnxt_init_one(struct pci_dev *pdev, const
struct pci_device_id *ent)
else
bp->flags |= BNXT_FLAG_DOUBLE_DB;
+ bp->flags = BNXT_FLAG_RX_AM;
+
rc = bnxt_hwrm_func_drv_rgtr(bp);
if (rc)
goto init_err_pci_clean;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index 5359a1f..8ab5801d 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -23,6 +23,7 @@
#include <net/devlink.h>
#include <net/dst_metadata.h>
#include <net/switchdev.h>
+#include <linux/net_rx_am.h>
struct tx_bd {
__le32 tx_bd_len_flags_type;
@@ -607,6 +608,17 @@ struct bnxt_tx_ring_info {
struct bnxt_ring_struct tx_ring_struct;
};
+struct bnxt_coal {
+ u16 coal_ticks;
+ u16 coal_ticks_irq;
+ u16 coal_bufs;
+ u16 coal_bufs_irq;
+ /* RING_IDLE enabled when coal ticks < idle_thresh */
+ u16 idle_thresh;
+ u8 bufs_per_record;
+ u8 budget;
+};
+
struct bnxt_tpa_info {
void *data;
u8 *data_ptr;
@@ -670,6 +682,13 @@ struct bnxt_cp_ring_info {
u32 cp_raw_cons;
void __iomem *cp_doorbell;
+ struct bnxt_coal rx_ring_coal;
+ u64 rx_packets;
+ u64 rx_bytes;
+ u64 event_ctr;
+
+ struct net_rx_am am;
+
struct tx_cmp *cp_desc_ring[MAX_CP_PAGES];
dma_addr_t cp_desc_mapping[MAX_CP_PAGES];
@@ -944,17 +963,6 @@ struct bnxt_test_info {
#define BNXT_CAG_REG_LEGACY_INT_STATUS 0x4014
#define BNXT_CAG_REG_BASE 0x300000
-struct bnxt_coal {
- u16 coal_ticks;
- u16 coal_ticks_irq;
- u16 coal_bufs;
- u16 coal_bufs_irq;
- /* RING_IDLE enabled when coal ticks < idle_thresh */
- u16 idle_thresh;
- u8 bufs_per_record;
- u8 budget;
-};
-
struct bnxt_tc_flow_stats {
u64 packets;
u64 bytes;
@@ -1126,6 +1134,7 @@ struct bnxt {
#define BNXT_FLAG_DOUBLE_DB 0x400000
#define BNXT_FLAG_FW_DCBX_AGENT 0x800000
#define BNXT_FLAG_CHIP_NITRO_A0 0x1000000
+ #define BNXT_FLAG_RX_AM 0x2000000
#define BNXT_FLAG_ALL_CONFIG_FEATS (BNXT_FLAG_TPA | \
BNXT_FLAG_RFS | \
@@ -1423,4 +1432,7 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc);
int bnxt_get_max_rings(struct bnxt *, int *, int *, bool);
void bnxt_restore_pf_fw_resources(struct bnxt *bp);
int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr);
+void bnxt_rx_am_work(struct work_struct *work);
+int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi);
+
#endif
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index 7ce1d4b..cffc649 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -49,6 +49,8 @@ static int bnxt_get_coalesce(struct net_device *dev,
memset(coal, 0, sizeof(*coal));
+ coal->use_adaptive_rx_coalesce = bp->flags & BNXT_FLAG_RX_AM;
+
hw_coal = &bp->rx_coal;
mult = hw_coal->bufs_per_record;
coal->rx_coalesce_usecs = hw_coal->coal_ticks;
@@ -77,6 +79,11 @@ static int bnxt_set_coalesce(struct net_device *dev,
int rc = 0;
u16 mult;
+ if (coal->use_adaptive_rx_coalesce)
+ bp->flags |= BNXT_FLAG_RX_AM;
+ else
+ bp->flags &= ~(BNXT_FLAG_RX_AM);
+
hw_coal = &bp->rx_coal;
mult = hw_coal->bufs_per_record;
hw_coal->coal_ticks = coal->rx_coalesce_usecs;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_rx_am.c
b/drivers/net/ethernet/broadcom/bnxt/bnxt_rx_am.c
new file mode 100644
index 0000000..8579ca5
--- /dev/null
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_rx_am.c
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2017 Broadcom Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/net_rx_am.h>
+#include "bnxt_hsi.h"
+#include "bnxt.h"
+
+void bnxt_rx_am_work(struct work_struct *work)
+{
+ struct net_rx_am *am = container_of(work, struct net_rx_am,
+ work);
+ struct bnxt_cp_ring_info *cpr = container_of(am,
+ struct bnxt_cp_ring_info,
+ am);
+ struct bnxt_napi *bnapi = container_of(cpr,
+ struct bnxt_napi,
+ cp_ring);
+ struct net_cq_moder cur_profile = net_am_get_profile(am->mode,
+ am->profile_ix);
+
+ cpr->rx_ring_coal.coal_ticks = cur_profile.usec;
+ cpr->rx_ring_coal.coal_bufs = cur_profile.pkts;
+
+ bnxt_hwrm_set_ring_coal(bnapi->bp, bnapi);
+ am->state = NET_AM_START_MEASURE;
+}
+
--
2.7.4
