Adds RX support for the cxgbe poll mode driver.  This patch:

1. Adds rx queue related eth_dev_ops.
2. Adds RSS support.
3. Adds dev_configure() and dev_infos_get() eth_dev_ops.
4. Adds rx_pkt_burst for receiving packets.

Signed-off-by: Rahul Lakkireddy <rahul.lakkireddy at chelsio.com>
Signed-off-by: Kumar Sanghvi <kumaras at chelsio.com>
---
v4:
- Fix 32-bit compilation.

v3:
- No changes.

v2:
- This patch is a subset of patch 2/5 submitted in v1.
- Cleanup some RX related macros and code.

 drivers/net/cxgbe/cxgbe.h        |   6 +
 drivers/net/cxgbe/cxgbe_ethdev.c | 183 ++++++++
 drivers/net/cxgbe/cxgbe_main.c   | 350 +++++++++++++++
 drivers/net/cxgbe/sge.c          | 915 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 1454 insertions(+)

diff --git a/drivers/net/cxgbe/cxgbe.h b/drivers/net/cxgbe/cxgbe.h
index 44d48dc..90d1db0 100644
--- a/drivers/net/cxgbe/cxgbe.h
+++ b/drivers/net/cxgbe/cxgbe.h
@@ -44,5 +44,11 @@
 #define CXGBE_DEFAULT_RX_DESC_SIZE    1024 /* Default RX ring size */

 int cxgbe_probe(struct adapter *adapter);
+void init_rspq(struct adapter *adap, struct sge_rspq *q, unsigned int us,
+              unsigned int cnt, unsigned int size, unsigned int iqe_size);
+int setup_sge_fwevtq(struct adapter *adapter);
+void cfg_queues(struct rte_eth_dev *eth_dev);
+int cfg_queue_count(struct rte_eth_dev *eth_dev);
+int setup_rss(struct port_info *pi);

 #endif /* _CXGBE_H_ */
diff --git a/drivers/net/cxgbe/cxgbe_ethdev.c b/drivers/net/cxgbe/cxgbe_ethdev.c
index 30d39b4..1c69973 100644
--- a/drivers/net/cxgbe/cxgbe_ethdev.c
+++ b/drivers/net/cxgbe/cxgbe_ethdev.c
@@ -85,7 +85,189 @@
  */
 #include "t4_pci_id_tbl.h"

+static uint16_t cxgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+                               uint16_t nb_pkts)
+{
+       struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)rx_queue;
+       unsigned int work_done;
+
+       CXGBE_DEBUG_RX(adapter, "%s: rxq->rspq.cntxt_id = %u; nb_pkts = %d\n",
+                      __func__, rxq->rspq.cntxt_id, nb_pkts);
+
+       if (cxgbe_poll(&rxq->rspq, rx_pkts, (unsigned int)nb_pkts, &work_done))
+               dev_err(adapter, "error in cxgbe poll\n");
+
+       CXGBE_DEBUG_RX(adapter, "%s: work_done = %u\n", __func__, work_done);
+       return work_done;
+}
+
+static void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev,
+                              struct rte_eth_dev_info *device_info)
+{
+       struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+       struct adapter *adapter = pi->adapter;
+       int max_queues = adapter->sge.max_ethqsets / adapter->params.nports;
+
+       device_info->min_rx_bufsize = 68; /* XXX: Smallest pkt size */
+       device_info->max_rx_pktlen = 1500; /* XXX: For now we support mtu */
+       device_info->max_rx_queues = max_queues;
+       device_info->max_tx_queues = max_queues;
+       device_info->max_mac_addrs = 1;
+       /* XXX: For now we support one MAC/port */
+       device_info->max_vfs = adapter->params.arch.vfcount;
+       device_info->max_vmdq_pools = 0; /* XXX: For now no support for VMDQ */
+
+       device_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
+                                      DEV_RX_OFFLOAD_IPV4_CKSUM |
+                                      DEV_RX_OFFLOAD_UDP_CKSUM |
+                                      DEV_RX_OFFLOAD_TCP_CKSUM;
+
+       device_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
+                                      DEV_TX_OFFLOAD_IPV4_CKSUM |
+                                      DEV_TX_OFFLOAD_UDP_CKSUM |
+                                      DEV_TX_OFFLOAD_TCP_CKSUM |
+                                      DEV_TX_OFFLOAD_TCP_TSO;
+
+       device_info->reta_size = pi->rss_size;
+}
+
+static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
+                                   uint16_t tx_queue_id);
+static void cxgbe_dev_rx_queue_release(void *q);
+
+static int cxgbe_dev_configure(struct rte_eth_dev *eth_dev)
+{
+       struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+       struct adapter *adapter = pi->adapter;
+       int err;
+
+       CXGBE_FUNC_TRACE();
+
+       if (!(adapter->flags & FW_QUEUE_BOUND)) {
+               err = setup_sge_fwevtq(adapter);
+               if (err)
+                       return err;
+               adapter->flags |= FW_QUEUE_BOUND;
+       }
+
+       err = cfg_queue_count(eth_dev);
+       if (err)
+               return err;
+
+       return 0;
+}
+
+static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
+                                   uint16_t rx_queue_id)
+{
+       struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+       struct adapter *adap = pi->adapter;
+       struct sge_rspq *q;
+
+       dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
+                 __func__, pi->port_id, rx_queue_id);
+
+       q = eth_dev->data->rx_queues[rx_queue_id];
+       return t4_sge_eth_rxq_start(adap, q);
+}
+
+static int cxgbe_dev_rx_queue_stop(struct rte_eth_dev *eth_dev,
+                                  uint16_t rx_queue_id)
+{
+       struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+       struct adapter *adap = pi->adapter;
+       struct sge_rspq *q;
+
+       dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
+                 __func__, pi->port_id, rx_queue_id);
+
+       q = eth_dev->data->rx_queues[rx_queue_id];
+       return t4_sge_eth_rxq_stop(adap, q);
+}
+
+static int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
+                                   uint16_t queue_idx, uint16_t nb_desc,
+                                   unsigned int socket_id,
+                                   const struct rte_eth_rxconf *rx_conf,
+                                   struct rte_mempool *mp)
+{
+       struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+       struct adapter *adapter = pi->adapter;
+       struct sge *s = &adapter->sge;
+       struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset + queue_idx];
+       int err = 0;
+       int msi_idx = 0;
+       unsigned int temp_nb_desc;
+
+       RTE_SET_USED(rx_conf);
+
+       dev_debug(adapter, "%s: eth_dev->data->nb_rx_queues = %d; queue_idx = 
%d; nb_desc = %d; socket_id = %d; mp = %p\n",
+                 __func__, eth_dev->data->nb_rx_queues, queue_idx, nb_desc,
+                 socket_id, mp);
+
+       /*  Free up the existing queue  */
+       if (eth_dev->data->rx_queues[queue_idx]) {
+               cxgbe_dev_rx_queue_release(eth_dev->data->rx_queues[queue_idx]);
+               eth_dev->data->rx_queues[queue_idx] = NULL;
+       }
+
+       eth_dev->data->rx_queues[queue_idx] = (void *)rxq;
+
+       /* Sanity Checking
+        *
+        * nb_desc should be > 0 and <= CXGBE_MAX_RING_DESC_SIZE
+        */
+       temp_nb_desc = nb_desc;
+       if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) {
+               dev_warn(adapter, "%s: number of descriptors must be >= %d. 
Using default [%d]\n",
+                        __func__, CXGBE_MIN_RING_DESC_SIZE,
+                        CXGBE_DEFAULT_RX_DESC_SIZE);
+               temp_nb_desc = CXGBE_DEFAULT_RX_DESC_SIZE;
+       } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) {
+               dev_err(adapter, "%s: number of descriptors must be between %d 
and %d inclusive. Default [%d]\n",
+                       __func__, CXGBE_MIN_RING_DESC_SIZE,
+                       CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_RX_DESC_SIZE);
+               return -(EINVAL);
+       }
+
+       rxq->rspq.size = temp_nb_desc;
+       if ((&rxq->fl) != NULL)
+               rxq->fl.size = temp_nb_desc;
+
+       err = t4_sge_alloc_rxq(adapter, &rxq->rspq, false, eth_dev, msi_idx,
+                              &rxq->fl, t4_ethrx_handler,
+                              t4_get_mps_bg_map(adapter, pi->tx_chan), mp,
+                              queue_idx, socket_id);
+
+       dev_debug(adapter, "%s: err = %d; port_id = %d; cntxt_id = %u\n",
+                 __func__, err, pi->port_id, rxq->rspq.cntxt_id);
+       return err;
+}
+
+static void cxgbe_dev_rx_queue_release(void *q)
+{
+       struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)q;
+       struct sge_rspq *rq = &rxq->rspq;
+
+       if (rq) {
+               struct port_info *pi = (struct port_info *)
+                                      (rq->eth_dev->data->dev_private);
+               struct adapter *adap = pi->adapter;
+
+               dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
+                         __func__, pi->port_id, rxq->rspq.cntxt_id);
+
+               t4_sge_eth_rxq_release(adap, rxq);
+       }
+}
+
 static struct eth_dev_ops cxgbe_eth_dev_ops = {
+       .dev_configure          = cxgbe_dev_configure,
+       .dev_infos_get          = cxgbe_dev_info_get,
+       .rx_queue_setup         = cxgbe_dev_rx_queue_setup,
+       .rx_queue_start         = cxgbe_dev_rx_queue_start,
+       .rx_queue_stop          = cxgbe_dev_rx_queue_stop,
+       .rx_queue_release       = cxgbe_dev_rx_queue_release,
 };

 /*
@@ -103,6 +285,7 @@ static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev)
        CXGBE_FUNC_TRACE();

        eth_dev->dev_ops = &cxgbe_eth_dev_ops;
+       eth_dev->rx_pkt_burst = &cxgbe_recv_pkts;

        /* for secondary processes, we don't initialise any further as primary
         * has already done this work.
diff --git a/drivers/net/cxgbe/cxgbe_main.c b/drivers/net/cxgbe/cxgbe_main.c
index 6f3a6db..abcef6b 100644
--- a/drivers/net/cxgbe/cxgbe_main.c
+++ b/drivers/net/cxgbe/cxgbe_main.c
@@ -67,6 +67,249 @@
 #include "t4_msg.h"
 #include "cxgbe.h"

+/*
+ * Response queue handler for the FW event queue.
+ */
+static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
+                         __rte_unused const struct pkt_gl *gl)
+{
+       u8 opcode = ((const struct rss_header *)rsp)->opcode;
+
+       rsp++;                                          /* skip RSS header */
+
+       /*
+        * FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
+        */
+       if (unlikely(opcode == CPL_FW4_MSG &&
+                    ((const struct cpl_fw4_msg *)rsp)->type ==
+                     FW_TYPE_RSSCPL)) {
+               rsp++;
+               opcode = ((const struct rss_header *)rsp)->opcode;
+               rsp++;
+               if (opcode != CPL_SGE_EGR_UPDATE) {
+                       dev_err(q->adapter, "unexpected FW4/CPL %#x on FW event 
queue\n",
+                               opcode);
+                       goto out;
+               }
+       }
+
+       if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
+               /* do nothing */
+       } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
+               const struct cpl_fw6_msg *msg = (const void *)rsp;
+
+               t4_handle_fw_rpl(q->adapter, msg->data);
+       } else {
+               dev_err(adapter, "unexpected CPL %#x on FW event queue\n",
+                       opcode);
+       }
+out:
+       return 0;
+}
+
+int setup_sge_fwevtq(struct adapter *adapter)
+{
+       struct sge *s = &adapter->sge;
+       int err = 0;
+       int msi_idx = 0;
+
+       err = t4_sge_alloc_rxq(adapter, &s->fw_evtq, true, adapter->eth_dev,
+                              msi_idx, NULL, fwevtq_handler, -1, NULL, 0,
+                              rte_socket_id());
+       return err;
+}
+
+static int closest_timer(const struct sge *s, int time)
+{
+       unsigned int i, match = 0;
+       int delta, min_delta = INT_MAX;
+
+       for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
+               delta = time - s->timer_val[i];
+               if (delta < 0)
+                       delta = -delta;
+               if (delta < min_delta) {
+                       min_delta = delta;
+                       match = i;
+               }
+       }
+       return match;
+}
+
+static int closest_thres(const struct sge *s, int thres)
+{
+       unsigned int i, match = 0;
+       int delta, min_delta = INT_MAX;
+
+       for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
+               delta = thres - s->counter_val[i];
+               if (delta < 0)
+                       delta = -delta;
+               if (delta < min_delta) {
+                       min_delta = delta;
+                       match = i;
+               }
+       }
+       return match;
+}
+
+/**
+ * cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters
+ * @q: the Rx queue
+ * @us: the hold-off time in us, or 0 to disable timer
+ * @cnt: the hold-off packet count, or 0 to disable counter
+ *
+ * Sets an Rx queue's interrupt hold-off time and packet count.  At least
+ * one of the two needs to be enabled for the queue to generate interrupts.
+ */
+int cxgb4_set_rspq_intr_params(struct sge_rspq *q, unsigned int us,
+                              unsigned int cnt)
+{
+       struct adapter *adap = q->adapter;
+       unsigned int timer_val;
+
+       if (cnt) {
+               int err;
+               u32 v, new_idx;
+
+               new_idx = closest_thres(&adap->sge, cnt);
+               if (q->desc && q->pktcnt_idx != new_idx) {
+                       /* the queue has already been created, update it */
+                       v = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
+                           V_FW_PARAMS_PARAM_X(
+                           FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
+                           V_FW_PARAMS_PARAM_YZ(q->cntxt_id);
+                       err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1,
+                                           &v, &new_idx);
+                       if (err)
+                               return err;
+               }
+               q->pktcnt_idx = new_idx;
+       }
+
+       timer_val = (us == 0) ? X_TIMERREG_RESTART_COUNTER :
+                               closest_timer(&adap->sge, us);
+
+       if ((us | cnt) == 0)
+               q->intr_params = V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX);
+       else
+               q->intr_params = V_QINTR_TIMER_IDX(timer_val) |
+                                V_QINTR_CNT_EN(cnt > 0);
+       return 0;
+}
+
+static inline bool is_x_1g_port(const struct link_config *lc)
+{
+       return ((lc->supported & FW_PORT_CAP_SPEED_1G) != 0);
+}
+
+static inline bool is_x_10g_port(const struct link_config *lc)
+{
+       return ((lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
+               (lc->supported & FW_PORT_CAP_SPEED_40G) != 0 ||
+               (lc->supported & FW_PORT_CAP_SPEED_100G) != 0);
+}
+
+inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
+                     unsigned int us, unsigned int cnt,
+                     unsigned int size, unsigned int iqe_size)
+{
+       q->adapter = adap;
+       cxgb4_set_rspq_intr_params(q, us, cnt);
+       q->iqe_len = iqe_size;
+       q->size = size;
+}
+
+int cfg_queue_count(struct rte_eth_dev *eth_dev)
+{
+       struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+       struct adapter *adap = pi->adapter;
+       struct sge *s = &adap->sge;
+       unsigned int max_queues = s->max_ethqsets / adap->params.nports;
+
+       if ((eth_dev->data->nb_rx_queues < 1) ||
+           (eth_dev->data->nb_tx_queues < 1))
+               return -EINVAL;
+
+       if ((eth_dev->data->nb_rx_queues > max_queues) ||
+           (eth_dev->data->nb_tx_queues > max_queues))
+               return -EINVAL;
+
+       if (eth_dev->data->nb_rx_queues > pi->rss_size)
+               return -EINVAL;
+
+       /* We must configure RSS, since config has changed*/
+       pi->flags &= ~PORT_RSS_DONE;
+
+       pi->n_rx_qsets = eth_dev->data->nb_rx_queues;
+       pi->n_tx_qsets = eth_dev->data->nb_tx_queues;
+
+       return 0;
+}
+
+void cfg_queues(struct rte_eth_dev *eth_dev)
+{
+       struct rte_config *config = rte_eal_get_configuration();
+       struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+       struct adapter *adap = pi->adapter;
+       struct sge *s = &adap->sge;
+       unsigned int i, nb_ports = 0, qidx = 0;
+       unsigned int q_per_port = 0;
+
+       if (!(adap->flags & CFG_QUEUES)) {
+               for_each_port(adap, i) {
+                       struct port_info *tpi = adap2pinfo(adap, i);
+
+                       nb_ports += (is_x_10g_port(&tpi->link_cfg)) ||
+                                    is_x_1g_port(&tpi->link_cfg) ? 1 : 0;
+               }
+
+               /*
+                * We default up to # of cores queues per 1G/10G port.
+                */
+               if (nb_ports)
+                       q_per_port = (MAX_ETH_QSETS -
+                                    (adap->params.nports - nb_ports)) /
+                                    nb_ports;
+
+               if (q_per_port > config->lcore_count)
+                       q_per_port = config->lcore_count;
+
+               for_each_port(adap, i) {
+                       struct port_info *pi = adap2pinfo(adap, i);
+
+                       pi->first_qset = qidx;
+
+                       /* Initially n_rx_qsets == n_tx_qsets */
+                       pi->n_rx_qsets = (is_x_10g_port(&pi->link_cfg) ||
+                                         is_x_1g_port(&pi->link_cfg)) ?
+                                         q_per_port : 1;
+                       pi->n_tx_qsets = pi->n_rx_qsets;
+
+                       if (pi->n_rx_qsets > pi->rss_size)
+                               pi->n_rx_qsets = pi->rss_size;
+
+                       qidx += pi->n_rx_qsets;
+               }
+
+               s->max_ethqsets = qidx;
+
+               for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
+                       struct sge_eth_rxq *r = &s->ethrxq[i];
+
+                       init_rspq(adap, &r->rspq, 0, 0, 1024, 64);
+                       r->usembufs = 1;
+                       r->fl.size = (r->usembufs ? 1024 : 72);
+               }
+
+               for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
+                       s->ethtxq[i].q.size = 1024;
+
+               init_rspq(adap, &adap->sge.fw_evtq, 0, 0, 1024, 64);
+               adap->flags |= CFG_QUEUES;
+       }
+}
+
 static void setup_memwin(struct adapter *adap)
 {
        u32 mem_win0_base;
@@ -89,6 +332,25 @@ static void setup_memwin(struct adapter *adap)
                                        MEMWIN_NIC));
 }

+static int init_rss(struct adapter *adap)
+{
+       unsigned int i;
+       int err;
+
+       err = t4_init_rss_mode(adap, adap->mbox);
+       if (err)
+               return err;
+
+       for_each_port(adap, i) {
+               struct port_info *pi = adap2pinfo(adap, i);
+
+               pi->rss = rte_zmalloc(NULL, pi->rss_size, 0);
+               if (!pi->rss)
+                       return -ENOMEM;
+       }
+       return 0;
+}
+
 static void print_port_info(struct adapter *adap)
 {
        int i;
@@ -564,6 +826,87 @@ void t4_os_portmod_changed(const struct adapter *adap, int 
port_id)
                         pi->port_id, pi->mod_type);
 }

+/**
+ * cxgb4_write_rss - write the RSS table for a given port
+ * @pi: the port
+ * @queues: array of queue indices for RSS
+ *
+ * Sets up the portion of the HW RSS table for the port's VI to distribute
+ * packets to the Rx queues in @queues.
+ */
+int cxgb4_write_rss(const struct port_info *pi, const u16 *queues)
+{
+       u16 *rss;
+       int i, err;
+       struct adapter *adapter = pi->adapter;
+       const struct sge_eth_rxq *rxq;
+
+       /*  Should never be called before setting up sge eth rx queues */
+       BUG_ON(!(adapter->flags & FULL_INIT_DONE));
+
+       rxq = &adapter->sge.ethrxq[pi->first_qset];
+       rss = rte_zmalloc(NULL, pi->rss_size * sizeof(u16), 0);
+       if (!rss)
+               return -ENOMEM;
+
+       /* map the queue indices to queue ids */
+       for (i = 0; i < pi->rss_size; i++, queues++)
+               rss[i] = rxq[*queues].rspq.abs_id;
+
+       err = t4_config_rss_range(adapter, adapter->pf, pi->viid, 0,
+                                 pi->rss_size, rss, pi->rss_size);
+       /*
+        * If Tunnel All Lookup isn't specified in the global RSS
+        * Configuration, then we need to specify a default Ingress
+        * Queue for any ingress packets which aren't hashed.  We'll
+        * use our first ingress queue ...
+        */
+       if (!err)
+               err = t4_config_vi_rss(adapter, adapter->mbox, pi->viid,
+                                      F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
+                                      F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
+                                      F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
+                                      F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN |
+                                      F_FW_RSS_VI_CONFIG_CMD_UDPEN,
+                                      rss[0]);
+       rte_free(rss);
+       return err;
+}
+
+/**
+ * setup_rss - configure RSS
+ * @adapter: the adapter
+ *
+ * Sets up RSS to distribute packets to multiple receive queues.  We
+ * configure the RSS CPU lookup table to distribute to the number of HW
+ * receive queues, and the response queue lookup table to narrow that
+ * down to the response queues actually configured for each port.
+ * We always configure the RSS mapping for all ports since the mapping
+ * table has plenty of entries.
+ */
+int setup_rss(struct port_info *pi)
+{
+       int j, err;
+       struct adapter *adapter = pi->adapter;
+
+       dev_debug(adapter, "%s:  pi->rss_size = %u; pi->n_rx_qsets = %u\n",
+                 __func__, pi->rss_size, pi->n_rx_qsets);
+
+       if (!pi->flags & PORT_RSS_DONE) {
+               if (adapter->flags & FULL_INIT_DONE) {
+                       /* Fill default values with equal distribution */
+                       for (j = 0; j < pi->rss_size; j++)
+                               pi->rss[j] = j % pi->n_rx_qsets;
+
+                       err = cxgb4_write_rss(pi, pi->rss);
+                       if (err)
+                               return err;
+                       pi->flags |= PORT_RSS_DONE;
+               }
+       }
+       return 0;
+}
+
 int cxgbe_probe(struct adapter *adapter)
 {
        struct port_info *pi;
@@ -662,6 +1005,7 @@ allocate_mac:
                pi->eth_dev->data->dev_private = pi;
                pi->eth_dev->driver = adapter->eth_dev->driver;
                pi->eth_dev->dev_ops = adapter->eth_dev->dev_ops;
+               pi->eth_dev->rx_pkt_burst = adapter->eth_dev->rx_pkt_burst;
                TAILQ_INIT(&pi->eth_dev->link_intr_cbs);

                pi->eth_dev->data->mac_addrs = rte_zmalloc(name,
@@ -683,8 +1027,14 @@ allocate_mac:
                }
        }

+       cfg_queues(adapter->eth_dev);
+
        print_port_info(adapter);

+       err = init_rss(adapter);
+       if (err)
+               goto out_free;
+
        return 0;

 out_free:
diff --git a/drivers/net/cxgbe/sge.c b/drivers/net/cxgbe/sge.c
index 20da0fd..c7abd8d 100644
--- a/drivers/net/cxgbe/sge.c
+++ b/drivers/net/cxgbe/sge.c
@@ -69,6 +69,13 @@
 #include "cxgbe.h"

 /*
+ * Max number of Rx buffers we replenish at a time.
+ */
+#define MAX_RX_REFILL 16U
+
+#define NOMEM_TMR_IDX (SGE_NTIMERS - 1)
+
+/*
  * Rx buffer sizes for "usembufs" Free List buffers (one ingress packet
  * per mbuf buffer).  We currently only support two sizes for 1500- and
  * 9000-byte MTUs. We could easily support more but there doesn't seem to be
@@ -118,6 +125,914 @@ enum {
 };

 /**
+ * fl_cap - return the capacity of a free-buffer list
+ * @fl: the FL
+ *
+ * Returns the capacity of a free-buffer list.  The capacity is less than
+ * the size because one descriptor needs to be left unpopulated, otherwise
+ * HW will think the FL is empty.
+ */
+static inline unsigned int fl_cap(const struct sge_fl *fl)
+{
+       return fl->size - 8;   /* 1 descriptor = 8 buffers */
+}
+
+/**
+ * fl_starving - return whether a Free List is starving.
+ * @adapter: pointer to the adapter
+ * @fl: the Free List
+ *
+ * Tests specified Free List to see whether the number of buffers
+ * available to the hardware has falled below our "starvation"
+ * threshold.
+ */
+static inline bool fl_starving(const struct adapter *adapter,
+                              const struct sge_fl *fl)
+{
+       const struct sge *s = &adapter->sge;
+
+       return fl->avail - fl->pend_cred <= s->fl_starve_thres;
+}
+
+static inline unsigned int get_buf_size(struct adapter *adapter,
+                                       const struct rx_sw_desc *d)
+{
+       struct sge *s = &adapter->sge;
+       unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE;
+       unsigned int buf_size;
+
+       switch (rx_buf_size_idx) {
+       case RX_SMALL_PG_BUF:
+               buf_size = PAGE_SIZE;
+               break;
+
+       case RX_LARGE_PG_BUF:
+               buf_size = PAGE_SIZE << s->fl_pg_order;
+               break;
+
+       case RX_SMALL_MTU_BUF:
+               buf_size = FL_MTU_SMALL_BUFSIZE(adapter);
+               break;
+
+       case RX_LARGE_MTU_BUF:
+               buf_size = FL_MTU_LARGE_BUFSIZE(adapter);
+               break;
+
+       default:
+               BUG_ON(1);
+               buf_size = 0; /* deal with bogus compiler warnings */
+               /* NOTREACHED */
+       }
+
+       return buf_size;
+}
+
+/**
+ * free_rx_bufs - free the Rx buffers on an SGE free list
+ * @q: the SGE free list to free buffers from
+ * @n: how many buffers to free
+ *
+ * Release the next @n buffers on an SGE free-buffer Rx queue.   The
+ * buffers must be made inaccessible to HW before calling this function.
+ */
+static void free_rx_bufs(struct sge_fl *q, int n)
+{
+       unsigned int cidx = q->cidx;
+       struct rx_sw_desc *d;
+
+       d = &q->sdesc[cidx];
+       while (n--) {
+               if (d->buf) {
+                       rte_pktmbuf_free(d->buf);
+                       d->buf = NULL;
+               }
+               ++d;
+               if (++cidx == q->size) {
+                       cidx = 0;
+                       d = q->sdesc;
+               }
+               q->avail--;
+       }
+       q->cidx = cidx;
+}
+
+/**
+ * unmap_rx_buf - unmap the current Rx buffer on an SGE free list
+ * @q: the SGE free list
+ *
+ * Unmap the current buffer on an SGE free-buffer Rx queue.   The
+ * buffer must be made inaccessible to HW before calling this function.
+ *
+ * This is similar to @free_rx_bufs above but does not free the buffer.
+ * Do note that the FL still loses any further access to the buffer.
+ */
+static void unmap_rx_buf(struct sge_fl *q)
+{
+       if (++q->cidx == q->size)
+               q->cidx = 0;
+       q->avail--;
+}
+
+static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
+{
+       if (q->pend_cred >= 8) {
+               u32 val = adap->params.arch.sge_fl_db;
+
+               if (is_t4(adap->params.chip))
+                       val |= V_PIDX(q->pend_cred / 8);
+               else
+                       val |= V_PIDX_T5(q->pend_cred / 8);
+
+               /*
+                * Make sure all memory writes to the Free List queue are
+                * committed before we tell the hardware about them.
+                */
+               wmb();
+
+               /*
+                * If we don't have access to the new User Doorbell (T5+), use
+                * the old doorbell mechanism; otherwise use the new BAR2
+                * mechanism.
+                */
+               if (unlikely(!q->bar2_addr)) {
+                       t4_write_reg(adap, MYPF_REG(A_SGE_PF_KDOORBELL),
+                                    val | V_QID(q->cntxt_id));
+               } else {
+                       writel(val | V_QID(q->bar2_qid),
+                              (void *)((uintptr_t)q->bar2_addr +
+                              SGE_UDB_KDOORBELL));
+
+                       /*
+                        * This Write memory Barrier will force the write to
+                        * the User Doorbell area to be flushed.
+                        */
+                       wmb();
+               }
+               q->pend_cred &= 7;
+       }
+}
+
+static inline struct rte_mbuf *cxgbe_rxmbuf_alloc(struct rte_mempool *mp)
+{
+       struct rte_mbuf *m;
+
+       m = __rte_mbuf_raw_alloc(mp);
+       __rte_mbuf_sanity_check_raw(m, 0);
+       return m;
+}
+
+static inline void set_rx_sw_desc(struct rx_sw_desc *sd, void *buf,
+                                 dma_addr_t mapping)
+{
+       sd->buf = buf;
+       sd->dma_addr = mapping;      /* includes size low bits */
+}
+
+/**
+ * refill_fl_usembufs - refill an SGE Rx buffer ring with mbufs
+ * @adap: the adapter
+ * @q: the ring to refill
+ * @n: the number of new buffers to allocate
+ *
+ * (Re)populate an SGE free-buffer queue with up to @n new packet buffers,
+ * allocated with the supplied gfp flags.  The caller must assure that
+ * @n does not exceed the queue's capacity.  If afterwards the queue is
+ * found critically low mark it as starving in the bitmap of starving FLs.
+ *
+ * Returns the number of buffers allocated.
+ */
+static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q,
+                                      int n)
+{
+       struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, fl);
+       unsigned int cred = q->avail;
+       __be64 *d = &q->desc[q->pidx];
+       struct rx_sw_desc *sd = &q->sdesc[q->pidx];
+       unsigned int buf_size_idx = RX_SMALL_MTU_BUF;
+
+       while (n--) {
+               struct rte_mbuf *mbuf = cxgbe_rxmbuf_alloc(rxq->rspq.mb_pool);
+               dma_addr_t mapping;
+
+               if (!mbuf) {
+                       dev_debug(adap, "%s: mbuf alloc failed\n", __func__);
+                       q->alloc_failed++;
+                       rxq->rspq.eth_dev->data->rx_mbuf_alloc_failed++;
+                       goto out;
+               }
+
+               mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+               mbuf->next = NULL;
+
+               mapping = (dma_addr_t)(mbuf->buf_physaddr + mbuf->data_off);
+
+               mapping |= buf_size_idx;
+               *d++ = cpu_to_be64(mapping);
+               set_rx_sw_desc(sd, mbuf, mapping);
+               sd++;
+
+               q->avail++;
+               if (++q->pidx == q->size) {
+                       q->pidx = 0;
+                       sd = q->sdesc;
+                       d = q->desc;
+               }
+       }
+
+out:    cred = q->avail - cred;
+       q->pend_cred += cred;
+       ring_fl_db(adap, q);
+
+       if (unlikely(fl_starving(adap, q))) {
+               /*
+                * Make sure data has been written to free list
+                */
+               wmb();
+               q->low++;
+       }
+
+       return cred;
+}
+
+/**
+ * refill_fl - refill an SGE Rx buffer ring with mbufs
+ * @adap: the adapter
+ * @q: the ring to refill
+ * @n: the number of new buffers to allocate
+ *
+ * (Re)populate an SGE free-buffer queue with up to @n new packet buffers,
+ * allocated with the supplied gfp flags.  The caller must assure that
+ * @n does not exceed the queue's capacity.  Returns the number of buffers
+ * allocated.
+ */
+static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n)
+{
+       return refill_fl_usembufs(adap, q, n);
+}
+
+static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
+{
+       refill_fl(adap, fl, min(MAX_RX_REFILL, fl_cap(fl) - fl->avail));
+}
+
+/**
+ * alloc_ring - allocate resources for an SGE descriptor ring
+ * @dev: the PCI device's core device
+ * @nelem: the number of descriptors
+ * @elem_size: the size of each descriptor
+ * @sw_size: the size of the SW state associated with each ring element
+ * @phys: the physical address of the allocated ring
+ * @metadata: address of the array holding the SW state for the ring
+ * @stat_size: extra space in HW ring for status information
+ * @node: preferred node for memory allocations
+ *
+ * Allocates resources for an SGE descriptor ring, such as Tx queues,
+ * free buffer lists, or response queues.  Each SGE ring requires
+ * space for its HW descriptors plus, optionally, space for the SW state
+ * associated with each HW entry (the metadata).  The function returns
+ * three values: the virtual address for the HW ring (the return value
+ * of the function), the bus address of the HW ring, and the address
+ * of the SW ring.
+ */
+static void *alloc_ring(size_t nelem, size_t elem_size,
+                       size_t sw_size, dma_addr_t *phys, void *metadata,
+                       size_t stat_size, __rte_unused uint16_t queue_id,
+                       int socket_id, const char *z_name,
+                       const char *z_name_sw)
+{
+       size_t len = CXGBE_MAX_RING_DESC_SIZE * elem_size + stat_size;
+       const struct rte_memzone *tz;
+       void *s = NULL;
+
+       dev_debug(adapter, "%s: nelem = %zu; elem_size = %zu; sw_size = %zu; "
+                 "stat_size = %zu; queue_id = %u; socket_id = %d; z_name = %s;"
+                 " z_name_sw = %s\n", __func__, nelem, elem_size, sw_size,
+                 stat_size, queue_id, socket_id, z_name, z_name_sw);
+
+       tz = rte_memzone_lookup(z_name);
+       if (tz) {
+               dev_debug(adapter, "%s: tz exists...returning existing..\n",
+                         __func__);
+               goto alloc_sw_ring;
+       }
+
+       /*
+        * Allocate TX/RX ring hardware descriptors. A memzone large enough to
+        * handle the maximum ring size is allocated in order to allow for
+        * resizing in later calls to the queue setup function.
+        */
+       tz = rte_memzone_reserve_aligned(z_name, len, socket_id, 0, 4096);
+       if (!tz)
+               return NULL;
+
+alloc_sw_ring:
+       memset(tz->addr, 0, len);
+       if (sw_size) {
+               s = rte_zmalloc_socket(z_name_sw, nelem * sw_size,
+                                      RTE_CACHE_LINE_SIZE, socket_id);
+
+               if (!s) {
+                       dev_err(adapter, "%s: failed to get sw_ring memory\n",
+                               __func__);
+                       return NULL;
+               }
+       }
+       if (metadata)
+               *(void **)metadata = s;
+
+       *phys = (uint64_t)tz->phys_addr;
+       return tz->addr;
+}
+
+/**
+ * t4_pktgl_to_mbuf_usembufs - build an mbuf from a packet gather list
+ * @gl: the gather list
+ *
+ * Builds an mbuf from the given packet gather list.  Returns the mbuf or
+ * %NULL if mbuf allocation failed.
+ */
+static struct rte_mbuf *t4_pktgl_to_mbuf_usembufs(const struct pkt_gl *gl)
+{
+       /*
+        * If there's only one mbuf fragment, just return that.
+        */
+       if (likely(gl->nfrags == 1))
+               return gl->mbufs[0];
+
+       return NULL;
+}
+
+/**
+ * t4_pktgl_to_mbuf - build an mbuf from a packet gather list
+ * @gl: the gather list
+ *
+ * Builds an mbuf from the given packet gather list.  Returns the mbuf or
+ * %NULL if mbuf allocation failed.
+ */
+static struct rte_mbuf *t4_pktgl_to_mbuf(const struct pkt_gl *gl)
+{
+       return t4_pktgl_to_mbuf_usembufs(gl);
+}
+
+#define RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mb) \
+       ((dma_addr_t) ((mb)->buf_physaddr + (mb)->data_off))
+
+/**
+ * t4_ethrx_handler - process an ingress ethernet packet
+ * @q: the response queue that received the packet
+ * @rsp: the response queue descriptor holding the RX_PKT message
+ * @si: the gather list of packet fragments
+ *
+ * Process an ingress ethernet packet and deliver it to the stack.
+ */
+int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
+                    const struct pkt_gl *si)
+{
+       struct rte_mbuf *mbuf;
+       const struct cpl_rx_pkt *pkt;
+       const struct rss_header *rss_hdr;
+       bool csum_ok;
+       struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
+
+       rss_hdr = (const void *)rsp;
+       pkt = (const void *)&rsp[1];
+       csum_ok = pkt->csum_calc && !pkt->err_vec;
+
+       mbuf = t4_pktgl_to_mbuf(si);
+       if (unlikely(!mbuf)) {
+               rxq->stats.rx_drops++;
+               return 0;
+       }
+
+       mbuf->port = pkt->iff;
+       if (pkt->l2info & htonl(F_RXF_IP)) {
+               mbuf->ol_flags |= PKT_RX_IPV4_HDR;
+               if (unlikely(!csum_ok))
+                       mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
+
+               if ((pkt->l2info & htonl(F_RXF_UDP | F_RXF_TCP)) && !csum_ok)
+                       mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
+       } else if (pkt->l2info & htonl(F_RXF_IP6)) {
+               mbuf->ol_flags |= PKT_RX_IPV6_HDR;
+       }
+
+       mbuf->port = pkt->iff;
+
+       if (!rss_hdr->filter_tid && rss_hdr->hash_type) {
+               mbuf->ol_flags |= PKT_RX_RSS_HASH;
+               mbuf->hash.rss = ntohl(rss_hdr->hash_val);
+       }
+
+       if (pkt->vlan_ex) {
+               mbuf->ol_flags |= PKT_RX_VLAN_PKT;
+               mbuf->vlan_tci = ntohs(pkt->vlan);
+       }
+       rxq->stats.pkts++;
+       rxq->stats.rx_bytes += mbuf->pkt_len;
+
+       return 0;
+}
+
+/**
+ * restore_rx_bufs - put back a packet's Rx buffers
+ * @q: the SGE free list
+ * @frags: number of FL buffers to restore
+ *
+ * Puts back on an FL the Rx buffers.  The buffers have already been
+ * unmapped and are left unmapped, we mark them so to prevent further
+ * unmapping attempts.
+ *
+ * This function undoes a series of @unmap_rx_buf calls when we find out
+ * that the current packet can't be processed right away afterall and we
+ * need to come back to it later.  This is a very rare event and there's
+ * no effort to make this particularly efficient.
+ */
+static void restore_rx_bufs(struct sge_fl *q, int frags)
+{
+       while (frags--) {
+               if (q->cidx == 0)
+                       q->cidx = q->size - 1;
+               else
+                       q->cidx--;
+               q->avail++;
+       }
+}
+
+/**
+ * is_new_response - check if a response is newly written
+ * @r: the response descriptor
+ * @q: the response queue
+ *
+ * Returns true if a response descriptor contains a yet unprocessed
+ * response.
+ */
+static inline bool is_new_response(const struct rsp_ctrl *r,
+                                  const struct sge_rspq *q)
+{
+       return (r->u.type_gen >> S_RSPD_GEN) == q->gen;
+}
+
+#define CXGB4_MSG_AN ((void *)1)
+
+/**
+ * rspq_next - advance to the next entry in a response queue
+ * @q: the queue
+ *
+ * Updates the state of a response queue to advance it to the next entry.
+ */
+static inline void rspq_next(struct sge_rspq *q)
+{
+       q->cur_desc = (const __be64 *)((const char *)q->cur_desc + q->iqe_len);
+       if (unlikely(++q->cidx == q->size)) {
+               q->cidx = 0;
+               q->gen ^= 1;
+               q->cur_desc = q->desc;
+       }
+}
+
+/**
+ * process_responses - process responses from an SGE response queue
+ * @q: the ingress queue to process
+ * @budget: how many responses can be processed in this round
+ * @rx_pkts: mbuf to put the pkts
+ *
+ * Process responses from an SGE response queue up to the supplied budget.
+ * Responses include received packets as well as control messages from FW
+ * or HW.
+ *
+ * Additionally choose the interrupt holdoff time for the next interrupt
+ * on this queue.  If the system is under memory shortage use a fairly
+ * long delay to help recovery.
+ */
+static int process_responses(struct sge_rspq *q, int budget,
+                            struct rte_mbuf **rx_pkts)
+{
+       int ret = 0, rsp_type;
+       int budget_left = budget;
+       const struct rsp_ctrl *rc;
+       struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
+       struct adapter *adapter = q->adapter;
+
+       while (likely(budget_left)) {
+               rc = (const struct rsp_ctrl *)
+                    ((const char *)q->cur_desc + (q->iqe_len - sizeof(*rc)));
+
+               if (!is_new_response(rc, q))
+                       break;
+
+               /*
+                * Ensure response has been read
+                */
+               rmb();
+               rsp_type = G_RSPD_TYPE(rc->u.type_gen);
+
+               if (likely(rsp_type == X_RSPD_TYPE_FLBUF)) {
+                       struct pkt_gl si;
+                       const struct rx_sw_desc *rsd;
+                       struct rte_mbuf *pkt = NULL;
+                       u32 len = ntohl(rc->pldbuflen_qid), bufsz, frags;
+
+                       si.usembufs = rxq->usembufs;
+                       /*
+                        * In "use mbufs" mode, we don't pack multiple
+                        * ingress packets per buffer (mbuf) so we
+                        * should _always_ get a "New Buffer" flags
+                        * from the SGE.  Also, since we hand the
+                        * mbuf's up to the host stack for it to
+                        * eventually free, we don't release the mbuf's
+                        * in the driver (in contrast to the "packed
+                        * page" mode where the driver needs to
+                        * release its reference on the page buffers).
+                        */
+                       BUG_ON(!(len & F_RSPD_NEWBUF));
+                       len = G_RSPD_LEN(len);
+                       si.tot_len = len;
+
+                       /* gather packet fragments */
+                       for (frags = 0; len; frags++) {
+                               rsd = &rxq->fl.sdesc[rxq->fl.cidx];
+                               bufsz = min(get_buf_size(adapter, rsd), len);
+                               pkt = rsd->buf;
+                               pkt->data_len = bufsz;
+                               pkt->pkt_len = bufsz;
+                               si.mbufs[frags] = pkt;
+                               len -= bufsz;
+                               unmap_rx_buf(&rxq->fl);
+                       }
+
+                       si.va = RTE_PTR_ADD(si.mbufs[0]->buf_addr,
+                                           si.mbufs[0]->data_off);
+                       rte_prefetch1(si.va);
+
+                       /*
+                        * For the "use mbuf" case here, we can end up
+                        * chewing through our Free List very rapidly
+                        * with one entry per Ingress packet getting
+                        * consumed.  So if the handler() successfully
+                        * consumed the mbuf, check to see if we can
+                        * refill the Free List incrementally in the
+                        * loop ...
+                        */
+                       si.nfrags = frags;
+                       ret = q->handler(q, q->cur_desc, &si);
+
+                       if (unlikely(ret != 0)) {
+                               restore_rx_bufs(&rxq->fl, frags);
+                       } else {
+                               rx_pkts[budget - budget_left] = pkt;
+                               if (fl_cap(&rxq->fl) - rxq->fl.avail >= 8)
+                                       __refill_fl(q->adapter, &rxq->fl);
+                       }
+
+               } else if (likely(rsp_type == X_RSPD_TYPE_CPL)) {
+                       ret = q->handler(q, q->cur_desc, NULL);
+               } else {
+                       ret = q->handler(q, (const __be64 *)rc, CXGB4_MSG_AN);
+               }
+
+               if (unlikely(ret)) {
+                       /* couldn't process descriptor, back off for recovery */
+                       q->next_intr_params = V_QINTR_TIMER_IDX(NOMEM_TMR_IDX);
+                       break;
+               }
+
+               rspq_next(q);
+               budget_left--;
+       }
+
+       /*
+        * If this is a Response Queue with an associated Free List and
+        * there's room for another chunk of new Free List buffer pointers,
+        * refill the Free List.
+        */
+
+       if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 8)
+               __refill_fl(q->adapter, &rxq->fl);
+
+       return budget - budget_left;
+}
+
+int cxgbe_poll(struct sge_rspq *q, struct rte_mbuf **rx_pkts,
+              unsigned int budget, unsigned int *work_done)
+{
+       unsigned int params;
+       u32 val;
+       int err = 0;
+
+       *work_done = process_responses(q, budget, rx_pkts);
+       params = V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX);
+       q->next_intr_params = params;
+       val = V_CIDXINC(*work_done) | V_SEINTARM(params);
+
+       if (*work_done) {
+               /*
+                * If we don't have access to the new User GTS (T5+),
+                * use the old doorbell mechanism; otherwise use the new
+                * BAR2 mechanism.
+                */
+               if (unlikely(!q->bar2_addr))
+                       t4_write_reg(q->adapter, MYPF_REG(A_SGE_PF_GTS),
+                                    val | V_INGRESSQID((u32)q->cntxt_id));
+               else {
+                       writel(val | V_INGRESSQID(q->bar2_qid),
+                              (void *)((uintptr_t)q->bar2_addr +
+                              SGE_UDB_GTS));
+                       /*
+                        * This Write memory Barrier will force the write to
+                        * the User Doorbell area to be flushed.
+                        */
+                       wmb();
+               }
+       }
+
+       return err;
+}
+
+/**
+ * bar2_address - return the BAR2 address for an SGE Queue's Registers
+ * @adapter: the adapter
+ * @qid: the SGE Queue ID
+ * @qtype: the SGE Queue Type (Egress or Ingress)
+ * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
+ *
+ * Returns the BAR2 address for the SGE Queue Registers associated with
+ * @qid.  If BAR2 SGE Registers aren't available, returns NULL.  Also
+ * returns the BAR2 Queue ID to be used with writes to the BAR2 SGE
+ * Queue Registers.  If the BAR2 Queue ID is 0, then "Inferred Queue ID"
+ * Registers are supported (e.g. the Write Combining Doorbell Buffer).
+ */
+static void __iomem *bar2_address(struct adapter *adapter, unsigned int qid,
+                                 enum t4_bar2_qtype qtype,
+                                 unsigned int *pbar2_qid)
+{
+       u64 bar2_qoffset;
+       int ret;
+
+       ret = t4_bar2_sge_qregs(adapter, qid, qtype, &bar2_qoffset, pbar2_qid);
+       if (ret)
+               return NULL;
+
+       return adapter->bar2 + bar2_qoffset;
+}
+
+int t4_sge_eth_rxq_start(struct adapter *adap, struct sge_rspq *rq)
+{
+       struct sge_eth_rxq *rxq = container_of(rq, struct sge_eth_rxq, rspq);
+       unsigned int fl_id = rxq->fl.size ? rxq->fl.cntxt_id : 0xffff;
+
+       return t4_iq_start_stop(adap, adap->mbox, true, adap->pf, 0,
+                               rq->cntxt_id, fl_id, 0xffff);
+}
+
+int t4_sge_eth_rxq_stop(struct adapter *adap, struct sge_rspq *rq)
+{
+       struct sge_eth_rxq *rxq = container_of(rq, struct sge_eth_rxq, rspq);
+       unsigned int fl_id = rxq->fl.size ? rxq->fl.cntxt_id : 0xffff;
+
+       return t4_iq_start_stop(adap, adap->mbox, false, adap->pf, 0,
+                               rq->cntxt_id, fl_id, 0xffff);
+}
+
+/*
+ * @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0
+ * @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map
+ */
+int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
+                    struct rte_eth_dev *eth_dev, int intr_idx,
+                    struct sge_fl *fl, rspq_handler_t hnd, int cong,
+                    struct rte_mempool *mp, int queue_id, int socket_id)
+{
+       int ret, flsz = 0;
+       struct fw_iq_cmd c;
+       struct sge *s = &adap->sge;
+       struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+       char z_name[RTE_MEMZONE_NAMESIZE];
+       char z_name_sw[RTE_MEMZONE_NAMESIZE];
+       unsigned int nb_refill;
+
+       /* Size needs to be multiple of 16, including status entry. */
+       iq->size = roundup(iq->size, 16);
+
+       snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
+                eth_dev->driver->pci_drv.name, fwevtq ? "fwq_ring" : "rx_ring",
+                eth_dev->data->port_id, queue_id);
+       snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
+
+       iq->desc = alloc_ring(iq->size, iq->iqe_len, 0, &iq->phys_addr, NULL, 0,
+                             queue_id, socket_id, z_name, z_name_sw);
+       if (!iq->desc)
+               return -ENOMEM;
+
+       memset(&c, 0, sizeof(c));
+       c.op_to_vfn = htonl(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
+                           F_FW_CMD_WRITE | F_FW_CMD_EXEC |
+                           V_FW_IQ_CMD_PFN(adap->pf) | V_FW_IQ_CMD_VFN(0));
+       c.alloc_to_len16 = htonl(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
+                                (sizeof(c) / 16));
+       c.type_to_iqandstindex =
+               htonl(V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
+                     V_FW_IQ_CMD_IQASYNCH(fwevtq) |
+                     V_FW_IQ_CMD_VIID(pi->viid) |
+                     V_FW_IQ_CMD_IQANDST(intr_idx < 0) |
+                     V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT) |
+                     V_FW_IQ_CMD_IQANDSTINDEX(intr_idx >= 0 ? intr_idx :
+                                                              -intr_idx - 1));
+       c.iqdroprss_to_iqesize =
+               htons(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
+                     F_FW_IQ_CMD_IQGTSMODE |
+                     V_FW_IQ_CMD_IQINTCNTTHRESH(iq->pktcnt_idx) |
+                     V_FW_IQ_CMD_IQESIZE(ilog2(iq->iqe_len) - 4));
+       c.iqsize = htons(iq->size);
+       c.iqaddr = cpu_to_be64(iq->phys_addr);
+       if (cong >= 0)
+               c.iqns_to_fl0congen = htonl(F_FW_IQ_CMD_IQFLINTCONGEN);
+
+       if (fl) {
+               struct sge_eth_rxq *rxq = container_of(fl, struct sge_eth_rxq,
+                                                      fl);
+               enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip);
+
+               /*
+                * Allocate the ring for the hardware free list (with space
+                * for its status page) along with the associated software
+                * descriptor ring.  The free list size needs to be a multiple
+                * of the Egress Queue Unit and at least 2 Egress Units larger
+                * than the SGE's Egress Congrestion Threshold
+                * (fl_starve_thres - 1).
+                */
+               if (fl->size < s->fl_starve_thres - 1 + 2 * 8)
+                       fl->size = s->fl_starve_thres - 1 + 2 * 8;
+               fl->size = roundup(fl->size, 8);
+
+               snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
+                        eth_dev->driver->pci_drv.name,
+                        fwevtq ? "fwq_ring" : "fl_ring",
+                        eth_dev->data->port_id, queue_id);
+               snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
+
+               fl->desc = alloc_ring(fl->size, sizeof(__be64),
+                                     sizeof(struct rx_sw_desc),
+                                     &fl->addr, &fl->sdesc, s->stat_len,
+                                     queue_id, socket_id, z_name, z_name_sw);
+
+               if (!fl->desc)
+                       goto fl_nomem;
+
+               flsz = fl->size / 8 + s->stat_len / sizeof(struct tx_desc);
+               c.iqns_to_fl0congen |=
+                       htonl(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
+                             (unlikely(rxq->usembufs) ?
+                              0 : F_FW_IQ_CMD_FL0PACKEN) |
+                             F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
+                             F_FW_IQ_CMD_FL0PADEN);
+               if (cong >= 0)
+                       c.iqns_to_fl0congen |=
+                               htonl(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
+                                     F_FW_IQ_CMD_FL0CONGCIF |
+                                     F_FW_IQ_CMD_FL0CONGEN);
+
+               /* In T6, for egress queue type FL there is internal overhead
+                * of 16B for header going into FLM module.
+                * Hence maximum allowed burst size will be 448 bytes.
+                */
+               c.fl0dcaen_to_fl0cidxfthresh =
+                       htons(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
+                             V_FW_IQ_CMD_FL0FBMAX((chip <= CHELSIO_T5) ?
+                             X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B));
+               c.fl0size = htons(flsz);
+               c.fl0addr = cpu_to_be64(fl->addr);
+       }
+
+       ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
+       if (ret)
+               goto err;
+
+       iq->cur_desc = iq->desc;
+       iq->cidx = 0;
+       iq->gen = 1;
+       iq->next_intr_params = iq->intr_params;
+       iq->cntxt_id = ntohs(c.iqid);
+       iq->abs_id = ntohs(c.physiqid);
+       iq->bar2_addr = bar2_address(adap, iq->cntxt_id, T4_BAR2_QTYPE_INGRESS,
+                                    &iq->bar2_qid);
+       iq->size--;                           /* subtract status entry */
+       iq->eth_dev = eth_dev;
+       iq->handler = hnd;
+       iq->mb_pool = mp;
+
+       /* set offset to -1 to distinguish ingress queues without FL */
+       iq->offset = fl ? 0 : -1;
+
+       if (fl) {
+               fl->cntxt_id = ntohs(c.fl0id);
+               fl->avail = 0;
+               fl->pend_cred = 0;
+               fl->pidx = 0;
+               fl->cidx = 0;
+               fl->alloc_failed = 0;
+
+               /*
+                * Note, we must initialize the BAR2 Free List User Doorbell
+                * information before refilling the Free List!
+                */
+               fl->bar2_addr = bar2_address(adap, fl->cntxt_id,
+                                            T4_BAR2_QTYPE_EGRESS,
+                                            &fl->bar2_qid);
+
+               nb_refill = refill_fl(adap, fl, fl_cap(fl));
+               if (nb_refill != fl_cap(fl)) {
+                       ret = -ENOMEM;
+                       dev_err(adap, "%s: mbuf alloc failed with error: %d\n",
+                               __func__, ret);
+                       goto refill_fl_err;
+               }
+       }
+
+       /*
+        * For T5 and later we attempt to set up the Congestion Manager values
+        * of the new RX Ethernet Queue.  This should really be handled by
+        * firmware because it's more complex than any host driver wants to
+        * get involved with and it's different per chip and this is almost
+        * certainly wrong.  Formware would be wrong as well, but it would be
+        * a lot easier to fix in one place ...  For now we do something very
+        * simple (and hopefully less wrong).
+        */
+       if (!is_t4(adap->params.chip) && cong >= 0) {
+               u32 param, val;
+               int i;
+
+               param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
+                        V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
+                        V_FW_PARAMS_PARAM_YZ(iq->cntxt_id));
+               if (cong == 0) {
+                       val = V_CONMCTXT_CNGTPMODE(X_CONMCTXT_CNGTPMODE_QUEUE);
+               } else {
+                       val = V_CONMCTXT_CNGTPMODE(
+                                       X_CONMCTXT_CNGTPMODE_CHANNEL);
+                       for (i = 0; i < 4; i++) {
+                               if (cong & (1 << i))
+                                       val |= V_CONMCTXT_CNGCHMAP(1 <<
+                                                                  (i << 2));
+                       }
+               }
+               ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1,
+                                   &param, &val);
+               if (ret)
+                       dev_warn(adap->pdev_dev, "Failed to set Congestion 
Manager Context for Ingress Queue %d: %d\n",
+                                iq->cntxt_id, -ret);
+       }
+
+       return 0;
+
+refill_fl_err:
+       t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
+                  iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff);
+fl_nomem:
+       ret = -ENOMEM;
+err:
+       iq->cntxt_id = 0;
+       iq->abs_id = 0;
+       if (iq->desc)
+               iq->desc = NULL;
+
+       if (fl && fl->desc) {
+               rte_free(fl->sdesc);
+               fl->cntxt_id = 0;
+               fl->sdesc = NULL;
+               fl->desc = NULL;
+       }
+       return ret;
+}
+
+static void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq,
+                        struct sge_fl *fl)
+{
+       unsigned int fl_id = fl ? fl->cntxt_id : 0xffff;
+
+       t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
+                  rq->cntxt_id, fl_id, 0xffff);
+       rq->cntxt_id = 0;
+       rq->abs_id = 0;
+       rq->desc = NULL;
+
+       if (fl) {
+               free_rx_bufs(fl, fl->avail);
+               rte_free(fl->sdesc);
+               fl->sdesc = NULL;
+               fl->cntxt_id = 0;
+               fl->desc = NULL;
+       }
+}
+
+void t4_sge_eth_rxq_release(struct adapter *adap, struct sge_eth_rxq *rxq)
+{
+       if (rxq->rspq.desc) {
+               t4_sge_eth_rxq_stop(adap, &rxq->rspq);
+               free_rspq_fl(adap, &rxq->rspq, rxq->fl.size ? &rxq->fl : NULL);
+       }
+}
+
+/**
  * t4_sge_init - initialize SGE
  * @adap: the adapter
  *
-- 
2.4.1

Reply via email to