Author: mw Date: Thu Oct 31 15:44:26 2019 New Revision: 354220 URL: https://svnweb.freebsd.org/changeset/base/354220
Log: Split Rx/Tx from initialization code in ENA driver Move Rx/Tx routines to separate file. Some functions: * ena_restore_device, * ena_destroy_device, * ena_up, * ena_down, * ena_refill_rx_bufs could be reused in upcoming netmap code in the driver. To make it possible, they were moved to ena.h header. Differential Revision: https://reviews.freebsd.org/D21933 Submitted by: Rafal Kozik <r...@semihalf.com> Obtained from: Semihalf Sponsored by: Amazon, Inc. Added: head/sys/dev/ena/ena_datapath.c (contents, props changed) head/sys/dev/ena/ena_datapath.h (contents, props changed) Modified: head/sys/dev/ena/ena.c head/sys/dev/ena/ena.h head/sys/modules/ena/Makefile Modified: head/sys/dev/ena/ena.c ============================================================================== --- head/sys/dev/ena/ena.c Thu Oct 31 15:39:54 2019 (r354219) +++ head/sys/dev/ena/ena.c Thu Oct 31 15:44:26 2019 (r354220) @@ -76,6 +76,7 @@ __FBSDID("$FreeBSD$"); #include <vm/vm.h> #include <vm/pmap.h> +#include "ena_datapath.h" #include "ena.h" #include "ena_sysctl.h" @@ -102,7 +103,6 @@ static int ena_setup_tx_resources(struct ena_adapter * static void ena_free_tx_resources(struct ena_adapter *, int); static int ena_setup_all_tx_resources(struct ena_adapter *); static void ena_free_all_tx_resources(struct ena_adapter *); -static inline int validate_rx_req_id(struct ena_ring *, uint16_t); static int ena_setup_rx_resources(struct ena_adapter *, unsigned int); static void ena_free_rx_resources(struct ena_adapter *, unsigned int); static int ena_setup_all_rx_resources(struct ena_adapter *); @@ -111,7 +111,6 @@ static inline int ena_alloc_rx_mbuf(struct ena_adapter struct ena_rx_buffer *); static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *, struct ena_rx_buffer *); -static int ena_refill_rx_bufs(struct ena_ring *, uint32_t); static void ena_free_rx_bufs(struct ena_adapter *, unsigned int); static void ena_refill_all_rx_bufs(struct ena_adapter *); static void ena_free_all_rx_bufs(struct ena_adapter *); @@ -121,16 +120,6 @@ static void ena_destroy_all_tx_queues(struct ena_adapt static void ena_destroy_all_rx_queues(struct ena_adapter *); static void ena_destroy_all_io_queues(struct ena_adapter *); static int ena_create_io_queues(struct ena_adapter *); -static int ena_tx_cleanup(struct ena_ring *); -static int ena_rx_cleanup(struct ena_ring *); -static inline int validate_tx_req_id(struct ena_ring *, uint16_t); -static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *, - struct mbuf *); -static struct mbuf* ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *, - struct ena_com_rx_ctx *, uint16_t *); -static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *, - struct mbuf *); -static void ena_cleanup(void *arg, int pending); static int ena_handle_msix(void *); static int ena_enable_msix(struct ena_adapter *); static void ena_setup_mgmnt_intr(struct ena_adapter *); @@ -144,8 +133,6 @@ static void ena_disable_msix(struct ena_adapter *); static void ena_unmask_all_io_irqs(struct ena_adapter *); static int ena_rss_configure(struct ena_adapter *); static int ena_up_complete(struct ena_adapter *); -static int ena_up(struct ena_adapter *); -static void ena_down(struct ena_adapter *); static uint64_t ena_get_counter(if_t, ift_counter); static int ena_media_change(if_t); static void ena_media_status(if_t, struct ifmediareq *); @@ -156,15 +143,6 @@ static void ena_update_host_info(struct ena_admin_host static void ena_update_hwassist(struct ena_adapter *); static int ena_setup_ifnet(device_t, struct ena_adapter *, struct ena_com_dev_get_features_ctx *); -static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *); -static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, - struct mbuf **mbuf); -static void ena_dmamap_llq(void *, bus_dma_segment_t *, int, int); -static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **); -static void ena_start_xmit(struct ena_ring *); -static int ena_mq_start(if_t, struct mbuf *); -static void ena_deferred_mq_start(void *, int); -static void ena_qflush(if_t); static int ena_enable_wc(struct resource *); static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *, struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *); @@ -766,25 +744,6 @@ ena_free_all_tx_resources(struct ena_adapter *adapter) ena_free_tx_resources(adapter, i); } -static inline int -validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id) -{ - if (likely(req_id < rx_ring->ring_size)) - return (0); - - device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n", - req_id); - counter_u64_add(rx_ring->rx_stats.bad_req_id, 1); - - /* Trigger device reset */ - if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter))) { - rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID; - ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter); - } - - return (EFAULT); -} - /** * ena_setup_rx_resources - allocate Rx resources (Descriptors) * @adapter: network interface device structure @@ -1024,7 +983,7 @@ ena_free_rx_mbuf(struct ena_adapter *adapter, struct e * @num: number of descriptors to refill * Refills the ring with newly allocated DMA-mapped mbufs for receiving **/ -static int +int ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num) { struct ena_adapter *adapter = rx_ring->adapter; @@ -1227,30 +1186,6 @@ ena_destroy_all_io_queues(struct ena_adapter *adapter) ena_destroy_all_rx_queues(adapter); } -static inline int -validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id) -{ - struct ena_adapter *adapter = tx_ring->adapter; - struct ena_tx_buffer *tx_info = NULL; - - if (likely(req_id < tx_ring->ring_size)) { - tx_info = &tx_ring->tx_buffer_info[req_id]; - if (tx_info->mbuf != NULL) - return (0); - device_printf(adapter->pdev, - "tx_info doesn't have valid mbuf\n"); - } - - device_printf(adapter->pdev, "Invalid req_id: %hu\n", req_id); - counter_u64_add(tx_ring->tx_stats.bad_req_id, 1); - - /* Trigger device reset */ - adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID; - ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); - - return (EFAULT); -} - static int ena_create_io_queues(struct ena_adapter *adapter) { @@ -1344,475 +1279,6 @@ err_tx: return (ENXIO); } -/** - * ena_tx_cleanup - clear sent packets and corresponding descriptors - * @tx_ring: ring for which we want to clean packets - * - * Once packets are sent, we ask the device in a loop for no longer used - * descriptors. We find the related mbuf chain in a map (index in an array) - * and free it, then update ring state. - * This is performed in "endless" loop, updating ring pointers every - * TX_COMMIT. The first check of free descriptor is performed before the actual - * loop, then repeated at the loop end. - **/ -static int -ena_tx_cleanup(struct ena_ring *tx_ring) -{ - struct ena_adapter *adapter; - struct ena_com_io_cq* io_cq; - uint16_t next_to_clean; - uint16_t req_id; - uint16_t ena_qid; - unsigned int total_done = 0; - int rc; - int commit = TX_COMMIT; - int budget = TX_BUDGET; - int work_done; - bool above_thresh; - - adapter = tx_ring->que->adapter; - ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); - io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; - next_to_clean = tx_ring->next_to_clean; - - do { - struct ena_tx_buffer *tx_info; - struct mbuf *mbuf; - - rc = ena_com_tx_comp_req_id_get(io_cq, &req_id); - if (unlikely(rc != 0)) - break; - - rc = validate_tx_req_id(tx_ring, req_id); - if (unlikely(rc != 0)) - break; - - tx_info = &tx_ring->tx_buffer_info[req_id]; - - mbuf = tx_info->mbuf; - - tx_info->mbuf = NULL; - bintime_clear(&tx_info->timestamp); - - /* Map is no longer required */ - if (tx_info->head_mapped == true) { - bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_head, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(adapter->tx_buf_tag, - tx_info->map_head); - tx_info->head_mapped = false; - } - if (tx_info->seg_mapped == true) { - bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_seg, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(adapter->tx_buf_tag, - tx_info->map_seg); - tx_info->seg_mapped = false; - } - - ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d mbuf %p completed\n", - tx_ring->qid, mbuf); - - m_freem(mbuf); - - total_done += tx_info->tx_descs; - - tx_ring->free_tx_ids[next_to_clean] = req_id; - next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean, - tx_ring->ring_size); - - if (unlikely(--commit == 0)) { - commit = TX_COMMIT; - /* update ring state every TX_COMMIT descriptor */ - tx_ring->next_to_clean = next_to_clean; - ena_com_comp_ack( - &adapter->ena_dev->io_sq_queues[ena_qid], - total_done); - ena_com_update_dev_comp_head(io_cq); - total_done = 0; - } - } while (likely(--budget)); - - work_done = TX_BUDGET - budget; - - ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d done. total pkts: %d\n", - tx_ring->qid, work_done); - - /* If there is still something to commit update ring state */ - if (likely(commit != TX_COMMIT)) { - tx_ring->next_to_clean = next_to_clean; - ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid], - total_done); - ena_com_update_dev_comp_head(io_cq); - } - - /* - * Need to make the rings circular update visible to - * ena_xmit_mbuf() before checking for tx_ring->running. - */ - mb(); - - above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, - ENA_TX_RESUME_THRESH); - if (unlikely(!tx_ring->running && above_thresh)) { - ENA_RING_MTX_LOCK(tx_ring); - above_thresh = - ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, - ENA_TX_RESUME_THRESH); - if (!tx_ring->running && above_thresh) { - tx_ring->running = true; - counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1); - taskqueue_enqueue(tx_ring->enqueue_tq, - &tx_ring->enqueue_task); - } - ENA_RING_MTX_UNLOCK(tx_ring); - } - - return (work_done); -} - -static void -ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, - struct mbuf *mbuf) -{ - struct ena_adapter *adapter = rx_ring->adapter; - - if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) { - mbuf->m_pkthdr.flowid = ena_rx_ctx->hash; - - if (ena_rx_ctx->frag && - (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) { - M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); - return; - } - - switch (ena_rx_ctx->l3_proto) { - case ENA_ETH_IO_L3_PROTO_IPV4: - switch (ena_rx_ctx->l4_proto) { - case ENA_ETH_IO_L4_PROTO_TCP: - M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4); - break; - case ENA_ETH_IO_L4_PROTO_UDP: - M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4); - break; - default: - M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4); - } - break; - case ENA_ETH_IO_L3_PROTO_IPV6: - switch (ena_rx_ctx->l4_proto) { - case ENA_ETH_IO_L4_PROTO_TCP: - M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6); - break; - case ENA_ETH_IO_L4_PROTO_UDP: - M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6); - break; - default: - M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6); - } - break; - case ENA_ETH_IO_L3_PROTO_UNKNOWN: - M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); - break; - default: - M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); - } - } else { - mbuf->m_pkthdr.flowid = rx_ring->qid; - M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); - } -} - -/** - * ena_rx_mbuf - assemble mbuf from descriptors - * @rx_ring: ring for which we want to clean packets - * @ena_bufs: buffer info - * @ena_rx_ctx: metadata for this packet(s) - * @next_to_clean: ring pointer, will be updated only upon success - * - **/ -static struct mbuf* -ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs, - struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean) -{ - struct mbuf *mbuf; - struct ena_rx_buffer *rx_info; - struct ena_adapter *adapter; - unsigned int descs = ena_rx_ctx->descs; - int rc; - uint16_t ntc, len, req_id, buf = 0; - - ntc = *next_to_clean; - adapter = rx_ring->adapter; - - len = ena_bufs[buf].len; - req_id = ena_bufs[buf].req_id; - rc = validate_rx_req_id(rx_ring, req_id); - if (unlikely(rc != 0)) - return (NULL); - - rx_info = &rx_ring->rx_buffer_info[req_id]; - if (unlikely(rx_info->mbuf == NULL)) { - device_printf(adapter->pdev, "NULL mbuf in rx_info"); - return (NULL); - } - - ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx\n", - rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr); - - bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, - BUS_DMASYNC_POSTREAD); - mbuf = rx_info->mbuf; - mbuf->m_flags |= M_PKTHDR; - mbuf->m_pkthdr.len = len; - mbuf->m_len = len; - mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp; - - /* Fill mbuf with hash key and it's interpretation for optimization */ - ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf); - - ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d\n", - mbuf, mbuf->m_flags, mbuf->m_pkthdr.len); - - /* DMA address is not needed anymore, unmap it */ - bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map); - - rx_info->mbuf = NULL; - rx_ring->free_rx_ids[ntc] = req_id; - ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size); - - /* - * While we have more than 1 descriptors for one rcvd packet, append - * other mbufs to the main one - */ - while (--descs) { - ++buf; - len = ena_bufs[buf].len; - req_id = ena_bufs[buf].req_id; - rc = validate_rx_req_id(rx_ring, req_id); - if (unlikely(rc != 0)) { - /* - * If the req_id is invalid, then the device will be - * reset. In that case we must free all mbufs that - * were already gathered. - */ - m_freem(mbuf); - return (NULL); - } - rx_info = &rx_ring->rx_buffer_info[req_id]; - - if (unlikely(rx_info->mbuf == NULL)) { - device_printf(adapter->pdev, "NULL mbuf in rx_info"); - /* - * If one of the required mbufs was not allocated yet, - * we can break there. - * All earlier used descriptors will be reallocated - * later and not used mbufs can be reused. - * The next_to_clean pointer will not be updated in case - * of an error, so caller should advance it manually - * in error handling routine to keep it up to date - * with hw ring. - */ - m_freem(mbuf); - return (NULL); - } - - bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, - BUS_DMASYNC_POSTREAD); - if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) { - counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); - ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p\n", - mbuf); - } - - ena_trace(ENA_DBG | ENA_RXPTH, - "rx mbuf updated. len %d\n", mbuf->m_pkthdr.len); - - /* Free already appended mbuf, it won't be useful anymore */ - bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map); - m_freem(rx_info->mbuf); - rx_info->mbuf = NULL; - - rx_ring->free_rx_ids[ntc] = req_id; - ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size); - } - - *next_to_clean = ntc; - - return (mbuf); -} - -/** - * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum - **/ -static inline void -ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, - struct mbuf *mbuf) -{ - - /* if IP and error */ - if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) && - ena_rx_ctx->l3_csum_err)) { - /* ipv4 checksum error */ - mbuf->m_pkthdr.csum_flags = 0; - counter_u64_add(rx_ring->rx_stats.bad_csum, 1); - ena_trace(ENA_DBG, "RX IPv4 header checksum error\n"); - return; - } - - /* if TCP/UDP */ - if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) || - (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) { - if (ena_rx_ctx->l4_csum_err) { - /* TCP/UDP checksum error */ - mbuf->m_pkthdr.csum_flags = 0; - counter_u64_add(rx_ring->rx_stats.bad_csum, 1); - ena_trace(ENA_DBG, "RX L4 checksum error\n"); - } else { - mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED; - mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID; - } - } -} - -/** - * ena_rx_cleanup - handle rx irq - * @arg: ring for which irq is being handled - **/ -static int -ena_rx_cleanup(struct ena_ring *rx_ring) -{ - struct ena_adapter *adapter; - struct mbuf *mbuf; - struct ena_com_rx_ctx ena_rx_ctx; - struct ena_com_io_cq* io_cq; - struct ena_com_io_sq* io_sq; - if_t ifp; - uint16_t ena_qid; - uint16_t next_to_clean; - uint32_t refill_required; - uint32_t refill_threshold; - uint32_t do_if_input = 0; - unsigned int qid; - int rc, i; - int budget = RX_BUDGET; - - adapter = rx_ring->que->adapter; - ifp = adapter->ifp; - qid = rx_ring->que->id; - ena_qid = ENA_IO_RXQ_IDX(qid); - io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; - io_sq = &adapter->ena_dev->io_sq_queues[ena_qid]; - next_to_clean = rx_ring->next_to_clean; - - ena_trace(ENA_DBG, "rx: qid %d\n", qid); - - do { - ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; - ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size; - ena_rx_ctx.descs = 0; - bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag, - io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD); - rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx); - - if (unlikely(rc != 0)) - goto error; - - if (unlikely(ena_rx_ctx.descs == 0)) - break; - - ena_trace(ENA_DBG | ENA_RXPTH, "rx: q %d got packet from ena. " - "descs #: %d l3 proto %d l4 proto %d hash: %x\n", - rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto, - ena_rx_ctx.l4_proto, ena_rx_ctx.hash); - - /* Receive mbuf from the ring */ - mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs, - &ena_rx_ctx, &next_to_clean); - bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag, - io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD); - /* Exit if we failed to retrieve a buffer */ - if (unlikely(mbuf == NULL)) { - for (i = 0; i < ena_rx_ctx.descs; ++i) { - rx_ring->free_rx_ids[next_to_clean] = - rx_ring->ena_bufs[i].req_id; - next_to_clean = - ENA_RX_RING_IDX_NEXT(next_to_clean, - rx_ring->ring_size); - - } - break; - } - - if (((ifp->if_capenable & IFCAP_RXCSUM) != 0) || - ((ifp->if_capenable & IFCAP_RXCSUM_IPV6) != 0)) { - ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf); - } - - counter_enter(); - counter_u64_add_protected(rx_ring->rx_stats.bytes, - mbuf->m_pkthdr.len); - counter_u64_add_protected(adapter->hw_stats.rx_bytes, - mbuf->m_pkthdr.len); - counter_exit(); - /* - * LRO is only for IP/TCP packets and TCP checksum of the packet - * should be computed by hardware. - */ - do_if_input = 1; - if (((ifp->if_capenable & IFCAP_LRO) != 0) && - ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) && - (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) { - /* - * Send to the stack if: - * - LRO not enabled, or - * - no LRO resources, or - * - lro enqueue fails - */ - if ((rx_ring->lro.lro_cnt != 0) && - (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0)) - do_if_input = 0; - } - if (do_if_input != 0) { - ena_trace(ENA_DBG | ENA_RXPTH, - "calling if_input() with mbuf %p\n", mbuf); - (*ifp->if_input)(ifp, mbuf); - } - - counter_enter(); - counter_u64_add_protected(rx_ring->rx_stats.cnt, 1); - counter_u64_add_protected(adapter->hw_stats.rx_packets, 1); - counter_exit(); - } while (--budget); - - rx_ring->next_to_clean = next_to_clean; - - refill_required = ena_com_free_desc(io_sq); - refill_threshold = min_t(int, - rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER, - ENA_RX_REFILL_THRESH_PACKET); - - if (refill_required > refill_threshold) { - ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); - ena_refill_rx_bufs(rx_ring, refill_required); - } - - tcp_lro_flush_all(&rx_ring->lro); - - return (RX_BUDGET - budget); - -error: - counter_u64_add(rx_ring->rx_stats.bad_desc_num, 1); - - /* Too many desc from the device. Trigger reset */ - if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { - adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS; - ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); - } - - return (0); -} - /********************************************************************* * * MSIX & Interrupt Service routine @@ -1833,52 +1299,6 @@ ena_intr_msix_mgmnt(void *arg) ena_com_aenq_intr_handler(adapter->ena_dev, arg); } -static void -ena_cleanup(void *arg, int pending) -{ - struct ena_que *que = arg; - struct ena_adapter *adapter = que->adapter; - if_t ifp = adapter->ifp; - struct ena_ring *tx_ring; - struct ena_ring *rx_ring; - struct ena_com_io_cq* io_cq; - struct ena_eth_io_intr_reg intr_reg; - int qid, ena_qid; - int txc, rxc, i; - - if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) - return; - - ena_trace(ENA_DBG, "MSI-X TX/RX routine\n"); - - tx_ring = que->tx_ring; - rx_ring = que->rx_ring; - qid = que->id; - ena_qid = ENA_IO_TXQ_IDX(qid); - io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; - - tx_ring->first_interrupt = true; - rx_ring->first_interrupt = true; - - for (i = 0; i < CLEAN_BUDGET; ++i) { - rxc = ena_rx_cleanup(rx_ring); - txc = ena_tx_cleanup(tx_ring); - - if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) - return; - - if ((txc != TX_BUDGET) && (rxc != RX_BUDGET)) - break; - } - - /* Signal that work is done and unmask interrupt */ - ena_com_update_intr_reg(&intr_reg, - RX_IRQ_INTERVAL, - TX_IRQ_INTERVAL, - true); - ena_com_unmask_intr(io_cq, &intr_reg); -} - /** * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx * @arg: queue @@ -2289,7 +1709,7 @@ ena_up_complete(struct ena_adapter *adapter) return (0); } -static int +int ena_up(struct ena_adapter *adapter) { int rc = 0; @@ -2666,7 +2086,7 @@ ena_setup_ifnet(device_t pdev, struct ena_adapter *ada return (0); } -static void +void ena_down(struct ena_adapter *adapter) { int rc; @@ -2701,579 +2121,7 @@ ena_down(struct ena_adapter *adapter) } } -static void -ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf) -{ - struct ena_com_tx_meta *ena_meta; - struct ether_vlan_header *eh; - struct mbuf *mbuf_next; - u32 mss; - bool offload; - uint16_t etype; - int ehdrlen; - struct ip *ip; - int iphlen; - struct tcphdr *th; - int offset; - - offload = false; - ena_meta = &ena_tx_ctx->ena_meta; - mss = mbuf->m_pkthdr.tso_segsz; - - if (mss != 0) - offload = true; - - if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) - offload = true; - - if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0) - offload = true; - - if (!offload) { - ena_tx_ctx->meta_valid = 0; - return; - } - - /* Determine where frame payload starts. */ - eh = mtod(mbuf, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - etype = ntohs(eh->evl_proto); - ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - } else { - etype = ntohs(eh->evl_encap_proto); - ehdrlen = ETHER_HDR_LEN; - } - - mbuf_next = m_getptr(mbuf, ehdrlen, &offset); - ip = (struct ip *)(mtodo(mbuf_next, offset)); - iphlen = ip->ip_hl << 2; - - mbuf_next = m_getptr(mbuf, iphlen + ehdrlen, &offset); - th = (struct tcphdr *)(mtodo(mbuf_next, offset)); - - if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) { - ena_tx_ctx->l3_csum_enable = 1; - } - if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) { - ena_tx_ctx->tso_enable = 1; - ena_meta->l4_hdr_len = (th->th_off); - } - - switch (etype) { - case ETHERTYPE_IP: - ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; - if ((ip->ip_off & htons(IP_DF)) != 0) - ena_tx_ctx->df = 1; - break; - case ETHERTYPE_IPV6: - ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; - - default: - break; - } - - if (ip->ip_p == IPPROTO_TCP) { - ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; - if ((mbuf->m_pkthdr.csum_flags & - (CSUM_IP_TCP | CSUM_IP6_TCP)) != 0) - ena_tx_ctx->l4_csum_enable = 1; - else - ena_tx_ctx->l4_csum_enable = 0; - } else if (ip->ip_p == IPPROTO_UDP) { - ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; - if ((mbuf->m_pkthdr.csum_flags & - (CSUM_IP_UDP | CSUM_IP6_UDP)) != 0) - ena_tx_ctx->l4_csum_enable = 1; - else - ena_tx_ctx->l4_csum_enable = 0; - } else { - ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; - ena_tx_ctx->l4_csum_enable = 0; - } - - ena_meta->mss = mss; - ena_meta->l3_hdr_len = iphlen; - ena_meta->l3_hdr_offset = ehdrlen; - ena_tx_ctx->meta_valid = 1; -} - static int -ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) -{ - struct ena_adapter *adapter; - struct mbuf *collapsed_mbuf; - int num_frags; - - adapter = tx_ring->adapter; - num_frags = ena_mbuf_count(*mbuf); - - /* One segment must be reserved for configuration descriptor. */ - if (num_frags < adapter->max_tx_sgl_size) - return (0); - counter_u64_add(tx_ring->tx_stats.collapse, 1); - - collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT, - adapter->max_tx_sgl_size - 1); - if (unlikely(collapsed_mbuf == NULL)) { - counter_u64_add(tx_ring->tx_stats.collapse_err, 1); - return (ENOMEM); - } - - /* If mbuf was collapsed succesfully, original mbuf is released. */ - *mbuf = collapsed_mbuf; - - return (0); -} - -static void -ena_dmamap_llq(void *arg, bus_dma_segment_t *segs, int nseg, int error) -{ - struct ena_com_buf *ena_buf = arg; - - if (unlikely(error != 0)) { - ena_buf->paddr = 0; - return; - } - - KASSERT(nseg == 1, ("Invalid num of segments for LLQ dma")); - - ena_buf->paddr = segs->ds_addr; - ena_buf->len = segs->ds_len; -} - -static int -ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info, - struct mbuf *mbuf, void **push_hdr, u16 *header_len) -{ - struct ena_adapter *adapter = tx_ring->adapter; - struct ena_com_buf *ena_buf; - bus_dma_segment_t segs[ENA_BUS_DMA_SEGS]; - uint32_t mbuf_head_len, frag_len; - uint16_t push_len = 0; - uint16_t delta = 0; - int i, rc, nsegs; - - mbuf_head_len = mbuf->m_len; - tx_info->mbuf = mbuf; - ena_buf = tx_info->bufs; - - if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { - /* - * When the device is LLQ mode, the driver will copy - * the header into the device memory space. - * the ena_com layer assumes the header is in a linear - * memory space. - * This assumption might be wrong since part of the header - * can be in the fragmented buffers. - * First check if header fits in the mbuf. If not, copy it to - * separate buffer that will be holding linearized data. - */ - push_len = min_t(uint32_t, mbuf->m_pkthdr.len, - tx_ring->tx_max_header_size); - *header_len = push_len; - /* If header is in linear space, just point into mbuf's data. */ - if (likely(push_len <= mbuf_head_len)) { - *push_hdr = mbuf->m_data; - /* - * Otherwise, copy whole portion of header from multiple mbufs - * to intermediate buffer. - */ - } else { - m_copydata(mbuf, 0, push_len, - tx_ring->push_buf_intermediate_buf); - *push_hdr = tx_ring->push_buf_intermediate_buf; - - counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1); - delta = push_len - mbuf_head_len; - } - - ena_trace(ENA_DBG | ENA_TXPTH, - "mbuf: %p header_buf->vaddr: %p push_len: %d\n", - mbuf, *push_hdr, push_len); - - /* - * If header was in linear memory space, map for the dma rest of the data - * in the first mbuf of the mbuf chain. - */ - if (mbuf_head_len > push_len) { - rc = bus_dmamap_load(adapter->tx_buf_tag, - tx_info->map_head, - mbuf->m_data + push_len, mbuf_head_len - push_len, - ena_dmamap_llq, ena_buf, BUS_DMA_NOWAIT); - if (unlikely((rc != 0) || (ena_buf->paddr == 0))) - goto single_dma_error; - - ena_buf++; - tx_info->num_of_bufs++; - - tx_info->head_mapped = true; - } - mbuf = mbuf->m_next; - } else { - *push_hdr = NULL; - /* - * header_len is just a hint for the device. Because FreeBSD is not - * giving us information about packet header length and it is not - * guaranteed that all packet headers will be in the 1st mbuf, setting - * header_len to 0 is making the device ignore this value and resolve - * header on it's own. - */ - *header_len = 0; - } - - /* - * If header is in non linear space (delta > 0), then skip mbufs - * containing header and map the last one containing both header and the - * packet data. - * The first segment is already counted in. - * If LLQ is not supported, the loop will be skipped. - */ - while (delta > 0) { - frag_len = mbuf->m_len; - - /* - * If whole segment contains header just move to the - * next one and reduce delta. - */ - if (unlikely(delta >= frag_len)) { - delta -= frag_len; - } else { - /* - * Map rest of the packet data that was contained in - * the mbuf. - */ - rc = bus_dmamap_load(adapter->tx_buf_tag, - tx_info->map_head, mbuf->m_data + delta, - frag_len - delta, ena_dmamap_llq, ena_buf, - BUS_DMA_NOWAIT); - if (unlikely((rc != 0) || (ena_buf->paddr == 0))) - goto single_dma_error; - - ena_buf++; - tx_info->num_of_bufs++; - tx_info->head_mapped = true; - - delta = 0; - } - - mbuf = mbuf->m_next; - } - - if (mbuf == NULL) { - return (0); - } - - /* Map rest of the mbufs */ - rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag, tx_info->map_seg, mbuf, - segs, &nsegs, BUS_DMA_NOWAIT); - if (unlikely((rc != 0) || (nsegs == 0))) { - ena_trace(ENA_WARNING, - "dmamap load failed! err: %d nsegs: %d\n", rc, nsegs); - goto dma_error; - } - - for (i = 0; i < nsegs; i++) { - ena_buf->len = segs[i].ds_len; - ena_buf->paddr = segs[i].ds_addr; - ena_buf++; - } - tx_info->num_of_bufs += nsegs; - tx_info->seg_mapped = true; - - return (0); - -dma_error: - if (tx_info->head_mapped == true) - bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map_head); -single_dma_error: - counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1); - tx_info->mbuf = NULL; - return (rc); -} - -static int -ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) -{ - struct ena_adapter *adapter; - struct ena_tx_buffer *tx_info; - struct ena_com_tx_ctx ena_tx_ctx; - struct ena_com_dev *ena_dev; - struct ena_com_io_sq* io_sq; - void *push_hdr; - uint16_t next_to_use; - uint16_t req_id; - uint16_t ena_qid; - uint16_t header_len; - int rc; - int nb_hw_desc; - - ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); - adapter = tx_ring->que->adapter; - ena_dev = adapter->ena_dev; - io_sq = &ena_dev->io_sq_queues[ena_qid]; - - rc = ena_check_and_collapse_mbuf(tx_ring, mbuf); - if (unlikely(rc != 0)) { - ena_trace(ENA_WARNING, - "Failed to collapse mbuf! err: %d\n", rc); *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
