[lng-odp] [API-NEXT PATCH 02/11] linux-generic: pktio: initial DPDK pktio implementation
Initial implementation of the DPDK pktio type. Initialization code copied from the odp-dpdk branch. Reviewed-by: Petri Savolainen Signed-off-by: Matias Elo --- platform/linux-generic/include/odp_internal.h| 5 + platform/linux-generic/include/odp_packet_dpdk.h | 23 + platform/linux-generic/odp_init.c| 12 + platform/linux-generic/pktio/dpdk.c | 534 ++- 4 files changed, 550 insertions(+), 24 deletions(-) diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index cdbed7d..c01cbef 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -107,6 +107,11 @@ int odp_tm_init_global(void); void _odp_flush_caches(void); +#ifdef ODP_DPDK +int odp_dpdk_pktio_init_global(void); +int odp_dpdk_pktio_init_local(void); +#endif + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h index 71fdbdc..5676f4c 100644 --- a/platform/linux-generic/include/odp_packet_dpdk.h +++ b/platform/linux-generic/include/odp_packet_dpdk.h @@ -10,11 +10,34 @@ #include #include +#include + +#ifdef ODP_DPDK +#include +#include + +#define DPDK_MEMORY_MB 512 +#define DPDK_NB_MBUF 8192 +#define DPDK_MBUF_BUF_SIZE RTE_MBUF_DEFAULT_BUF_SIZE +#define DPDK_MEMPOOL_CACHE_SIZE 32 +#define DPDK_NM_RX_DESC 128 +#define DPDK_NM_TX_DESC 512 + +_ODP_STATIC_ASSERT(DPDK_NB_MBUF % DPDK_MEMPOOL_CACHE_SIZE == 0 && + DPDK_MEMPOOL_CACHE_SIZE <= RTE_MEMPOOL_CACHE_MAX_SIZE && + DPDK_MEMPOOL_CACHE_SIZE <= DPDK_MBUF_BUF_SIZE / 1.5 + , "DPDK mempool cache size failure"); +#endif /** Packet IO using DPDK interface */ typedef struct { odp_pool_t pool; /**< pool to alloc packets from */ + struct rte_mempool *pkt_pool; /**< DPDK packet pool */ odp_pktio_capability_t capa; /**< interface capabilities */ + uint32_t data_room; /**< maximum packet length */ + /** DPDK packet pool name (pktpool_) */ + char pool_name[IF_NAMESIZE + 8]; + uint8_t port_id; /**< DPDK port identifier */ } pkt_dpdk_t; #endif diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 6ad3320..584e659 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -37,6 +37,12 @@ int odp_init_global(const odp_init_t *params, } stage = SYSINFO_INIT; +#ifdef ODP_DPDK + if (odp_dpdk_pktio_init_global() < 0) { + ODP_ERR("ODP dpdk pktio init failed.\n"); + return -1; + } +#endif if (odp_shm_init_global()) { ODP_ERR("ODP shm init failed.\n"); goto init_failed; @@ -215,6 +221,12 @@ int odp_init_local(odp_thread_type_t thr_type) } stage = THREAD_INIT; +#ifdef ODP_DPDK + if (odp_dpdk_pktio_init_local()) { + ODP_ERR("ODP dpdk pktio local init failed.\n"); + return -1; + } +#endif if (odp_pktio_init_local()) { ODP_ERR("ODP packet io local init failed.\n"); goto init_fail; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 31fe9ae..be622dd 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -8,64 +8,550 @@ #include +#include +#include + #include #include #include +#include #include #include +#include +#include + +/* Has dpdk_pktio_init() been called */ +static odp_bool_t dpdk_initialized; + +#define PMD_EXT(drv) \ +extern void devinitfn_##drv(void) + +PMD_EXT(cryptodev_aesni_mb_pmd_drv); +PMD_EXT(pmd_qat_drv); +PMD_EXT(pmd_af_packet_drv); +PMD_EXT(rte_bnx2x_driver); +PMD_EXT(rte_bnx2xvf_driver); +PMD_EXT(bond_drv); +PMD_EXT(rte_cxgbe_driver); +PMD_EXT(em_pmd_drv); +PMD_EXT(pmd_igb_drv); +PMD_EXT(pmd_igbvf_drv); +PMD_EXT(rte_enic_driver); +PMD_EXT(rte_fm10k_driver); +PMD_EXT(rte_i40e_driver); +PMD_EXT(rte_i40evf_driver); +PMD_EXT(rte_ixgbe_driver); +PMD_EXT(rte_ixgbevf_driver); +PMD_EXT(rte_mlx4_driver); +PMD_EXT(rte_mlx5_driver); +PMD_EXT(pmd_mpipe_xgbe_drv); +PMD_EXT(pmd_mpipe_gbe_drv); +PMD_EXT(rte_nfp_net_driver); +PMD_EXT(pmd_null_drv); +PMD_EXT(pmd_pcap_drv); +PMD_EXT(pmd_ring_drv); +PMD_EXT(pmd_szedata2_drv); +PMD_EXT(rte_virtio_driver); +PMD_EXT(rte_vmxnet3_driver); +PMD_EXT(pmd_xenvirt_drv); + +/* + * This function is not called from anywhere, it's only purpose is to make sure + * that if ODP and DPDK are statically linked to an application, the GCC + * constuctors of the PMDs are linked as well. Otherwise the linker would omit + * them. It's not an issue with dynamic linking. */ +void refer_constructors(void); +void refer_constructors(void) +{ +#ifdef RTE_LIBRTE_PMD_AESNI_MB + devinitfn_cryptodev_aesni_mb_pmd_dr
Re: [lng-odp] [API-NEXT PATCH 02/11] linux-generic: pktio: initial DPDK pktio implementation
On 28/01/16 07:03, Matias Elo wrote: +static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, + odp_packet_t pkt_table[], + struct rte_mbuf *mbuf_table[], + uint16_t num) { - return 0; + odp_packet_t pkt; + int i; + odp_packet_hdr_t *pkt_hdr; + uint16_t pkt_len; + void *buf; + + for (i = 0; i < num; i++) { + buf = rte_pktmbuf_mtod(mbuf_table[i], char *); + odp_prefetch(buf); + + pkt_len = rte_pktmbuf_pkt_len(mbuf_table[i]); + + pkt = packet_alloc(pktio_entry->s.pkt_dpdk.pool, pkt_len, 1); + if (pkt == ODP_PACKET_INVALID) { + ODP_ERR("packet_alloc failed\n"); + goto fail; + } + + pkt_hdr = odp_packet_hdr(pkt); + + /* For now copy the data in the mbuf, + worry about zero-copy later */ + if (odp_packet_copydata_in(pkt, 0, pkt_len, buf) != 0) { I think this doesn't work if someone sets the MTU higher than your mbuf size. Although ODP doesn't support that now, you should at least assert that nb_segs == 1, and abort if it's not true. Also, the RSS hash is worth to copy so you don't have to calculate it in sw. + ODP_ERR("odp_packet_copydata_in failed\n"); + odp_packet_free(pkt); + goto fail; + } + + packet_parse_l2(pkt_hdr); + + pkt_hdr->input = pktio_entry->s.handle; + + pkt_table[i] = pkt; + + rte_pktmbuf_free(mbuf_table[i]); + } + + return i; + +fail: + ODP_ERR("Creating ODP packet failed\n"); + rte_pktmbuf_free(mbuf_table[i]); + return (i > 0 ? i++ : -1); } -static int dpdk_recv(pktio_entry_t *pktio_entry ODP_UNUSED, -odp_packet_t pkt_table[] ODP_UNUSED, -unsigned num ODP_UNUSED) +static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, + struct rte_mbuf *mbuf_table[], + odp_packet_t pkt_table[], uint16_t num) { - return 0; + pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; + int i; + char *data; + uint16_t pkt_len; + + for (i = 0; i < num; i++) { + mbuf_table[i] = rte_pktmbuf_alloc(pkt_dpdk->pkt_pool); + if (mbuf_table[i] == NULL) { + ODP_ERR("Failed to alloc mbuf\n"); + return -1; + } + + rte_pktmbuf_reset(mbuf_table[i]); + pkt_len = odp_packet_len(pkt_table[i]); + + data = rte_pktmbuf_append(mbuf_table[i], pkt_len); + + if (data != NULL) + odp_packet_copydata_out(pkt_table[i], 0, pkt_len, data); + } + return i; } -static int dpdk_send_queue(pktio_entry_t *pktio_entry ODP_UNUSED, - int index ODP_UNUSED, - odp_packet_t pkt_table[] ODP_UNUSED, - int num ODP_UNUSED) +static int dpdk_recv_queue(pktio_entry_t *pktio_entry, + int index, + odp_packet_t pkt_table[], + int num) { - return 0; + uint16_t nb_rx; + + struct rte_mbuf *rx_mbufs[num]; + + nb_rx = rte_eth_rx_burst(pktio_entry->s.pkt_dpdk.port_id, index, +rx_mbufs, num); + + if (nb_rx > 0) + nb_rx = mbuf_to_pkt(pktio_entry, pkt_table, rx_mbufs, nb_rx); + + return nb_rx; } -static int dpdk_send(pktio_entry_t *pktio_entry ODP_UNUSED, -odp_packet_t pkt_table[] ODP_UNUSED, -unsigned num ODP_UNUSED) +static int dpdk_recv(pktio_entry_t *pktio_entry, +odp_packet_t pkt_table[], +unsigned num) { - return 0; + return dpdk_recv_queue(pktio_entry, 0, pkt_table, num); +} + +static int dpdk_send_queue(pktio_entry_t *pktio_entry, + int index, + odp_packet_t pkt_table[], + int num) +{ + struct rte_mbuf *tx_mbufs[num]; + pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; + int tx_pkts; + int i; + int mbufs; + + mbufs = pkt_to_mbuf(pktio_entry, tx_mbufs, pkt_table, num); + + tx_pkts = rte_eth_tx_burst(pkt_dpdk->port_id, index, + tx_mbufs, mbufs); + + if (odp_unlikely(tx_pkts < num)) { + for (i = tx_pkts; i < mbufs; i++) + rte_pktmbuf_free(tx_mbufs[i]); + } + + odp_packet_free_multi(pkt_table, tx_pkts); + + if (odp_unlikely(tx_pkts == 0 && __odp_errno != 0)) + return -1; + + return tx_pkts; +} + +static int dpdk_send(pktio_entry_
Re: [lng-odp] [API-NEXT PATCH 02/11] linux-generic: pktio: initial DPDK pktio implementation
> -Original Message- > From: EXT Zoltan Kiss [mailto:zoltan.k...@linaro.org] > Sent: Thursday, January 28, 2016 1:34 PM > To: Elo, Matias (Nokia - FI/Espoo) ; lng- > o...@lists.linaro.org > Subject: Re: [lng-odp] [API-NEXT PATCH 02/11] linux-generic: pktio: initial > DPDK > pktio implementation > > > > On 28/01/16 07:03, Matias Elo wrote: > > > +static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, > > + odp_packet_t pkt_table[], > > + struct rte_mbuf *mbuf_table[], > > + uint16_t num) > > { > > - return 0; > > + odp_packet_t pkt; > > + int i; > > + odp_packet_hdr_t *pkt_hdr; > > + uint16_t pkt_len; > > + void *buf; > > + > > + for (i = 0; i < num; i++) { > > + buf = rte_pktmbuf_mtod(mbuf_table[i], char *); > > + odp_prefetch(buf); > > + > > + pkt_len = rte_pktmbuf_pkt_len(mbuf_table[i]); > > + > > + pkt = packet_alloc(pktio_entry->s.pkt_dpdk.pool, pkt_len, 1); > > + if (pkt == ODP_PACKET_INVALID) { > > + ODP_ERR("packet_alloc failed\n"); > > + goto fail; > > + } > > + > > + pkt_hdr = odp_packet_hdr(pkt); > > + > > + /* For now copy the data in the mbuf, > > + worry about zero-copy later */ > > + if (odp_packet_copydata_in(pkt, 0, pkt_len, buf) != 0) { > > > I think this doesn't work if someone sets the MTU higher than your mbuf > size. Although ODP doesn't support that now, you should at least assert > that nb_segs == 1, and abort if it's not true. > I'll add the nb_segs check. A proper segmentation support will be added later in a separate patch. > Also, the RSS hash is worth to copy so you don't have to calculate it in sw. OK, I'll add this. > > > > + ODP_ERR("odp_packet_copydata_in failed\n"); > > + odp_packet_free(pkt); > > + goto fail; > > + } > > + > > + packet_parse_l2(pkt_hdr); > > + > > + pkt_hdr->input = pktio_entry->s.handle; > > + > > + pkt_table[i] = pkt; > > + > > + rte_pktmbuf_free(mbuf_table[i]); > > + } > > + > > + return i; > > + > > +fail: > > + ODP_ERR("Creating ODP packet failed\n"); > > + rte_pktmbuf_free(mbuf_table[i]); > > + return (i > 0 ? i++ : -1); > > } > > > > -static int dpdk_recv(pktio_entry_t *pktio_entry ODP_UNUSED, > > -odp_packet_t pkt_table[] ODP_UNUSED, > > -unsigned num ODP_UNUSED) > > +static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, > > + struct rte_mbuf *mbuf_table[], > > + odp_packet_t pkt_table[], uint16_t num) > > { > > - return 0; > > + pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; > > + int i; > > + char *data; > > + uint16_t pkt_len; > > + > > + for (i = 0; i < num; i++) { > > + mbuf_table[i] = rte_pktmbuf_alloc(pkt_dpdk->pkt_pool); > > + if (mbuf_table[i] == NULL) { > > + ODP_ERR("Failed to alloc mbuf\n"); > > + return -1; > > + } > > + > > + rte_pktmbuf_reset(mbuf_table[i]); > > + pkt_len = odp_packet_len(pkt_table[i]); > > + > > + data = rte_pktmbuf_append(mbuf_table[i], pkt_len); > > + > > + if (data != NULL) > > + odp_packet_copydata_out(pkt_table[i], 0, pkt_len, > data); > > + } > > + return i; > > } > > > > -static int dpdk_send_queue(pktio_entry_t *pktio_entry ODP_UNUSED, > > - int index ODP_UNUSED, > > - odp_packet_t pkt_table[] ODP_UNUSED, > > - int num ODP_UNUSED) > > +static int dpdk_recv_queue(pktio_entry_t *pktio_entry, > > + int index, > > + odp_packet_t pkt_table[], > > + int num) > > { > > - return 0; > > + uint16_t nb_rx; > > + > > + struct rte_mbuf *rx_mbufs[num]; > > + > > + nb_rx = rte_eth_rx_burst(pktio_entry->s.pkt_dpdk.port_id, index, > > +rx_mbufs, num); > > + > > + if (nb_rx > 0) > > + nb_rx =