On Fri, Mar 18, 2022 at 11:33 PM David Marchand <david.march...@redhat.com> wrote: > > DPDK based dp-packets points to data buffers that can't be expanded > dynamically. > Their layout is as follows: > - a 128 bytes headroom chosen at DPDK build time (RTE_PKTMBUF_HEADROOM), > - a maximum size chosen at mempool creation, > > In some usecases though (like encapsulating with multiple tunnels), > a 128 bytes headroom is too short. > > Dynamically allocate buffers in DPDK memory and make use of DPDK > external buffers API (previously used for userspace TSO). > > Signed-off-by: David Marchand <david.march...@redhat.com> > --- > lib/dp-packet.c | 17 ++++++++++++++++- > lib/netdev-dpdk.c | 47 +++++++++++++++++++++++++++++++++++------------ > lib/netdev-dpdk.h | 3 +++ > 3 files changed, 54 insertions(+), 13 deletions(-) > > diff --git a/lib/dp-packet.c b/lib/dp-packet.c > index 35c72542a2..07fa67b1a1 100644 > --- a/lib/dp-packet.c > +++ b/lib/dp-packet.c > @@ -250,8 +250,23 @@ dp_packet_resize(struct dp_packet *b, size_t > new_headroom, size_t new_tailroom) > new_allocated = new_headroom + dp_packet_size(b) + new_tailroom; > > switch (b->source) { > - case DPBUF_DPDK: > + case DPBUF_DPDK: { > +#ifdef DPDK_NETDEV > + uint32_t buf_len; > + > + buf_len = new_allocated; > + new_base = netdev_dpdk_extbuf_allocate(&buf_len); > + if (!new_base) { > + out_of_memory(); > + } > + ovs_assert(buf_len <= UINT16_MAX); > + dp_packet_copy__(b, new_base, new_headroom, new_tailroom); > + netdev_dpdk_extbuf_replace(b, new_base, buf_len);
It seems that this is the first example in dp-packet.c to use the APIs in netdev-dpdk.c. Why not move them to dp-packet.c directly? > + break; > +#else > OVS_NOT_REACHED(); > +#endif > + } > > case DPBUF_MALLOC: > if (new_headroom == dp_packet_headroom(b)) { > diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c > index fbc3b42d84..47e16f22c5 100644 > --- a/lib/netdev-dpdk.c > +++ b/lib/netdev-dpdk.c > @@ -2646,41 +2646,64 @@ out: > } > } > > +void * > +netdev_dpdk_extbuf_allocate(uint32_t *data_len) > +{ > + *data_len += sizeof(struct rte_mbuf_ext_shared_info) + sizeof(uintptr_t); > + *data_len = RTE_ALIGN_CEIL(*data_len, sizeof(uintptr_t)); > + return rte_malloc(NULL, *data_len, RTE_CACHE_LINE_SIZE); > +} > + > static void > netdev_dpdk_extbuf_free(void *addr OVS_UNUSED, void *opaque) > { > rte_free(opaque); > } > > +void > +netdev_dpdk_extbuf_replace(struct dp_packet *b, void *buf, uint32_t data_len) > +{ > + struct rte_mbuf *pkt = (struct rte_mbuf *) b; > + struct rte_mbuf_ext_shared_info *shinfo; > + uint16_t buf_len = data_len; > + > + shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len, > + netdev_dpdk_extbuf_free, > + buf); > + ovs_assert(shinfo != NULL); > + > + if (RTE_MBUF_HAS_EXTBUF(pkt)) { > + rte_pktmbuf_detach_extbuf(pkt); > + } > + rte_pktmbuf_attach_extbuf(pkt, buf, rte_malloc_virt2iova(buf), buf_len, > + shinfo); > +} > + > static struct rte_mbuf * > dpdk_pktmbuf_attach_extbuf(struct rte_mbuf *pkt, uint32_t data_len) > { > uint32_t total_len = RTE_PKTMBUF_HEADROOM + data_len; > - struct rte_mbuf_ext_shared_info *shinfo = NULL; > + struct rte_mbuf_ext_shared_info *shinfo; > uint16_t buf_len; > void *buf; > > - total_len += sizeof *shinfo + sizeof(uintptr_t); > - total_len = RTE_ALIGN_CEIL(total_len, sizeof(uintptr_t)); > - > + buf = netdev_dpdk_extbuf_allocate(&total_len); > + if (OVS_UNLIKELY(buf == NULL)) { > + VLOG_ERR("Failed to allocate memory using rte_malloc: %u", > total_len); > + return NULL; > + } > if (OVS_UNLIKELY(total_len > UINT16_MAX)) { > + netdev_dpdk_extbuf_free(NULL, buf); > VLOG_ERR("Can't copy packet: too big %u", total_len); > return NULL; > } > > buf_len = total_len; > - buf = rte_malloc(NULL, buf_len, RTE_CACHE_LINE_SIZE); > - if (OVS_UNLIKELY(buf == NULL)) { > - VLOG_ERR("Failed to allocate memory using rte_malloc: %u", buf_len); > - return NULL; > - } > - > - /* Initialize shinfo. */ > shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len, > netdev_dpdk_extbuf_free, > buf); > if (OVS_UNLIKELY(shinfo == NULL)) { > - rte_free(buf); > + netdev_dpdk_extbuf_free(NULL, buf); > VLOG_ERR("Failed to initialize shared info for mbuf while " > "attempting to attach an external buffer."); > return NULL; > diff --git a/lib/netdev-dpdk.h b/lib/netdev-dpdk.h > index 699be3fb41..95594f07fb 100644 > --- a/lib/netdev-dpdk.h > +++ b/lib/netdev-dpdk.h > @@ -31,6 +31,9 @@ struct netdev; > void netdev_dpdk_register(void); > void free_dpdk_buf(struct dp_packet *); > > +void *netdev_dpdk_extbuf_allocate(uint32_t *); > +void netdev_dpdk_extbuf_replace(struct dp_packet *, void *, uint32_t); > + > bool netdev_dpdk_flow_api_supported(struct netdev *); > > int > -- > 2.23.0 > > _______________________________________________ > dev mailing list > d...@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-dev -- Thanks, Harold. _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev