From: Petri Savolainen <petri.savolai...@linaro.org> Use always inlined versions of packet to/from memory copy functions.
Signed-off-by: Petri Savolainen <petri.savolai...@linaro.org> --- /** Email created from pull request 437 (psavol:master-packet-optim) ** https://github.com/Linaro/odp/pull/437 ** Patch: https://github.com/Linaro/odp/pull/437.patch ** Base sha: b95ccd3db6eeb7358a877541747e06354429acdd ** Merge commit sha: ab7875c186fe80cfe103c5be5634b0eda312aed4 **/ platform/linux-generic/odp_traffic_mngr.c | 24 ++++++++++++------------ platform/linux-generic/pktio/dpdk.c | 4 ++-- platform/linux-generic/pktio/netmap.c | 4 ++-- platform/linux-generic/pktio/pcap.c | 5 +++-- platform/linux-generic/pktio/socket_mmap.c | 7 ++++--- platform/linux-generic/pktio/tap.c | 5 +++-- 6 files changed, 26 insertions(+), 23 deletions(-) diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 557e77ba3..03094cc37 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -1941,8 +1941,8 @@ static void egress_vlan_marking(tm_vlan_marking_t *vlan_marking, * correctness rather then performance. */ split_hdr = hdr_len < (_ODP_ETHHDR_LEN + _ODP_VLANHDR_LEN); if (split_hdr) { - odp_packet_copy_to_mem(odp_pkt, _ODP_ETHHDR_LEN, - _ODP_VLANHDR_LEN, &vlan_hdr); + _odp_packet_copy_to_mem(odp_pkt, _ODP_ETHHDR_LEN, + _ODP_VLANHDR_LEN, &vlan_hdr); vlan_hdr_ptr = &vlan_hdr; } @@ -1956,8 +1956,8 @@ static void egress_vlan_marking(tm_vlan_marking_t *vlan_marking, vlan_hdr_ptr->tci = _odp_cpu_to_be_16(new_tci); if (split_hdr) - odp_packet_copy_from_mem(odp_pkt, _ODP_ETHHDR_LEN, - _ODP_VLANHDR_LEN, &vlan_hdr); + _odp_packet_copy_from_mem(odp_pkt, _ODP_ETHHDR_LEN, + _ODP_VLANHDR_LEN, &vlan_hdr); } static void egress_ipv4_tos_marking(tm_tos_marking_t *tos_marking, @@ -1980,8 +1980,8 @@ static void egress_ipv4_tos_marking(tm_tos_marking_t *tos_marking, * correctness rather then performance. */ split_hdr = hdr_len < 12; if (split_hdr) { - odp_packet_copy_to_mem(odp_pkt, l3_offset, - _ODP_IPV4HDR_LEN, &ipv4_hdr); + _odp_packet_copy_to_mem(odp_pkt, l3_offset, + _ODP_IPV4HDR_LEN, &ipv4_hdr); ipv4_hdr_ptr = &ipv4_hdr; } @@ -2022,8 +2022,8 @@ static void egress_ipv4_tos_marking(tm_tos_marking_t *tos_marking, ipv4_hdr_ptr->tos = new_tos; ipv4_hdr_ptr->chksum = _odp_cpu_to_be_16((~ones_compl_sum) & 0xFFFF); if (split_hdr) - odp_packet_copy_from_mem(odp_pkt, l3_offset, - _ODP_IPV4HDR_LEN, &ipv4_hdr); + _odp_packet_copy_from_mem(odp_pkt, l3_offset, + _ODP_IPV4HDR_LEN, &ipv4_hdr); } static void egress_ipv6_tc_marking(tm_tos_marking_t *tos_marking, @@ -2046,8 +2046,8 @@ static void egress_ipv6_tc_marking(tm_tos_marking_t *tos_marking, * correctness rather then performance. */ split_hdr = hdr_len < 4; if (split_hdr) { - odp_packet_copy_to_mem(odp_pkt, l3_offset, - _ODP_IPV6HDR_LEN, &ipv6_hdr); + _odp_packet_copy_to_mem(odp_pkt, l3_offset, + _ODP_IPV6HDR_LEN, &ipv6_hdr); ipv6_hdr_ptr = &ipv6_hdr; } @@ -2075,8 +2075,8 @@ static void egress_ipv6_tc_marking(tm_tos_marking_t *tos_marking, ipv6_hdr_ptr->ver_tc_flow = _odp_cpu_to_be_32(new_ver_tc_flow); if (split_hdr) - odp_packet_copy_from_mem(odp_pkt, l3_offset, - _ODP_IPV6HDR_LEN, &ipv6_hdr); + _odp_packet_copy_from_mem(odp_pkt, l3_offset, + _ODP_IPV6HDR_LEN, &ipv6_hdr); } static void tm_egress_marking(tm_system_t *tm_system, odp_packet_t odp_pkt) diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 4ae72e10a..86d7a4513 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -471,7 +471,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, pkt_hdr = odp_packet_hdr(pkt); pull_tail(pkt_hdr, alloc_len - pkt_len); - if (odp_packet_copy_from_mem(pkt, 0, pkt_len, data) != 0) + if (_odp_packet_copy_from_mem(pkt, 0, pkt_len, data) != 0) goto fail; pkt_hdr->input = pktio_entry->s.handle; @@ -649,7 +649,7 @@ static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, /* Packet always fits in mbuf */ data = rte_pktmbuf_append(mbuf_table[i], pkt_len); - odp_packet_copy_to_mem(pkt_table[i], 0, pkt_len, data); + _odp_packet_copy_to_mem(pkt_table[i], 0, pkt_len, data); if (odp_unlikely(pktio_entry->s.chksum_insert_ena)) { odp_pktout_config_opt_t *pktout_capa = diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index a7c479001..c03564f4e 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -652,7 +652,7 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, /* For now copy the data in the mbuf, worry about zero-copy later */ - if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) + if (_odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) goto fail; pkt_hdr->input = pktio_entry->s.handle; @@ -938,7 +938,7 @@ static int netmap_send(pktio_entry_t *pktio_entry, int index, buf = NETMAP_BUF(ring, ring->slot[slot_id].buf_idx); - if (odp_packet_copy_to_mem(pkt, 0, pkt_len, buf)) { + if (_odp_packet_copy_to_mem(pkt, 0, pkt_len, buf)) { i = NM_INJECT_RETRIES; break; } diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index 7fc925149..76cbf9eef 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -39,6 +39,7 @@ #include <odp_posix_extensions.h> #include <odp_api.h> +#include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> @@ -249,7 +250,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_hdr = odp_packet_hdr(pkt); - if (odp_packet_copy_from_mem(pkt, 0, hdr->caplen, data) != 0) { + if (_odp_packet_copy_from_mem(pkt, 0, hdr->caplen, data) != 0) { ODP_ERR("failed to copy packet data\n"); break; } @@ -283,7 +284,7 @@ static int _pcapif_dump_pkt(pkt_pcap_t *pcap, odp_packet_t pkt) hdr.len = hdr.caplen; (void)gettimeofday(&hdr.ts, NULL); - if (odp_packet_copy_to_mem(pkt, 0, hdr.len, pcap->buf) != 0) + if (_odp_packet_copy_to_mem(pkt, 0, hdr.len, pcap->buf) != 0) return -1; pcap_dump(pcap->tx_dump, &hdr, pcap->buf); diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index b53173d29..2ace1a597 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -26,6 +26,7 @@ #include <time.h> #include <odp_api.h> +#include <odp/api/plat/packet_inlines.h> #include <odp_packet_socket.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> @@ -243,8 +244,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, continue; } hdr = odp_packet_hdr(pkt_table[nb_rx]); - ret = odp_packet_copy_from_mem(pkt_table[nb_rx], 0, - pkt_len, pkt_buf); + ret = _odp_packet_copy_from_mem(pkt_table[nb_rx], 0, + pkt_len, pkt_buf); if (ret != 0) { odp_packet_free(pkt_table[nb_rx]); mmap_rx_user_ready(ppd.raw); /* drop */ @@ -345,7 +346,7 @@ static inline unsigned pkt_mmap_v2_tx(int sock, struct ring *ring, buf = (uint8_t *)ppd.raw + TPACKET2_HDRLEN - sizeof(struct sockaddr_ll); - odp_packet_copy_to_mem(pkt_table[i], 0, pkt_len, buf); + _odp_packet_copy_to_mem(pkt_table[i], 0, pkt_len, buf); mmap_tx_user_ready(ppd.raw); diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index ebd4d94be..ccfa19b25 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -42,6 +42,7 @@ #include <linux/if_tun.h> #include <odp_api.h> +#include <odp/api/plat/packet_inlines.h> #include <odp_packet_socket.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> @@ -273,7 +274,7 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, if (num != 1) return ODP_PACKET_INVALID; - if (odp_packet_copy_from_mem(pkt, 0, len, data) < 0) { + if (_odp_packet_copy_from_mem(pkt, 0, len, data) < 0) { ODP_ERR("failed to copy packet data\n"); odp_packet_free(pkt); return ODP_PACKET_INVALID; @@ -352,7 +353,7 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, break; } - if (odp_packet_copy_to_mem(pkts[i], 0, pkt_len, buf) < 0) { + if (_odp_packet_copy_to_mem(pkts[i], 0, pkt_len, buf) < 0) { ODP_ERR("failed to copy packet data\n"); break; }