Signed-off-by: Bill Fischofer <bill.fischo...@linaro.org> --- v2 reflects review comments by Stuart
- Correct miscelaneous errors in packet parser example/generator/odp_generator.c | 82 +- example/ipsec/odp_ipsec.c | 37 +- example/ipsec/odp_ipsec_stream.c | 25 +- example/l2fwd/odp_l2fwd.c | 3 +- example/packet/odp_pktio.c | 9 +- helper/include/odph_ip.h | 35 +- helper/include/odph_packet.h | 97 --- helper/include/odph_udp.h | 5 +- platform/linux-generic/Makefile.am | 1 - platform/linux-generic/include/api/odp_packet.h | 727 ++++++++++++---- .../linux-generic/include/api/odp_platform_types.h | 21 +- .../linux-generic/include/odp_buffer_inlines.h | 54 ++ .../linux-generic/include/odp_buffer_internal.h | 1 + .../linux-generic/include/odp_packet_internal.h | 81 +- platform/linux-generic/odp_crypto.c | 11 +- platform/linux-generic/odp_packet.c | 912 ++++++++++++++++----- platform/linux-generic/odp_packet_socket.c | 93 +-- 17 files changed, 1558 insertions(+), 636 deletions(-) delete mode 100644 helper/include/odph_packet.h diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index 476cbef..b6dbd9d 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -21,7 +21,6 @@ #include <odp.h> #include <odph_linux.h> -#include <odph_packet.h> #include <odph_eth.h> #include <odph_ip.h> #include <odph_udp.h> @@ -171,32 +170,31 @@ static int scan_mac(char *in, odph_ethaddr_t *des) * * @param obuf packet buffer */ -static void pack_udp_pkt(odp_buffer_t obuf) +static odp_packet_t pack_udp_pkt(odp_buffer_pool_t pool) { - char *buf; - int max; odp_packet_t pkt; + char *buf; odph_ethhdr_t *eth; odph_ipv4hdr_t *ip; odph_udphdr_t *udp; unsigned short seq; - buf = odp_buffer_addr(obuf); - if (buf == NULL) - return; - max = odp_buffer_size(obuf); - if (max <= 0) - return; + pkt = odp_packet_alloc(pool, args->appl.payload + ODPH_UDPHDR_LEN + + ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); + + if (pkt == ODP_PACKET_INVALID) + return pkt; + + buf = odp_packet_data(pkt); - pkt = odp_packet_from_buffer(obuf); /* ether */ - odp_packet_set_l2_offset(pkt, 0); + odp_packet_l2_offset_set(pkt, 0); eth = (odph_ethhdr_t *)buf; memcpy((char *)eth->src.addr, args->appl.srcmac.addr, ODPH_ETHADDR_LEN); memcpy((char *)eth->dst.addr, args->appl.dstmac.addr, ODPH_ETHADDR_LEN); eth->type = odp_cpu_to_be_16(ODPH_ETHTYPE_IPV4); /* ip */ - odp_packet_set_l3_offset(pkt, ODPH_ETHHDR_LEN); + odp_packet_l3_offset_set(pkt, ODPH_ETHHDR_LEN); ip = (odph_ipv4hdr_t *)(buf + ODPH_ETHHDR_LEN); ip->dst_addr = odp_cpu_to_be_32(args->appl.dstip); ip->src_addr = odp_cpu_to_be_32(args->appl.srcip); @@ -209,15 +207,15 @@ static void pack_udp_pkt(odp_buffer_t obuf) ip->chksum = 0; odph_ipv4_csum_update(pkt); /* udp */ - odp_packet_set_l4_offset(pkt, ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN); + odp_packet_l4_offset_set(pkt, ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN); udp = (odph_udphdr_t *)(buf + ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN); udp->src_port = 0; udp->dst_port = 0; udp->length = odp_cpu_to_be_16(args->appl.payload + ODPH_UDPHDR_LEN); udp->chksum = 0; udp->chksum = odp_cpu_to_be_16(odph_ipv4_udp_chksum(pkt)); - odp_packet_set_len(pkt, args->appl.payload + ODPH_UDPHDR_LEN + - ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); + + return pkt; } /** @@ -225,11 +223,10 @@ static void pack_udp_pkt(odp_buffer_t obuf) * * @param obuf packet buffer */ -static void pack_icmp_pkt(odp_buffer_t obuf) +static odp_packet_t pack_icmp_pkt(odp_buffer_pool_t pool) { - char *buf; - int max; odp_packet_t pkt; + char *buf; odph_ethhdr_t *eth; odph_ipv4hdr_t *ip; odph_icmphdr_t *icmp; @@ -237,23 +234,23 @@ static void pack_icmp_pkt(odp_buffer_t obuf) uint8_t *tval_d; unsigned short seq; - buf = odp_buffer_addr(obuf); - if (buf == NULL) - return; - max = odp_buffer_size(obuf); - if (max <= 0) - return; - args->appl.payload = 56; - pkt = odp_packet_from_buffer(obuf); + pkt = odp_packet_alloc(pool, args->appl.payload + ODPH_ICMPHDR_LEN + + ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); + + if (pkt == ODP_PACKET_INVALID) + return pkt; + + buf = odp_packet_data(pkt); + /* ether */ - odp_packet_set_l2_offset(pkt, 0); + odp_packet_l2_offset_set(pkt, 0); eth = (odph_ethhdr_t *)buf; memcpy((char *)eth->src.addr, args->appl.srcmac.addr, ODPH_ETHADDR_LEN); memcpy((char *)eth->dst.addr, args->appl.dstmac.addr, ODPH_ETHADDR_LEN); eth->type = odp_cpu_to_be_16(ODPH_ETHTYPE_IPV4); /* ip */ - odp_packet_set_l3_offset(pkt, ODPH_ETHHDR_LEN); + odp_packet_l3_offset_set(pkt, ODPH_ETHHDR_LEN); ip = (odph_ipv4hdr_t *)(buf + ODPH_ETHHDR_LEN); ip->dst_addr = odp_cpu_to_be_32(args->appl.dstip); ip->src_addr = odp_cpu_to_be_32(args->appl.srcip); @@ -281,8 +278,7 @@ static void pack_icmp_pkt(odp_buffer_t obuf) icmp->chksum = odp_chksum(icmp, args->appl.payload + ODPH_ICMPHDR_LEN); - odp_packet_set_len(pkt, args->appl.payload + ODPH_ICMPHDR_LEN + - ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); + return pkt; } /** @@ -298,7 +294,7 @@ static void *gen_send_thread(void *arg) thread_args_t *thr_args; odp_queue_t outq_def; - odp_buffer_t buf; + odp_packet_t pkt; thr = odp_thread_id(); thr_args = arg; @@ -319,18 +315,20 @@ static void *gen_send_thread(void *arg) printf(" [%02i] created mode: SEND\n", thr); for (;;) { int err; - buf = odp_buffer_alloc(thr_args->pool); - if (!odp_buffer_is_valid(buf)) { - EXAMPLE_ERR(" [%2i] alloc_single failed\n", thr); - return NULL; - } if (args->appl.mode == APPL_MODE_UDP) - pack_udp_pkt(buf); + pkt = pack_udp_pkt(thr_args->pool); else if (args->appl.mode == APPL_MODE_PING) - pack_icmp_pkt(buf); + pkt = pack_icmp_pkt(thr_args->pool); + else + pkt = ODP_PACKET_INVALID; + + if (!odp_packet_is_valid(pkt)) { + EXAMPLE_ERR(" [%2i] alloc_single failed\n", thr); + return NULL; + } - err = odp_queue_enq(outq_def, buf); + err = odp_queue_enq(outq_def, odp_packet_to_buffer(pkt)); if (err != 0) { EXAMPLE_ERR(" [%02i] send pkt err!\n", thr); return NULL; @@ -503,13 +501,13 @@ static void *gen_recv_thread(void *arg) pkt = odp_packet_from_buffer(buf); /* Drop packets with errors */ if (odp_unlikely(odp_packet_error(pkt))) { - odph_packet_free(pkt); + odp_packet_free(pkt); continue; } print_pkts(thr, &pkt, 1); - odph_packet_free(pkt); + odp_packet_free(pkt); } return arg; diff --git a/example/ipsec/odp_ipsec.c b/example/ipsec/odp_ipsec.c index f96338c..ddfcbe6 100644 --- a/example/ipsec/odp_ipsec.c +++ b/example/ipsec/odp_ipsec.c @@ -20,7 +20,6 @@ #include <odp.h> #include <odph_linux.h> -#include <odph_packet.h> #include <odph_eth.h> #include <odph_ip.h> #include <odph_icmp.h> @@ -168,7 +167,7 @@ static odp_buffer_pool_t ctx_pool = ODP_BUFFER_POOL_INVALID; static pkt_ctx_t *get_pkt_ctx_from_pkt(odp_packet_t pkt) { - return (pkt_ctx_t *)odp_packet_get_ctx(pkt); + return (pkt_ctx_t *)odp_packet_user_ptr(pkt); } /** @@ -192,7 +191,7 @@ pkt_ctx_t *alloc_pkt_ctx(odp_packet_t pkt) ctx = odp_buffer_addr(ctx_buf); memset(ctx, 0, sizeof(*ctx)); ctx->buffer = ctx_buf; - odp_packet_set_ctx(pkt, ctx); + odp_packet_user_ptr_set(pkt, ctx); return ctx; } @@ -640,13 +639,15 @@ pkt_disposition_e do_input_verify(odp_packet_t pkt, static pkt_disposition_e do_route_fwd_db(odp_packet_t pkt, pkt_ctx_t *ctx) { - odph_ipv4hdr_t *ip = (odph_ipv4hdr_t *)odp_packet_l3(pkt); + uint32_t seglen; + odph_ipv4hdr_t *ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &seglen); fwd_db_entry_t *entry; entry = find_fwd_db_entry(odp_be_to_cpu_32(ip->dst_addr)); if (entry) { - odph_ethhdr_t *eth = (odph_ethhdr_t *)odp_packet_l2(pkt); + odph_ethhdr_t *eth = + (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, &seglen); memcpy(ð->dst, entry->dst_mac, ODPH_ETHADDR_LEN); memcpy(ð->src, entry->src_mac, ODPH_ETHADDR_LEN); @@ -676,8 +677,9 @@ pkt_disposition_e do_ipsec_in_classify(odp_packet_t pkt, pkt_ctx_t *ctx, bool *skip) { - uint8_t *buf = odp_packet_addr(pkt); - odph_ipv4hdr_t *ip = (odph_ipv4hdr_t *)odp_packet_l3(pkt); + uint32_t seglen; + uint8_t *buf = odp_packet_data(pkt); + odph_ipv4hdr_t *ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &seglen); int hdr_len; odph_ahhdr_t *ah = NULL; odph_esphdr_t *esp = NULL; @@ -764,6 +766,7 @@ pkt_disposition_e do_ipsec_in_finish(odp_packet_t pkt, odph_ipv4hdr_t *ip; int hdr_len = ctx->ipsec.hdr_len; int trl_len = 0; + uint32_t seglen; /* Check crypto result */ event = odp_packet_to_buffer(pkt); @@ -772,13 +775,13 @@ pkt_disposition_e do_ipsec_in_finish(odp_packet_t pkt, return PKT_DROP; if (!is_crypto_compl_status_ok(&auth_rc)) return PKT_DROP; - ip = (odph_ipv4hdr_t *)odp_packet_l3(pkt); + ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &seglen); /* * Finish auth */ if (ctx->ipsec.ah_offset) { - uint8_t *buf = odp_packet_addr(pkt); + uint8_t *buf = odp_packet_data(pkt); odph_ahhdr_t *ah; ah = (odph_ahhdr_t *)(ctx->ipsec.ah_offset + buf); @@ -807,10 +810,10 @@ pkt_disposition_e do_ipsec_in_finish(odp_packet_t pkt, odph_ipv4_csum_update(pkt); /* Correct the packet length and move payload into position */ - odp_packet_set_len(pkt, odp_packet_get_len(pkt) - (hdr_len + trl_len)); memmove(ipv4_data_p(ip), ipv4_data_p(ip) + hdr_len, odp_be_to_cpu_16(ip->tot_len)); + odp_packet_pull_tail(pkt, hdr_len + trl_len); /* Fall through to next state */ return PKT_CONTINUE; @@ -836,8 +839,9 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt, pkt_ctx_t *ctx, bool *skip) { - uint8_t *buf = odp_packet_addr(pkt); - odph_ipv4hdr_t *ip = (odph_ipv4hdr_t *)odp_packet_l3(pkt); + uint32_t seglen; + uint8_t *buf = odp_packet_data(pkt); + odph_ipv4hdr_t *ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &seglen); uint16_t ip_data_len = ipv4_data_len(ip); uint8_t *ip_data = ipv4_data_p(ip); ipsec_cache_entry_t *entry; @@ -924,7 +928,7 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt, /* Set IPv4 length before authentication */ ipv4_adjust_len(ip, hdr_len + trl_len); - odp_packet_set_len(pkt, odp_packet_get_len(pkt) + (hdr_len + trl_len)); + odp_packet_push_tail(pkt, hdr_len + trl_len); /* Save remaining context */ ctx->ipsec.hdr_len = hdr_len; @@ -954,7 +958,7 @@ static pkt_disposition_e do_ipsec_out_seq(odp_packet_t pkt, pkt_ctx_t *ctx) { - uint8_t *buf = odp_packet_addr(pkt); + uint8_t *buf = odp_packet_data(pkt); bool posted = 0; /* We were dispatched from atomic queue, assign sequence numbers */ @@ -996,6 +1000,7 @@ pkt_disposition_e do_ipsec_out_finish(odp_packet_t pkt, odp_crypto_compl_status_t cipher_rc; odp_crypto_compl_status_t auth_rc; odph_ipv4hdr_t *ip; + uint32_t seglen; /* Check crypto result */ event = odp_packet_to_buffer(pkt); @@ -1004,7 +1009,7 @@ pkt_disposition_e do_ipsec_out_finish(odp_packet_t pkt, return PKT_DROP; if (!is_crypto_compl_status_ok(&auth_rc)) return PKT_DROP; - ip = (odph_ipv4hdr_t *)odp_packet_l3(pkt); + ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &seglen); /* Finalize the IPv4 header */ ip->ttl = ctx->ipsec.ip_ttl; @@ -1148,7 +1153,7 @@ void *pktio_thread(void *arg EXAMPLE_UNUSED) /* Check for drop */ if (PKT_DROP == rc) - odph_packet_free(pkt); + odp_packet_free(pkt); /* Print packet counts every once in a while */ if (PKT_DONE == rc) { diff --git a/example/ipsec/odp_ipsec_stream.c b/example/ipsec/odp_ipsec_stream.c index 139d00e..cd01f68 100644 --- a/example/ipsec/odp_ipsec_stream.c +++ b/example/ipsec/odp_ipsec_stream.c @@ -16,7 +16,6 @@ #include <odp.h> -#include <odph_packet.h> #include <odph_eth.h> #include <odph_ip.h> #include <odph_icmp.h> @@ -176,7 +175,6 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream, odp_buffer_pool_t pkt_pool) { ipsec_cache_entry_t *entry = stream->input.entry; - odp_buffer_t bfr; odp_packet_t pkt; uint8_t *base; uint8_t *data; @@ -188,18 +186,16 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream, stream_pkt_hdr_t *test; uint i; - /* Get buffer */ - bfr = odp_buffer_alloc(pkt_pool); - if (ODP_BUFFER_INVALID == bfr) + /* Get packet */ + pkt = odp_packet_alloc(pkt_pool, 0); + if (ODP_PACKET_INVALID == pkt) return ODP_PACKET_INVALID; - pkt = odp_packet_from_buffer(bfr); - odp_packet_init(pkt); base = odp_packet_data(pkt); - data = odp_packet_data(pkt); + data = base; /* Ethernet */ odp_packet_set_inflag_eth(pkt, 1); - odp_packet_set_l2_offset(pkt, data - base); + odp_packet_l2_offset_set(pkt, data - base); eth = (odph_ethhdr_t *)data; data += sizeof(*eth); @@ -209,10 +205,10 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream, /* IPv4 */ odp_packet_set_inflag_ipv4(pkt, 1); - odp_packet_set_l3_offset(pkt, data - base); + odp_packet_l3_offset_set(pkt, data - base); ip = (odph_ipv4hdr_t *)data; data += sizeof(*ip); - odp_packet_set_l4_offset(pkt, data - base); + odp_packet_l4_offset_set(pkt, data - base); /* Wait until almost finished to fill in mutable fields */ memset((char *)ip, 0, sizeof(*ip)); @@ -306,7 +302,7 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream, /* Since ESP can pad we can now fix IP length */ ip->tot_len = odp_cpu_to_be_16(data - (uint8_t *)ip); - odp_packet_set_len(pkt, data - base); + odp_packet_push_tail(pkt, data - base); /* Close AH if specified */ if (ah) { @@ -347,9 +343,10 @@ bool verify_ipv4_packet(stream_db_entry_t *stream, int hdr_len; odph_icmphdr_t *icmp; stream_pkt_hdr_t *test; + uint32_t seglen; /* Basic IPv4 verify (add checksum verification) */ - data = odp_packet_l3(pkt); + data = odp_packet_l3_ptr(pkt, &seglen); ip = (odph_ipv4hdr_t *)data; data += sizeof(*ip); if (0x45 != ip->ver_ihl) @@ -549,7 +546,7 @@ bool verify_stream_db_outputs(void) good = verify_ipv4_packet(stream, pkt); if (good) stream->verified++; - odph_packet_free(pkt); + odp_packet_free(pkt); } } diff --git a/example/l2fwd/odp_l2fwd.c b/example/l2fwd/odp_l2fwd.c index 3c1fd6a..9721ab7 100644 --- a/example/l2fwd/odp_l2fwd.c +++ b/example/l2fwd/odp_l2fwd.c @@ -19,7 +19,6 @@ #include <odp.h> #include <odph_linux.h> -#include <odph_packet.h> #include <odph_eth.h> #include <odph_ip.h> @@ -476,7 +475,7 @@ static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len) pkt = pkt_tbl[i]; if (odp_unlikely(odp_packet_error(pkt))) { - odph_packet_free(pkt); /* Drop */ + odp_packet_free(pkt); /* Drop */ pkt_cnt--; } else if (odp_unlikely(i != j++)) { pkt_tbl[j-1] = pkt; diff --git a/example/packet/odp_pktio.c b/example/packet/odp_pktio.c index f2e7b2d..6f5225a 100644 --- a/example/packet/odp_pktio.c +++ b/example/packet/odp_pktio.c @@ -19,7 +19,6 @@ #include <odp.h> #include <odph_linux.h> -#include <odph_packet.h> #include <odph_eth.h> #include <odph_ip.h> @@ -460,7 +459,7 @@ static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len) pkt = pkt_tbl[i]; if (odp_unlikely(odp_packet_error(pkt))) { - odph_packet_free(pkt); /* Drop */ + odp_packet_free(pkt); /* Drop */ pkt_cnt--; } else if (odp_unlikely(i != j++)) { pkt_tbl[j-1] = pkt; @@ -485,11 +484,12 @@ static void swap_pkt_addrs(odp_packet_t pkt_tbl[], unsigned len) odph_ipv4hdr_t *ip; uint32be_t ip_tmp_addr; /* tmp ip addr */ unsigned i; + uint32_t seglen; for (i = 0; i < len; ++i) { pkt = pkt_tbl[i]; if (odp_packet_inflag_eth(pkt)) { - eth = (odph_ethhdr_t *)odp_packet_l2(pkt); + eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, &seglen); tmp_addr = eth->dst; eth->dst = eth->src; @@ -497,7 +497,8 @@ static void swap_pkt_addrs(odp_packet_t pkt_tbl[], unsigned len) if (odp_packet_inflag_ipv4(pkt)) { /* IPv4 */ - ip = (odph_ipv4hdr_t *)odp_packet_l3(pkt); + ip = (odph_ipv4hdr_t *) + odp_packet_l3_ptr(pkt, &seglen); ip_tmp_addr = ip->src_addr; ip->src_addr = ip->dst_addr; diff --git a/helper/include/odph_ip.h b/helper/include/odph_ip.h index 2c83c0f..9f5b501 100644 --- a/helper/include/odph_ip.h +++ b/helper/include/odph_ip.h @@ -82,7 +82,9 @@ static inline int odph_ipv4_csum_valid(odp_packet_t pkt) if (!odp_packet_l3_offset(pkt)) return 0; - memcpy(&ip, odp_packet_l3(pkt), sizeof(odph_ipv4hdr_t)); + odp_packet_copydata_out(pkt, odp_packet_l3_offset(pkt), + sizeof(odph_ipv4hdr_t), &ip); + w = (uint16_t *)(void *)&ip; chksum = ip.chksum; ip.chksum = 0x0; @@ -105,12 +107,13 @@ static inline uint16sum_t odph_ipv4_csum_update(odp_packet_t pkt) { uint16_t *w; odph_ipv4hdr_t *ip; + uint32_t seglen; int nleft = sizeof(odph_ipv4hdr_t); if (!odp_packet_l3_offset(pkt)) return 0; - ip = (odph_ipv4hdr_t *)odp_packet_l3(pkt); + ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &seglen); w = (uint16_t *)(void *)ip; ip->chksum = odp_chksum(w, nleft); return ip->chksum; @@ -137,16 +140,30 @@ typedef struct ODP_PACKED { /** @internal Compile time assert */ ODP_STATIC_ASSERT(sizeof(odph_ipv6hdr_t) == ODPH_IPV6HDR_LEN, "ODPH_IPV6HDR_T__SIZE_ERROR"); +/** + * IPv6 Header extensions + */ +typedef struct ODP_PACKED { + uint8_t next_hdr; /**< Protocol of next header */ + uint8_t ext_len; /**< Length of this extention in 8 byte units, + not counting first 8 bytes, so 0 = 8 bytes + 1 = 16 bytes, etc. */ + uint8_t filler[6]; /**< Fill out first 8 byte segment */ +} odph_ipv6hdr_ext_t; + /** @name * IP protocol values (IPv4:'proto' or IPv6:'next_hdr') * @{*/ -#define ODPH_IPPROTO_ICMP 0x01 /**< Internet Control Message Protocol (1) */ -#define ODPH_IPPROTO_TCP 0x06 /**< Transmission Control Protocol (6) */ -#define ODPH_IPPROTO_UDP 0x11 /**< User Datagram Protocol (17) */ -#define ODPH_IPPROTO_SCTP 0x84 /**< Stream Control Transmission Protocol (132) */ -#define ODPH_IPPROTO_FRAG 0x2C /**< Fragment (44) */ -#define ODPH_IPPROTO_AH 0x33 /**< Authentication Header (51) */ -#define ODPH_IPPROTO_ESP 0x32 /**< Encapsulating Security Payload (50) */ +#define ODPH_IPPROTO_HOPOPTS 0x00 /**< IPv6 hop-by-hop options */ +#define ODPH_IPPROTO_ICMP 0x01 /**< Internet Control Message Protocol (1) */ +#define ODPH_IPPROTO_TCP 0x06 /**< Transmission Control Protocol (6) */ +#define ODPH_IPPROTO_UDP 0x11 /**< User Datagram Protocol (17) */ +#define ODPH_IPPROTO_ROUTE 0x2B /**< IPv6 Routing header (43) */ +#define ODPH_IPPROTO_FRAG 0x2C /**< IPv6 Fragment (44) */ +#define ODPH_IPPROTO_AH 0x33 /**< Authentication Header (51) */ +#define ODPH_IPPROTO_ESP 0x32 /**< Encapsulating Security Payload (50) */ +#define ODPH_IPPROTO_INVALID 0xFF /**< Reserved invalid by IANA */ + /**@}*/ #ifdef __cplusplus diff --git a/helper/include/odph_packet.h b/helper/include/odph_packet.h deleted file mode 100644 index 3d53593..0000000 --- a/helper/include/odph_packet.h +++ /dev/null @@ -1,97 +0,0 @@ -/* Copyright (c) 2014, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - - -/** - * @file - * - * Optional ODP packet helper functions - */ - -#ifndef ODPH_PACKET_HELPER_H_ -#define ODPH_PACKET_HELPER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <odp.h> - -/** - * Helper: Tests if packet is valid - * - * Allows for more thorough checking than "if (pkt == ODP_PACKET_INVALID)" - * - * @param pkt Packet handle - * - * @return 1 if valid, otherwise 0 - */ -static inline int odph_packet_is_valid(odp_packet_t pkt) -{ - odp_buffer_t buf = odp_packet_to_buffer(pkt); - - return odp_buffer_is_valid(buf); -} - -/** - * Helper: Allocate and initialize a packet buffer from a packet pool - * - * @param pool_id Pool handle - * - * @note The pool must have been created with 'buf_type=ODP_BUFFER_TYPE_PACKET' - * - * @return Packet handle or ODP_PACKET_INVALID - */ -static inline odp_packet_t odph_packet_alloc(odp_buffer_pool_t pool_id) -{ - odp_packet_t pkt; - odp_buffer_t buf; - - buf = odp_buffer_alloc(pool_id); - if (odp_unlikely(!odp_buffer_is_valid(buf))) - return ODP_PACKET_INVALID; - - pkt = odp_packet_from_buffer(buf); - odp_packet_init(pkt); - - return pkt; -} - -/** - * Helper: Free a packet buffer back into the packet pool - * - * @param pkt Packet handle - */ -static inline void odph_packet_free(odp_packet_t pkt) -{ - odp_buffer_t buf = odp_packet_to_buffer(pkt); - - odp_buffer_free(buf); -} - -/** - * Helper: Packet buffer maximum data size - * - * @note odp_packet_buf_size(pkt) != odp_packet_get_len(pkt), the former returns - * the max length of the buffer, the latter the size of a received packet. - * - * @param pkt Packet handle - * - * @return Packet buffer maximum data size - */ -static inline size_t odph_packet_buf_size(odp_packet_t pkt) -{ - odp_buffer_t buf = odp_packet_to_buffer(pkt); - - return odp_buffer_size(buf); -} - - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/helper/include/odph_udp.h b/helper/include/odph_udp.h index b2eaf03..6539913 100644 --- a/helper/include/odph_udp.h +++ b/helper/include/odph_udp.h @@ -57,6 +57,7 @@ static inline uint16_t odph_ipv4_udp_chksum(odp_packet_t pkt) odph_udphdr_t *udph; odph_ipv4hdr_t *iph; uint16_t udplen; + uint32_t l3_seglen, l4_seglen; if (!odp_packet_l3_offset(pkt)) return 0; @@ -64,8 +65,8 @@ static inline uint16_t odph_ipv4_udp_chksum(odp_packet_t pkt) if (!odp_packet_l4_offset(pkt)) return 0; - iph = (odph_ipv4hdr_t *)odp_packet_l3(pkt); - udph = (odph_udphdr_t *)odp_packet_l4(pkt); + iph = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &l3_seglen); + udph = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, &l4_seglen); udplen = odp_be_to_cpu_16(udph->length); /* the source ip */ diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index cc78de3..911f925 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -47,7 +47,6 @@ subdirheaders_HEADERS = \ $(top_srcdir)/helper/include/odph_ip.h \ $(top_srcdir)/helper/include/odph_ipsec.h \ $(top_srcdir)/helper/include/odph_linux.h \ - $(top_srcdir)/helper/include/odph_packet.h \ $(top_srcdir)/helper/include/odph_ring.h \ $(top_srcdir)/helper/include/odph_udp.h diff --git a/platform/linux-generic/include/api/odp_packet.h b/platform/linux-generic/include/api/odp_packet.h index 5298fa0..311f1ab 100644 --- a/platform/linux-generic/include/api/odp_packet.h +++ b/platform/linux-generic/include/api/odp_packet.h @@ -1,7 +1,7 @@ /* Copyright (c) 2013, Linaro Limited * All rights reserved. * - * SPDX-License-Identifier: BSD-3-Clause + * SPDX-License-Identifier: BSD-3-Clause */ @@ -19,6 +19,7 @@ extern "C" { #endif #include <odp_buffer.h> +#include <odp_platform_types.h> /** @defgroup odp_packet ODP PACKET * Operations on a packet. @@ -26,15 +27,68 @@ extern "C" { */ +/* + * Packet API v0.5 notes + * - Push/pull operations only on packet level + * - Push/pull within limits of segment headroom/tailroom/data lengths + * - Segment data length must be always at least one byte (i.e. there are no + * empty segments) + * - Head/tailroom content belong to packet content (in addition to data + * and meta-data) and thus is preserved over packet ownership changes. + * - _addr refer to a fixed address, which operations do not modify + * - _ptr refer to pointer to data, which may be modified by operations + */ + + +/* + * + * Alloc and free + * ******************************************************** + * + */ + /** - * Initialize the packet + * Allocate a packet from a buffer pool * - * Needs to be called if the user allocates a packet buffer, i.e. the packet - * has not been received from I/O through ODP. + * Allocates packet and creates segmentation for requested packet data length. + * Pool must have been created with buffer type ODP_BUFFER_TYPE_PACKET. The + * packet is initialized with data pointers and lengths set according to + * len and the default headroom length setting (defined by + * ODP_CONFIG_PACKET_HEADROOM ??). All other packet meta-data is set to their + * default values. * - * @param pkt Packet handle + * @param pool Pool handle + * @param len Packet data length + * + * @return Packet handle or ODP_PACKET_INVALID + */ +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool, uint32_t len); + +/** + * Free packet + * + * Frees the packet into the buffer pool it was allocated from. + * + * @param pkt Packet handle */ -void odp_packet_init(odp_packet_t pkt); +void odp_packet_free(odp_packet_t pkt); + +/** + * Reset packet + * + * Resets all packet meta-data to their default values. Packet length is used + * to initialize pointers and lengths. It must be less than the total buffer + * length of the packet minus the default headroom length. Packet is not + * modified on failure. + * + * @param pkt Packet handle + * @param len Packet data length + * + * @return 0 on success, non-zero on failure + * + * @see odp_packet_buf_len() + */ +int odp_packet_reset(odp_packet_t pkt, uint32_t len); /** * Convert a buffer handle to a packet handle @@ -54,181 +108,465 @@ odp_packet_t odp_packet_from_buffer(odp_buffer_t buf); */ odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt); + +/* + * + * Pointers and lengths + * ******************************************************** + * + */ + +/** + * Packet head address + * + * Returns start address of the first segment. Packet level headroom starts + * from here. Use e.g. odp_packet_data() or odp_packet_l2_ptr() to find packet + * data start address. + * + * @param pkt Packet handle + * + * @return Start address of the first segment, or NULL on an error + * + * @see odp_packet_data(), odp_packet_l2_ptr(), odp_packet_headroom() + */ +void *odp_packet_head(odp_packet_t pkt); + +/** + * Total packet buffer length + * + * Returns sum of buffer lengths over all packet segments. + * + * @param pkt Packet handle + * + * @return Total buffer length in bytes + * + * @see odp_packet_reset() + */ +uint32_t odp_packet_buf_len(odp_packet_t pkt); + +/** + * Packet data pointer + * + * Returns the current packet data pointer. When a packet is received + * from packet input, this points to the first byte of the received + * packet. Packet level offsets are calculated relative to this position. + * + * User can adjust the data pointer with head_push/head_pull (does not modify + * segmentation) and add_data/rem_data calls (may modify segmentation). + * + * @param pkt Packet handle + * + * @return Pointer to the packet data, or NULL on an error + * + * @see odp_packet_l2_ptr(), odp_packet_seg_len() + */ +void *odp_packet_data(odp_packet_t pkt); + +/** + * Packet segment data length + * + * Returns number of data bytes following the current data pointer + * (odp_packet_data()) location in the segment. + * + * @param pkt Packet handle + * + * @return Segment data length in bytes (pointed by odp_packet_data()) + * + * @see odp_packet_data() + */ +uint32_t odp_packet_seg_len(odp_packet_t pkt); + /** - * Set the packet length + * Packet data length + * + * Returns sum of data lengths over all packet segments. * * @param pkt Packet handle - * @param len Length of packet in bytes + * + * @return Packet data length */ -void odp_packet_set_len(odp_packet_t pkt, size_t len); +uint32_t odp_packet_len(odp_packet_t pkt); /** - * Get the packet length + * Packet headroom length + * + * Returns the current packet level headroom length. * * @param pkt Packet handle * - * @return Packet length in bytes + * @return Headroom length */ -size_t odp_packet_get_len(odp_packet_t pkt); +uint32_t odp_packet_headroom(odp_packet_t pkt); /** - * Set packet user context + * Packet tailroom length * - * @param buf Packet handle - * @param ctx User context + * Returns the current packet level tailroom length. * + * @param pkt Packet handle + * + * @return Tailroom length */ -void odp_packet_set_ctx(odp_packet_t buf, const void *ctx); +uint32_t odp_packet_tailroom(odp_packet_t pkt); /** - * Get packet user context + * Packet tailroom pointer + * + * Returns pointer to the start of the current packet level tailroom. * - * @param buf Packet handle + * User can adjust the tail pointer with tail_push/tail_pull (does not modify + * segmentation) and add_data/rem_data calls (may modify segmentation). * - * @return User context + * @param pkt Packet handle + * + * @return Tailroom pointer, or NULL on an error + * + * @see odp_packet_tailroom() */ -void *odp_packet_get_ctx(odp_packet_t buf); +void *odp_packet_tail(odp_packet_t pkt); /** - * Packet buffer start address + * Push out packet head + * + * Increase packet data length by moving packet head into packet headroom. + * Packet headroom is decreased with the same amount. The packet head may be + * pushed out up to 'headroom' bytes. Packet is not modified if there's not + * enough headroom space. * - * Returns a pointer to the start of the packet buffer. The address is not - * necessarily the same as packet data address. E.g. on a received Ethernet - * frame, the protocol header may start 2 or 6 bytes within the buffer to - * ensure 32 or 64-bit alignment of the IP header. + * odp_packet_xxx: + * seg_len += len + * len += len + * headroom -= len + * data -= len * - * Use odp_packet_l2(pkt) to get the start address of a received valid frame - * or odp_packet_data(pkt) to get the current packet data address. + * Operation does not modify packet segmentation or move data. Handles and + * pointers remain valid. User is responsible to update packet meta-data + * offsets when needed. * * @param pkt Packet handle + * @param len Number of bytes to push the head (0 ... headroom) * - * @return Pointer to the start of the packet buffer + * @return The new data pointer, or NULL in case of an error. * - * @see odp_packet_l2(), odp_packet_data() + * @see odp_packet_headroom(), odp_packet_pull_head() */ -uint8_t *odp_packet_addr(odp_packet_t pkt); +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len); /** - * Packet data address + * Pull in packet head * - * Returns the current packet data address. When a packet is received from - * packet input, the data address points to the first byte of the packet. + * Decrease packet data length by removing data from the head of the packet. + * Packet headroom is increased with the same amount. Packet head may be pulled + * in up to seg_len - 1 bytes (i.e. packet data pointer must stay in the + * first segment). Packet is not modified if there's not enough data. + * + * odp_packet_xxx: + * seg_len -= len + * len -= len + * headroom += len + * data += len + * + * Operation does not modify packet segmentation or move data. Handles and + * pointers remain valid. User is responsible to update packet meta-data + * offsets when needed. * * @param pkt Packet handle + * @param len Number of bytes to pull the head (0 ... seg_len - 1) * - * @return Pointer to the packet data + * @return The new data pointer, or NULL in case of an error. * - * @see odp_packet_l2(), odp_packet_addr() + * @see odp_packet_seg_len(), odp_packet_push_head() */ -uint8_t *odp_packet_data(odp_packet_t pkt); +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len); /** - * Get pointer to the start of the L2 frame + * Push out packet tail + * + * Increase packet data length by moving packet tail into packet tailroom. + * Packet tailroom is decreased with the same amount. The packet tail may be + * pushed out up to 'tailroom' bytes. Packet is not modified if there's not + * enough tailroom. * - * The L2 frame header address is not necessarily the same as the address of the - * packet buffer, see odp_packet_addr() + * last_seg: + * data_len += len + * + * odp_packet_xxx: + * len += len + * tail += len + * tailroom -= len + * + * Operation does not modify packet segmentation or move data. Handles, + * pointers and offsets remain valid. * * @param pkt Packet handle + * @param len Number of bytes to push the tail (0 ... tailroom) * - * @return Pointer to L2 header or NULL if not found + * @return The old tail pointer, or NULL in case of an error. * - * @see odp_packet_addr(), odp_packet_data() + * @see odp_packet_tailroom(), odp_packet_pull_tail() */ -uint8_t *odp_packet_l2(odp_packet_t pkt); +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len); /** - * Return the byte offset from the packet buffer to the L2 frame + * Pull in packet tail + * + * Decrease packet data length by removing data from the tail of the packet. + * Packet tailroom is increased with the same amount. Packet tail may be pulled + * in up to last segment data_len - 1 bytes. (i.e. packet tail must stay in the + * last segment). Packet is not modified if there's not enough data. + * + * last_seg: + * data_len -= len + * + * odp_packet_xxx: + * len -= len + * tail -= len + * tailroom += len + * + * Operation does not modify packet segmentation or move data. Handles and + * pointers remain valid. User is responsible to update packet meta-data + * offsets when needed. * * @param pkt Packet handle + * @param len Number of bytes to pull the tail (0 ... last_seg:data_len - 1) * - * @return L2 byte offset or ODP_PACKET_OFFSET_INVALID if not found + * @return The new tail pointer, or NULL in case of an error. */ -size_t odp_packet_l2_offset(odp_packet_t pkt); +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len); /** - * Set the byte offset to the L2 frame + * Packet offset pointer * - * @param pkt Packet handle - * @param offset L2 byte offset + * Returns pointer to data in the packet offset. The packet level byte offset is + * calculated from the current odp_packet_data() position. Optionally outputs + * handle to the segment and number of data bytes in the segment following the + * pointer. + * + * @param[in] pkt Packet handle + * @param[in] offset Byte offset into the packet + * @param[out] len Number of data bytes remaining in the segment (output). + * Ignored when NULL. + * @param[out] seg Handle to the segment containing the address (output). + * Ignored when NULL. + * + * @return Pointer to the offset, or NULL in case of an error. + */ +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, + odp_packet_seg_t *seg); + +/* + * + * Meta-data + * ******************************************************** + * + */ + +/** + * Packet pool + * + * Returns handle to the buffer pool where the packet was allocated from. + * + * @param pkt Packet handle + * + * @return Buffer pool handle */ -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset); +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt); +/** + * Packet input interface + * + * Returns handle to the packet IO interface which received the packet or + * ODP_PKTIO_INVALID when the packet was allocated/reset by the application. + * + * @param pkt Packet handle + * + * @return Packet interface handle, or ODP_PKTIO_INVALID + */ +odp_pktio_t odp_packet_input(odp_packet_t pkt); /** - * Get pointer to the start of the L3 packet + * User context pointer + * + * Return previously stored user context pointer. * * @param pkt Packet handle * - * @return Pointer to L3 packet or NULL if not found + * @return User context pointer + */ +void *odp_packet_user_ptr(odp_packet_t pkt); + +/** + * Set user context pointer + * + * Each packet has room for a user defined context. The context can be stored + * either as a pointer OR as a uint64_t value, but not both at the same time. + * The latest context set operation determines which one has been stored. * + * @param pkt Packet handle + * @param ctx User context pointer */ -uint8_t *odp_packet_l3(odp_packet_t pkt); +void odp_packet_user_ptr_set(odp_packet_t buf, const void *ctx); /** - * Return the byte offset from the packet buffer to the L3 packet + * User context data (uint64_t) + * + * Return previously stored user context uint64_t value. * * @param pkt Packet handle * - * @return L3 byte offset or ODP_PACKET_OFFSET_INVALID if not found + * @return User context data */ -size_t odp_packet_l3_offset(odp_packet_t pkt); +uint64_t odp_packet_user_u64(odp_packet_t pkt); /** - * Set the byte offset to the L3 packet + * Set user context data (uint64_t) * - * @param pkt Packet handle - * @param offset L3 byte offset + * Each packet has room for a user defined context. The context can be stored + * either as a pointer OR as a uint64_t value, but not both at the same time. + * The latest context set operation determines which one has been stored. + * + * @param pkt Packet handle + * @param ctx User context data */ -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset); +void odp_packet_user_u64_set(odp_packet_t buf, uint64_t ctx); +/** + * Layer 2 start pointer + * + * Returns pointer to the start of the layer 2 header. Optionally, outputs + * number of data bytes in the segment following the pointer. + * + * @param[in] pkt Packet handle + * @param[out] len Number of data bytes remaining in the segment (output). + * Ignored when NULL. + * + * @return Layer 2 start pointer, or NULL + * + * @see odp_packet_l2_offset(), odp_packet_l2_offset_set() + */ +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len); /** - * Get pointer to the start of the L4 packet + * Layer 2 start offset + * + * Returns offset to the start of the layer 2 header. The offset is calculated + * from the current odp_packet_data() position in bytes. + * + * User is responsible to update the offset when modifying the packet data + * pointer position. * * @param pkt Packet handle * - * @return Pointer to L4 packet or NULL if not found + * @return Layer 2 start offset or ODP_PACKET_OFFSET_INVALID if not found + */ +uint32_t odp_packet_l2_offset(odp_packet_t pkt); + +/** + * Set layer 2 start offset + * + * Set offset to the start of the layer 2 header. The offset is calculated from + * the current odp_packet_data() position in bytes. Offset must not exceed + * packet data length. Packet is not modified on an error. * + * @param pkt Packet handle + * @param offset Layer 2 start offset (0 ... odp_packet_len()-1) + * + * @return 0 on success, non-zero on error */ -uint8_t *odp_packet_l4(odp_packet_t pkt); +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset); /** - * Return the byte offset from the packet buffer to the L4 packet + * Layer 3 start pointer + * + * Returns pointer to the start of the layer 3 header. Optionally, outputs + * number of data bytes in the segment following the pointer. + * + * @param[in] pkt Packet handle + * @param[out] len Number of data bytes remaining in the segment (output). + * Ignored when NULL. + * + * @return Layer 3 start pointer, or NULL + * + * @see odp_packet_l3_offset(), odp_packet_l3_offset_set() + */ +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len); + +/** + * Layer 3 start offset + * + * Returns offset to the start of the layer 3 header. The offset is calculated + * from the current odp_packet_data() position in bytes. + * + * User is responsible to update the offset when modifying the packet data + * pointer position. * * @param pkt Packet handle * - * @return L4 byte offset or ODP_PACKET_OFFSET_INVALID if not found + * @return Layer 3 start offset or ODP_PACKET_OFFSET_INVALID if not found */ -size_t odp_packet_l4_offset(odp_packet_t pkt); +uint32_t odp_packet_l3_offset(odp_packet_t pkt); /** - * Set the byte offset to the L4 packet + * Set layer 3 start offset + * + * Set offset to the start of the layer 3 header. The offset is calculated from + * the current odp_packet_data() position in bytes. Offset must not exceed + * packet data length. Packet is not modified on an error. * * @param pkt Packet handle - * @param offset L4 byte offset + * @param offset Layer 3 start offset (0 ... odp_packet_len()-1) + * + * @return 0 on success, non-zero on error */ -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset); +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset); /** - * Print (debug) information about the packet + * Layer 4 start pointer + * + * Returns pointer to the start of the layer 4 header. Optionally, outputs + * number of data bytes in the segment following the pointer. + * + * @param[in] pkt Packet handle + * @param[out] len Number of data bytes remaining in the segment (output). + * Ignored when NULL. + * + * @return Layer 4 start pointer, or NULL + * + * @see odp_packet_l4_offset(), odp_packet_l4_offset_set() + */ +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len); + +/** + * Layer 4 start offset + * + * Returns offset to the start of the layer 4 header. The offset is calculated + * from the current odp_packet_data() position in bytes. + * + * User is responsible to update the offset when modifying the packet data + * pointer position. * * @param pkt Packet handle + * + * @return Layer 4 start offset or ODP_PACKET_OFFSET_INVALID if not found */ -void odp_packet_print(odp_packet_t pkt); +uint32_t odp_packet_l4_offset(odp_packet_t pkt); /** - * Copy contents and metadata from pkt_src to pkt_dst - * Useful when creating copies of packets + * Set layer 4 start offset + * + * Set offset to the start of the layer 4 header. The offset is calculated from + * the current odp_packet_data() position in bytes. Offset must not exceed + * packet data length. Packet is not modified on an error. * - * @param pkt_dst Destination packet - * @param pkt_src Source packet + * @param pkt Packet handle + * @param offset Layer 4 start offset (0 ... odp_packet_len()-1) * - * @return 0 if successful + * @return 0 on success, non-zero on error */ -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src); +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset); /** - * Tests if packet is segmented (a scatter/gather list) + * Tests if packet is segmented * * @param pkt Packet handle * @@ -237,184 +575,245 @@ int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src); int odp_packet_is_segmented(odp_packet_t pkt); /** - * Segment count + * Number of segments * * Returns number of segments in the packet. A packet has always at least one - * segment (the packet buffer itself). + * segment. * * @param pkt Packet handle * - * @return Segment count + * @return Number of segments (>0) */ -int odp_packet_seg_count(odp_packet_t pkt); +int odp_packet_num_segs(odp_packet_t pkt); /** - * Get segment by index + * First segment in packet * - * @param pkt Packet handle - * @param index Segment index (0 ... seg_count-1) + * A packet has always the first segment (has at least one segment). * - * @return Segment handle, or ODP_PACKET_SEG_INVALID on an error + * @param pkt Packet handle + * + * @return Handle to the first segment */ -odp_packet_seg_t odp_packet_seg(odp_packet_t pkt, int index); +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt); /** - * Get next segment + * Last segment in packet * - * @param pkt Packet handle - * @param seg Current segment handle + * A packet has always the last segment (has at least one segment). + * + * @param pkt Packet handle * - * @return Handle to next segment, or ODP_PACKET_SEG_INVALID on an error + * @return Handle to the last segment */ -odp_packet_seg_t odp_packet_seg_next(odp_packet_t pkt, odp_packet_seg_t seg); +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt); /** - * Segment info + * Next segment in packet * - * Copies segment parameters into the info structure. + * Returns handle to the next segment after the current segment, or + * ODP_PACKET_SEG_INVALID if there are no more segments. Use + * odp_packet_first_seg() to get handle to the first segment. * - * @param pkt Packet handle - * @param seg Segment handle - * @param info Pointer to segment info structure + * @param pkt Packet handle + * @param seg Current segment handle + * + * @return Handle to the next segment, or ODP_PACKET_SEG_INVALID + */ +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg); + + +/* + * + * Segment level + * ******************************************************** * - * @return 0 if successful, otherwise non-zero */ -int odp_packet_seg_info(odp_packet_t pkt, odp_packet_seg_t seg, - odp_packet_seg_info_t *info); /** - * Segment start address + * Segment buffer address + * + * Returns start address of the segment. * * @param pkt Packet handle * @param seg Segment handle * - * @return Segment start address, or NULL on an error + * @return Start address of the segment, or NULL on an error + * + * @see odp_packet_seg_buf_len() */ -void *odp_packet_seg_addr(odp_packet_t pkt, odp_packet_seg_t seg); +void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg); /** - * Segment maximum data size + * Segment buffer length + * + * Returns segment buffer length in bytes. * * @param pkt Packet handle * @param seg Segment handle * - * @return Segment maximum data size + * @return Segment buffer length in bytes + * + * @see odp_packet_seg_buf_addr() */ -size_t odp_packet_seg_size(odp_packet_t pkt, odp_packet_seg_t seg); +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, odp_packet_seg_t seg); /** - * Segment data address + * Segment data pointer + * + * Returns pointer to the first byte of data in the segment. * * @param pkt Packet handle * @param seg Segment handle * - * @return Segment data address + * @return Pointer to the segment data, or NULL on an error + * + * @see odp_packet_seg_data_len() */ void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg); /** * Segment data length * + * Returns segment data length in bytes. + * * @param pkt Packet handle * @param seg Segment handle * - * @return Segment data length + * @return Segment data length in bytes + * + * @see odp_packet_seg_data() */ -size_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg); +uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg); -/** - * Segment headroom + +/* * - * seg_headroom = seg_data - seg_addr + * Manipulation + * ******************************************************** * - * @param pkt Packet handle - * @param seg Segment handle - * - * @return Number of octets from seg_addr to seg_data */ -size_t odp_packet_seg_headroom(odp_packet_t pkt, odp_packet_seg_t seg); + /** - * Segment tailroom + * Add data into an offset * - * seg_tailroom = seg_size - seg_headroom - seg_data_len + * Increases packet data length by adding new data area into the specified + * offset. The operation returns a new packet handle on success. It may modify + * packet segmentation and move data. Handles and pointers must be updated + * after the operation. User is responsible to update packet meta-data offsets + * when needed. The packet is not modified on an error. * - * @param pkt Packet handle - * @param seg Segment handle + * @param pkt Packet handle + * @param offset Byte offset into the packet + * @param len Number of bytes to add into the offset * - * @return Number of octets from end-of-data to end-of-segment + * @return New packet handle, or ODP_PACKET_INVALID in case of an error. */ -size_t odp_packet_seg_tailroom(odp_packet_t pkt, odp_packet_seg_t seg); +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, + uint32_t len); /** - * Push out segment head + * Remove data from an offset * - * Push out segment data address (away from data) and increase data length. - * Does not modify packet in case of an error. + * Decreases packet data length by removing data from the specified offset. + * The operation returns a new packet handle on success, and may modify + * packet segmentation and move data. Handles and pointers must be updated + * after the operation. User is responsible to update packet meta-data offsets + * when needed. The packet is not modified on an error. * - * seg_data -= len - * seg_data_len += len + * @param pkt Packet handle + * @param offset Byte offset into the packet + * @param len Number of bytes to remove from the offset * - * @param pkt Packet handle - * @param seg Segment handle - * @param len Number of octets to push head (0 ... seg_headroom) + * @return New packet handle, or ODP_PACKET_INVALID in case of an error. + */ +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, + uint32_t len); + + +/* + * + * Copy + * ******************************************************** * - * @return New segment data address, or NULL on an error */ -void *odp_packet_seg_push_head(odp_packet_t pkt, odp_packet_seg_t seg, - size_t len); /** - * Pull in segment head + * Copy packet * - * Pull in segment data address (towards data) and decrease data length. - * Does not modify packet in case of an error. + * Create a new copy of the packet. The new packet is exact copy of the source + * packet (incl. data and meta-data). The pool must have been created with + * buffer type ODP_BUFFER_TYPE_PACKET. * - * seg_data += len - * seg_data_len -= len + * @param pkt Packet handle + * @param pool Buffer pool for allocation of the new packet. * - * @param pkt Packet handle - * @param seg Segment handle - * @param len Number of octets to pull head (0 ... seg_data_len) + * @return Handle to the copy of the packet, or ODP_PACKET_INVALID + */ +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool); + +/** + * Copy data from packet * - * @return New segment data address, or NULL on an error + * Copy 'len' bytes of data from the packet level offset to the destination + * address. + * + * @param pkt Packet handle + * @param offset Byte offset into the packet + * @param len Number of bytes to copy + * @param dst Destination address + * + * @return 0 on success, otherwise non-zero */ -void *odp_packet_seg_pull_head(odp_packet_t pkt, odp_packet_seg_t seg, - size_t len); +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, + uint32_t len, void *dst); /** - * Push out segment tail + * Copy data into packet * - * Increase segment data length. - * Does not modify packet in case of an error. + * Copy 'len' bytes of data from the source address into the packet level + * offset. Maximum number of bytes to copy is packet data length minus the + * offset. Packet is not modified on an error. * - * seg_data_len += len + * @param pkt Packet handle + * @param offset Byte offset into the packet + * @param len Number of bytes to copy + * @param src Source address * - * @param pkt Packet handle - * @param seg Segment handle - * @param len Number of octets to push tail (0 ... seg_tailroom) + * @return 0 on success, otherwise non-zero + */ +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, + uint32_t len, const void *src); + +/* + * + * Debugging + * ******************************************************** * - * @return New segment data length, or -1 on an error */ -int odp_packet_seg_push_tail(odp_packet_t pkt, odp_packet_seg_t seg, - size_t len); /** - * Pull in segment tail + * Print packet to the console + * + * Print all packet debug information to the console. * - * Decrease segment data length. - * Does not modify packet in case of an error. + * @param pkt Packet handle + */ +void odp_packet_print(odp_packet_t pkt); + +/** + * Perform full packet validity check * - * seg_data_len -= len + * The operation may consume considerable number of cpu cycles depending on + * the check level. * * @param pkt Packet handle - * @param seg Segment handle - * @param len Number of octets to pull tail (0 ... seg_data_len) * - * @return New segment data length, or -1 on an error + * @return 1 if valid, otherwise 0 */ -int odp_packet_seg_pull_tail(odp_packet_t pkt, odp_packet_seg_t seg, - size_t len); +int odp_packet_is_valid(odp_packet_t pkt); + /** * @} diff --git a/platform/linux-generic/include/api/odp_platform_types.h b/platform/linux-generic/include/api/odp_platform_types.h index 2181eb6..2cfba87 100644 --- a/platform/linux-generic/include/api/odp_platform_types.h +++ b/platform/linux-generic/include/api/odp_platform_types.h @@ -35,28 +35,23 @@ typedef uint32_t odp_buffer_t; /** Invalid buffer */ #define ODP_BUFFER_INVALID (0xffffffff) +/** ODP buffer segment */ +typedef odp_buffer_t odp_buffer_seg_t; + +/** Invalid segment */ +#define ODP_SEGMENT_INVALID ODP_BUFFER_INVALID + /** ODP packet */ typedef odp_buffer_t odp_packet_t; /** Invalid packet */ #define ODP_PACKET_INVALID ODP_BUFFER_INVALID -/** Invalid offset */ -#define ODP_PACKET_OFFSET_INVALID ((uint32_t)-1) - /** ODP packet segment */ -typedef int odp_packet_seg_t; +typedef odp_buffer_t odp_packet_seg_t; /** Invalid packet segment */ -#define ODP_PACKET_SEG_INVALID -1 - -/** ODP packet segment info */ -typedef struct odp_packet_seg_info_t { - void *addr; /**< Segment start address */ - size_t size; /**< Segment maximum data size */ - void *data; /**< Segment data address */ - size_t data_len; /**< Segment data length */ -} odp_packet_seg_info_t; +#define ODP_PACKET_SEG_INVALID ODP_BUFFER_INVALID /** ODP packet IO handle */ typedef uint32_t odp_pktio_t; diff --git a/platform/linux-generic/include/odp_buffer_inlines.h b/platform/linux-generic/include/odp_buffer_inlines.h index 9eb425c..79adf0a 100644 --- a/platform/linux-generic/include/odp_buffer_inlines.h +++ b/platform/linux-generic/include/odp_buffer_inlines.h @@ -143,6 +143,60 @@ static inline void *buffer_map(odp_buffer_hdr_t *buf, return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]); } +static inline odp_buffer_seg_t segment_next(odp_buffer_hdr_t *buf, + odp_buffer_seg_t seg) +{ + odp_buffer_bits_t seghandle; + seghandle.u32 = seg; + + if (seg == ODP_SEGMENT_INVALID || + seghandle.prefix != buf->handle.prefix || + seghandle.seg >= buf->segcount - 1) + return ODP_SEGMENT_INVALID; + else { + seghandle.seg++; + return (odp_buffer_seg_t)seghandle.u32; + } +} + +static inline void *segment_map(odp_buffer_hdr_t *buf, + odp_buffer_seg_t seg, + uint32_t *seglen, + uint32_t limit, + uint32_t hr) +{ + uint32_t seg_offset, buf_left; + odp_buffer_bits_t seghandle; + uint8_t *seg_addr; + seghandle.u32 = seg; + + if (seghandle.prefix != buf->handle.prefix || + seghandle.seg >= buf->segcount) + return NULL; + + seg_addr = (uint8_t *)buf->addr[seghandle.seg]; + seg_offset = seghandle.seg * buf->segsize; + + /* Special handling for packets to account for headroom */ + if (hr > seg_offset) { + /* Can't map this segment if it's nothing but hr */ + if (hr >= seg_offset + buf->segsize) + return NULL; + + /* ...else adjust for hr */ + seg_addr += hr % buf->segsize; + limit += hr % buf->segsize; + } + + if (seglen != NULL) { + buf_left = limit - seg_offset; + *seglen = buf_left < buf->segsize ? buf_left : buf->segsize; + } + + return (void *)seg_addr; +} + + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 859633e..933097b 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -124,6 +124,7 @@ typedef struct odp_buffer_hdr_t { union { uint64_t buf_u64; /* user u64 */ void *buf_ctx; /* user context */ + const void *buf_cctx; /* const alias for ctx */ void *udata_addr; /* user metadata addr */ }; size_t udata_size; /* size of user metadata */ diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index f34a83d..63071c4 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -44,6 +44,7 @@ typedef union { uint32_t vlan:1; /**< VLAN hdr found */ uint32_t vlan_qinq:1; /**< Stacked VLAN found, QinQ */ + uint32_t snap:1; /**< SNAP */ uint32_t arp:1; /**< ARP */ uint32_t ipv4:1; /**< IPv4 */ @@ -54,6 +55,7 @@ typedef union { uint32_t udp:1; /**< UDP */ uint32_t tcp:1; /**< TCP */ + uint32_t tcpopt:1; /**< TCP options present */ uint32_t sctp:1; /**< SCTP */ uint32_t icmp:1; /**< ICMP */ }; @@ -70,7 +72,9 @@ typedef union { struct { /* Bitfield flags for each detected error */ + uint32_t app_error:1; /**< Error bit for application use */ uint32_t frame_len:1; /**< Frame length error */ + uint32_t snap_len:1; /**< Snap length error */ uint32_t l2_chksum:1; /**< L2 checksum error, checks TBD */ uint32_t ip_err:1; /**< IP error, checks TBD */ uint32_t tcp_err:1; /**< TCP error, checks TBD */ @@ -89,7 +93,10 @@ typedef union { struct { /* Bitfield flags for each output option */ - uint32_t l4_chksum:1; /**< Request L4 checksum calculation */ + uint32_t l3_chksum_set:1; /**< L3 chksum bit is valid */ + uint32_t l3_chksum:1; /**< L3 chksum override */ + uint32_t l4_chksum_set:1; /**< L3 chksum bit is valid */ + uint32_t l4_chksum:1; /**< L4 chksum override */ }; } output_flags_t; @@ -110,13 +117,19 @@ typedef struct { uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ + uint32_t payload_offset; /**< offset to payload */ + + uint32_t vlan_s_tag; /**< Parsed 1st VLAN header (S-TAG) */ + uint32_t vlan_c_tag; /**< Parsed 2nd VLAN header (C-TAG) */ + uint32_t l3_protocol; /**< Parsed L3 protocol */ + uint32_t l3_len; /**< Layer 3 length */ + uint32_t l4_protocol; /**< Parsed L4 protocol */ + uint32_t l4_len; /**< Layer 4 length */ uint32_t frame_len; uint32_t headroom; uint32_t tailroom; - uint64_t user_ctx; /* user context */ - odp_pktio_t input; } odp_packet_hdr_t; @@ -134,11 +147,6 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) } /** - * Parse packet and set internal metadata - */ -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset); - -/** * Initialize packet buffer */ static inline void packet_init(pool_entry_t *pool, @@ -169,6 +177,63 @@ static inline void packet_init(pool_entry_t *pool, (pool->s.headroom + size); } +static inline void *packet_offset_map(odp_packet_hdr_t *pkt_hdr, + uint32_t offset, uint32_t *seglen) +{ + if (offset > pkt_hdr->frame_len) + return NULL; + + return buffer_map(&pkt_hdr->buf_hdr, + pkt_hdr->headroom + offset, + seglen, pkt_hdr->frame_len); +} + +#define pull_offset(x, len) (x = x < len ? 0 : x - len) + +static inline void push_head(odp_packet_hdr_t *pkt_hdr, size_t len) +{ + pkt_hdr->headroom -= len; + pkt_hdr->frame_len += len; + pkt_hdr->l2_offset += len; + pkt_hdr->l3_offset += len; + pkt_hdr->l4_offset += len; +} + +static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len) +{ + pkt_hdr->headroom += len; + pkt_hdr->frame_len -= len; + pull_offset(pkt_hdr->l2_offset, len); + pull_offset(pkt_hdr->l3_offset, len); + pull_offset(pkt_hdr->l4_offset, len); +} + +static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len) +{ + pkt_hdr->tailroom -= len; + pkt_hdr->frame_len += len; +} + + +static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len) +{ + pkt_hdr->tailroom += len; + pkt_hdr->frame_len -= len; +} + +static inline void packet_set_len(odp_packet_t pkt, uint32_t len) +{ + odp_packet_hdr(pkt)->frame_len = len; +} + +/* Forward declarations */ +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, + odp_packet_t dstpkt, uint32_t dstoffset, + uint32_t len); + +odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl); + +int _odp_packet_parse(odp_packet_t pkt); #ifdef __cplusplus } diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index d3cdec7..2efe155 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -15,7 +15,6 @@ #include <odp_crypto_internal.h> #include <odp_debug_internal.h> #include <odp_hints.h> -#include <odph_packet.h> #include <string.h> @@ -79,7 +78,7 @@ static enum crypto_alg_err md5_gen(odp_crypto_op_params_t *params, odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_addr(params->out_pkt); + uint8_t *data = odp_packet_data(params->out_pkt); uint8_t *icv = data; uint32_t len = params->auth_range.length; uint8_t hash[EVP_MAX_MD_SIZE]; @@ -107,7 +106,7 @@ static enum crypto_alg_err md5_check(odp_crypto_op_params_t *params, odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_addr(params->out_pkt); + uint8_t *data = odp_packet_data(params->out_pkt); uint8_t *icv = data; uint32_t len = params->auth_range.length; uint32_t bytes = session->auth.data.md5.bytes; @@ -145,7 +144,7 @@ static enum crypto_alg_err des_encrypt(odp_crypto_op_params_t *params, odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_addr(params->out_pkt); + uint8_t *data = odp_packet_data(params->out_pkt); uint32_t len = params->cipher_range.length; DES_cblock *iv = NULL; DES_cblock iv_temp; @@ -188,7 +187,7 @@ static enum crypto_alg_err des_decrypt(odp_crypto_op_params_t *params, odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_addr(params->out_pkt); + uint8_t *data = odp_packet_data(params->out_pkt); uint32_t len = params->cipher_range.length; DES_cblock *iv = (DES_cblock *)session->cipher.iv.data; @@ -360,7 +359,7 @@ odp_crypto_operation(odp_crypto_op_params_t *params, if (completion_event == odp_packet_to_buffer(params->pkt)) completion_event = odp_packet_to_buffer(params->out_pkt); - odph_packet_free(params->pkt); + odp_packet_free(params->pkt); params->pkt = ODP_PACKET_INVALID; } diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 726e086..2618240 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -12,21 +12,57 @@ #include <odph_eth.h> #include <odph_ip.h> +#include <odph_tcp.h> +#include <odph_udp.h> #include <string.h> #include <stdio.h> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, - odph_ipv4hdr_t *ipv4, size_t *offset_out); -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, - odph_ipv6hdr_t *ipv6, size_t *offset_out); +/* + * + * Alloc and free + * ******************************************************** + * + */ -void odp_packet_init(odp_packet_t pkt) +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool_hdl, uint32_t len) +{ + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); + + if (pool->s.params.buf_type != ODP_BUFFER_TYPE_PACKET) + return ODP_PACKET_INVALID; + + /* Handle special case for zero-length packets */ + if (len == 0) { + odp_packet_t pkt = + (odp_packet_t)buffer_alloc(pool_hdl, + pool->s.params.buf_size); + if (pkt != ODP_PACKET_INVALID) + pull_tail(odp_packet_hdr(pkt), + pool->s.params.buf_size); + + return pkt; + } + + return (odp_packet_t)buffer_alloc(pool_hdl, len); +} + +void odp_packet_free(odp_packet_t pkt) +{ + odp_buffer_free((odp_buffer_t)pkt); +} + +int odp_packet_reset(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr); + uint32_t totsize = pool->s.headroom + len + pool->s.tailroom; + + if (totsize > pkt_hdr->buf_hdr.size) + return -1; - packet_init(pool, pkt_hdr, 0); + packet_init(pool, pkt_hdr, len); + return 0; } odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) @@ -39,265 +75,447 @@ odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt) return (odp_buffer_t)pkt; } -void odp_packet_set_len(odp_packet_t pkt, size_t len) +/* + * + * Pointers and lengths + * ******************************************************** + * + */ + +void *odp_packet_head(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + return buffer_map(&pkt_hdr->buf_hdr, 0, NULL, 0); +} + +uint32_t odp_packet_buf_len(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.size; +} + +void *odp_packet_data(odp_packet_t pkt) { - odp_packet_hdr(pkt)->frame_len = len; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + return buffer_map(&pkt_hdr->buf_hdr, pkt_hdr->headroom, NULL, 0); } -size_t odp_packet_get_len(odp_packet_t pkt) +uint32_t odp_packet_seg_len(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.segsize; +} + +uint32_t odp_packet_len(odp_packet_t pkt) { return odp_packet_hdr(pkt)->frame_len; } -uint8_t *odp_packet_addr(odp_packet_t pkt) +uint32_t odp_packet_headroom(odp_packet_t pkt) { - return odp_buffer_addr(odp_packet_to_buffer(pkt)); + return odp_packet_hdr(pkt)->headroom; } -uint8_t *odp_packet_data(odp_packet_t pkt) +uint32_t odp_packet_tailroom(odp_packet_t pkt) { - return odp_packet_addr(pkt) + odp_packet_hdr(pkt)->headroom; + return odp_packet_hdr(pkt)->tailroom; } +void *odp_packet_tail(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + return buffer_map(&pkt_hdr->buf_hdr, pkt_hdr->frame_len, NULL, 0); +} -uint8_t *odp_packet_l2(odp_packet_t pkt) +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) { - const size_t offset = odp_packet_l2_offset(pkt); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) + if (len > pkt_hdr->headroom) return NULL; - return odp_packet_addr(pkt) + offset; + push_head(pkt_hdr, len); + return buffer_map(&pkt_hdr->buf_hdr, 0, NULL, 0); } -size_t odp_packet_l2_offset(odp_packet_t pkt) +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) { - return odp_packet_hdr(pkt)->l2_offset; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (len > pkt_hdr->frame_len) + return NULL; + + pull_head(pkt_hdr, len); + return buffer_map(&pkt_hdr->buf_hdr, 0, NULL, 0); } -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr(pkt)->l2_offset = offset; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint32_t origin = pkt_hdr->frame_len; + + if (len > pkt_hdr->tailroom) + return NULL; + + push_tail(pkt_hdr, len); + return buffer_map(&pkt_hdr->buf_hdr, origin, NULL, 0); } -uint8_t *odp_packet_l3(odp_packet_t pkt) +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) { - const size_t offset = odp_packet_l3_offset(pkt); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) + if (len > pkt_hdr->frame_len) return NULL; - return odp_packet_addr(pkt) + offset; + pull_tail(pkt_hdr, len); + return buffer_map(&pkt_hdr->buf_hdr, pkt_hdr->frame_len, NULL, 0); } -size_t odp_packet_l3_offset(odp_packet_t pkt) +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, + odp_packet_seg_t *seg) { - return odp_packet_hdr(pkt)->l3_offset; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + void *addr = packet_offset_map(pkt_hdr, offset, len); + + if (addr != NULL && seg != NULL) { + odp_buffer_bits_t seghandle; + seghandle.u32 = (uint32_t)pkt; + seghandle.seg = (pkt_hdr->headroom + offset) / + pkt_hdr->buf_hdr.segsize; + *seg = seghandle.handle; + } + + return addr; +} + +/* + * + * Meta-data + * ******************************************************** + * + */ + +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; } -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) +odp_pktio_t odp_packet_input(odp_packet_t pkt) { - odp_packet_hdr(pkt)->l3_offset = offset; + return odp_packet_hdr(pkt)->input; } -uint8_t *odp_packet_l4(odp_packet_t pkt) +void *odp_packet_user_ptr(odp_packet_t pkt) { - const size_t offset = odp_packet_l4_offset(pkt); + return odp_packet_hdr(pkt)->buf_hdr.buf_ctx; +} - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) - return NULL; +void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx) +{ + odp_packet_hdr(pkt)->buf_hdr.buf_cctx = ctx; +} - return odp_packet_addr(pkt) + offset; +uint64_t odp_packet_user_u64(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.buf_u64; } -size_t odp_packet_l4_offset(odp_packet_t pkt) +void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx) { - return odp_packet_hdr(pkt)->l4_offset; + odp_packet_hdr(pkt)->buf_hdr.buf_u64 = ctx; +} + +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + return buffer_map(&pkt_hdr->buf_hdr, + pkt_hdr->headroom + pkt_hdr->l2_offset, + len, pkt_hdr->frame_len); +} + +uint32_t odp_packet_l2_offset(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->l2_offset; +} + +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (offset >= pkt_hdr->frame_len) + return -1; + + pkt_hdr->l2_offset = offset; + return 0; +} + +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + return buffer_map(&pkt_hdr->buf_hdr, + pkt_hdr->headroom + pkt_hdr->l3_offset, + len, pkt_hdr->frame_len); +} + +uint32_t odp_packet_l3_offset(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->l3_offset; } -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) { - odp_packet_hdr(pkt)->l4_offset = offset; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (offset >= pkt_hdr->frame_len) + return -1; + + pkt_hdr->l3_offset = offset; + return 0; +} + +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + return buffer_map(&pkt_hdr->buf_hdr, + pkt_hdr->headroom + pkt_hdr->l4_offset, + len, pkt_hdr->frame_len); +} + +uint32_t odp_packet_l4_offset(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->l4_offset; } +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (offset >= pkt_hdr->frame_len) + return -1; + + pkt_hdr->l4_offset = offset; + return 0; +} int odp_packet_is_segmented(odp_packet_t pkt) { return odp_packet_hdr(pkt)->buf_hdr.segcount > 1; } - -int odp_packet_seg_count(odp_packet_t pkt) +int odp_packet_num_segs(odp_packet_t pkt) { return odp_packet_hdr(pkt)->buf_hdr.segcount; } +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) +{ + return (odp_packet_seg_t)pkt; +} + +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_buffer_bits_t seghandle; -/** - * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP + seghandle.u32 = (uint32_t)pkt; + seghandle.seg = pkt_hdr->buf_hdr.segcount - 1; + return seghandle.handle; +} + +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + return segment_next(&pkt_hdr->buf_hdr, seg); +} + +/* * - * Internal function: caller is resposible for passing only valid packet handles - * , lengths and offsets (usually done&called in packet input). + * Segment level + * ******************************************************** * - * @param pkt Packet handle - * @param len Packet length in bytes - * @param frame_offset Byte offset to L2 header */ -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t frame_offset) + +void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg) { - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); - odph_ethhdr_t *eth; - odph_vlanhdr_t *vlan; - odph_ipv4hdr_t *ipv4; - odph_ipv6hdr_t *ipv6; - uint16_t ethtype; - size_t offset = 0; - uint8_t ip_proto = 0; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - pkt_hdr->input_flags.eth = 1; - pkt_hdr->l2_offset = frame_offset; - pkt_hdr->frame_len = len; + return segment_map(&pkt_hdr->buf_hdr, seg, NULL, + pkt_hdr->buf_hdr.size, 0); +} - if (len > ODPH_ETH_LEN_MAX) - pkt_hdr->input_flags.jumbo = 1; +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, + odp_packet_seg_t seg ODP_UNUSED) +{ + return odp_packet_hdr(pkt)->buf_hdr.segsize; +} - /* Assume valid L2 header, no CRC/FCS check in SW */ - pkt_hdr->input_flags.l2 = 1; - pkt_hdr->l2_offset = frame_offset; +void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - eth = (odph_ethhdr_t *)odp_packet_data(pkt); - ethtype = odp_be_to_cpu_16(eth->type); - vlan = (odph_vlanhdr_t *)ð->type; + return segment_map(&pkt_hdr->buf_hdr, seg, NULL, + pkt_hdr->frame_len, pkt_hdr->headroom); +} - if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { - pkt_hdr->input_flags.vlan_qinq = 1; - ethtype = odp_be_to_cpu_16(vlan->tpid); - offset += sizeof(odph_vlanhdr_t); - vlan = &vlan[1]; - } +uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint32_t seglen = 0; - if (ethtype == ODPH_ETHTYPE_VLAN) { - pkt_hdr->input_flags.vlan = 1; - ethtype = odp_be_to_cpu_16(vlan->tpid); - offset += sizeof(odph_vlanhdr_t); - } + segment_map(&pkt_hdr->buf_hdr, seg, &seglen, + pkt_hdr->frame_len, pkt_hdr->headroom); - /* Set l3_offset+flag only for known ethtypes */ - switch (ethtype) { - case ODPH_ETHTYPE_IPV4: - pkt_hdr->input_flags.ipv4 = 1; - pkt_hdr->input_flags.l3 = 1; - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + offset; - ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt); - ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset); - break; - case ODPH_ETHTYPE_IPV6: - pkt_hdr->input_flags.ipv6 = 1; - pkt_hdr->input_flags.l3 = 1; - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + offset; - ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt); - ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset); - break; - case ODPH_ETHTYPE_ARP: - pkt_hdr->input_flags.arp = 1; - /* fall through */ - default: - ip_proto = 0; - break; - } + return seglen; +} - switch (ip_proto) { - case ODPH_IPPROTO_UDP: - pkt_hdr->input_flags.udp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - case ODPH_IPPROTO_TCP: - pkt_hdr->input_flags.tcp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - case ODPH_IPPROTO_SCTP: - pkt_hdr->input_flags.sctp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - case ODPH_IPPROTO_ICMP: - pkt_hdr->input_flags.icmp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - default: - /* 0 or unhandled IP protocols, don't set L4 flag+offset */ - if (pkt_hdr->input_flags.ipv6) { - /* IPv6 next_hdr is not L4, mark as IP-option instead */ - pkt_hdr->input_flags.ipopt = 1; +/* + * + * Manipulation + * ******************************************************** + * + */ + +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, + uint32_t len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint32_t pktlen = pkt_hdr->frame_len; + odp_packet_t newpkt; + + if (offset > pktlen) + return ODP_PACKET_INVALID; + + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len); + + if (newpkt != ODP_PACKET_INVALID) { + if (_odp_packet_copy_to_packet(pkt, 0, + newpkt, 0, offset) != 0 || + _odp_packet_copy_to_packet(pkt, offset, newpkt, + offset + len, + pktlen - offset) != 0) { + odp_packet_free(newpkt); + newpkt = ODP_PACKET_INVALID; } - break; } + + return newpkt; } -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, - odph_ipv4hdr_t *ipv4, size_t *offset_out) +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, + uint32_t len) { - uint8_t ihl; - uint16_t frag_offset; - - ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); - if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) { - pkt_hdr->error_flags.ip_err = 1; - return 0; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint32_t pktlen = pkt_hdr->frame_len; + odp_packet_t newpkt; + + if (offset > pktlen || len > offset) + return ODP_PACKET_INVALID; + + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen - len); + + if (newpkt != ODP_PACKET_INVALID) { + if (_odp_packet_copy_to_packet(pkt, 0, + newpkt, 0, offset) != 0 || + _odp_packet_copy_to_packet(pkt, offset + len, + newpkt, offset, + pktlen - offset - len) != 0) { + odp_packet_free(newpkt); + newpkt = ODP_PACKET_INVALID; + } } - if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) { - pkt_hdr->input_flags.ipopt = 1; - return 0; - } + return newpkt; +} - /* A packet is a fragment if: - * "more fragments" flag is set (all fragments except the last) - * OR - * "fragment offset" field is nonzero (all fragments except the first) - */ - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); - if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) { - pkt_hdr->input_flags.ipfrag = 1; - return 0; - } +/* + * + * Copy + * ******************************************************** + * + */ - if (ipv4->proto == ODPH_IPPROTO_ESP || - ipv4->proto == ODPH_IPPROTO_AH) { - pkt_hdr->input_flags.ipsec = 1; - return 0; +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool) +{ + odp_packet_hdr_t *srchdr = odp_packet_hdr(pkt); + uint32_t pktlen = srchdr->frame_len; + uint32_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr); + odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); + + if (newpkt != ODP_PACKET_INVALID) { + odp_packet_hdr_t *newhdr = odp_packet_hdr(newpkt); + uint8_t *newstart, *srcstart; + + /* Must copy meta data first, followed by packet data */ + newstart = (uint8_t *)newhdr + meta_offset; + srcstart = (uint8_t *)srchdr + meta_offset; + + memcpy(newstart, srcstart, + sizeof(odp_packet_hdr_t) - meta_offset); + + if (_odp_packet_copy_to_packet(pkt, 0, + newpkt, 0, pktlen) != 0) { + odp_packet_free(newpkt); + newpkt = ODP_PACKET_INVALID; + } } - /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after return */ - - *offset_out = sizeof(uint32_t) * ihl; - return ipv4->proto; + return newpkt; } -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, - odph_ipv6hdr_t *ipv6, size_t *offset_out) +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, + uint32_t len, void *dst) { - if (ipv6->next_hdr == ODPH_IPPROTO_ESP || - ipv6->next_hdr == ODPH_IPPROTO_AH) { - pkt_hdr->input_flags.ipopt = 1; - pkt_hdr->input_flags.ipsec = 1; - return 0; + void *mapaddr; + uint32_t seglen, cpylen; + uint8_t *dstaddr = (uint8_t *)dst; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + while (len > 0) { + mapaddr = packet_offset_map(pkt_hdr, offset, &seglen); + if (mapaddr == NULL) + return -1; + cpylen = len > seglen ? seglen : len; + memcpy(dstaddr, mapaddr, cpylen); + offset += cpylen; + dstaddr += cpylen; + len -= cpylen; } - if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { - pkt_hdr->input_flags.ipopt = 1; - pkt_hdr->input_flags.ipfrag = 1; - return 0; + return 0; +} + +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, + uint32_t len, const void *src) +{ + void *mapaddr; + uint32_t seglen, cpylen; + const uint8_t *srcaddr = (const uint8_t *)src; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + while (len > 0) { + mapaddr = packet_offset_map(pkt_hdr, offset, &seglen); + if (mapaddr == NULL) + return -1; + cpylen = len > seglen ? seglen : len; + memcpy(mapaddr, srcaddr, cpylen); + offset += cpylen; + srcaddr += cpylen; + len -= cpylen; } - /* Don't step through more extensions */ - *offset_out = ODPH_IPV6HDR_LEN; - return ipv6->next_hdr; + return 0; } +/* + * + * Debugging + * ******************************************************** + * + */ + void odp_packet_print(odp_packet_t pkt) { int max_len = 512; @@ -329,42 +547,338 @@ void odp_packet_print(odp_packet_t pkt) ODP_PRINT("\n%s\n", str); } -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src) +int odp_packet_is_valid(odp_packet_t pkt) { - odp_packet_hdr_t *const pkt_hdr_dst = odp_packet_hdr(pkt_dst); - odp_packet_hdr_t *const pkt_hdr_src = odp_packet_hdr(pkt_src); - const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr); - uint8_t *start_src; - uint8_t *start_dst; - size_t len; + odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt); - if (pkt_dst == ODP_PACKET_INVALID || pkt_src == ODP_PACKET_INVALID) - return -1; - - if (pkt_hdr_dst->buf_hdr.size < pkt_hdr_src->frame_len) - return -1; + return (buf == NULL || buf->type != ODP_BUFFER_TYPE_PACKET) ? 0 : 1; +} - /* Copy packet header */ - start_dst = (uint8_t *)pkt_hdr_dst + start_offset; - start_src = (uint8_t *)pkt_hdr_src + start_offset; - len = sizeof(odp_packet_hdr_t) - start_offset; - memcpy(start_dst, start_src, len); +/* + * + * Internal Use Routines + * ******************************************************** + * + */ - /* Copy frame payload */ - start_dst = (uint8_t *)odp_packet_data(pkt_dst); - start_src = (uint8_t *)odp_packet_data(pkt_src); - len = pkt_hdr_src->frame_len; - memcpy(start_dst, start_src, len); +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, + odp_packet_t dstpkt, uint32_t dstoffset, + uint32_t len) +{ + odp_packet_hdr_t *srchdr = odp_packet_hdr(srcpkt); + odp_packet_hdr_t *dsthdr = odp_packet_hdr(dstpkt); + void *srcmap; + void *dstmap; + uint32_t cpylen, minseg, srcseglen = 0, dstseglen; + + while (len > 0) { + srcmap = packet_offset_map(srchdr, srcoffset, &srcseglen); + dstmap = packet_offset_map(dsthdr, dstoffset, &dstseglen); + if (srcmap == NULL || dstmap == NULL) + return -1; + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; + cpylen = len > minseg ? minseg : len; + memcpy(dstmap, srcmap, cpylen); + srcoffset += cpylen; + dstoffset += cpylen; + len -= cpylen; + } return 0; } -void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx) +odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl) +{ + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); + + if (pool->s.params.buf_type != ODP_BUFFER_TYPE_PACKET) + return ODP_PACKET_INVALID; + + return (odp_packet_t)buffer_alloc(pool_hdl, + pool->s.params.buf_size); +} + +/** + * Parser helper function for IPv4 + */ +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) +{ + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); + uint16_t frag_offset; + + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); + + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || + odp_unlikely(ver != 4) || + (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) { + pkt_hdr->error_flags.ip_err = 1; + return 0; + } + + *offset += ihl * 4; + *parseptr += ihl * 4; + + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) + pkt_hdr->input_flags.ipopt = 1; + + /* A packet is a fragment if: + * "more fragments" flag is set (all fragments except the last) + * OR + * "fragment offset" field is nonzero (all fragments except the first) + */ + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) + pkt_hdr->input_flags.ipfrag = 1; + + return ipv4->proto; +} + +/** + * Parser helper function for IPv6 + */ +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) +{ + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; + odph_ipv6hdr_ext_t *ipv6ext; + + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); + + /* Basic sanity checks on IPv6 header */ + if ((ipv6->ver_tc_flow >> 28) != 6 || + pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) { + pkt_hdr->error_flags.ip_err = 1; + return 0; + } + + /* Skip past IPv6 header */ + *offset += sizeof(odph_ipv6hdr_t); + *parseptr += sizeof(odph_ipv6hdr_t); + + + /* Skip past any IPv6 extension headers */ + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { + pkt_hdr->input_flags.ipopt = 1; + + do { + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; + uint16_t extlen = 8 + ipv6ext->ext_len * 8; + + *offset += extlen; + *parseptr += extlen; + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && + *offset < pkt_hdr->frame_len); + + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) { + pkt_hdr->error_flags.ip_err = 1; + return 0; + } + + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) + pkt_hdr->input_flags.ipfrag = 1; + + return ipv6ext->next_hdr; + } + + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { + pkt_hdr->input_flags.ipopt = 1; + pkt_hdr->input_flags.ipfrag = 1; + } + + return ipv6->next_hdr; +} + +/** + * Parser helper function for TCP + */ +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) { - odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx; + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; + + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) + pkt_hdr->error_flags.tcp_err = 1; + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) + pkt_hdr->input_flags.tcpopt = 1; + + pkt_hdr->l4_len = pkt_hdr->l3_len + + pkt_hdr->l3_offset - pkt_hdr->l4_offset; + + *offset += (uint32_t)tcp->hl * 4; + *parseptr += (uint32_t)tcp->hl * 4; } -void *odp_packet_get_ctx(odp_packet_t pkt) +/** + * Parser helper function for UDP + */ +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) { - return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx; + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; + uint32_t udplen = odp_be_to_cpu_16(udp->length); + + if (udplen < sizeof(odph_udphdr_t) || + udplen > (pkt_hdr->l3_len + + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { + pkt_hdr->error_flags.udp_err = 1; + } + + pkt_hdr->l4_len = udplen; + + *offset += sizeof(odph_udphdr_t); + *parseptr += sizeof(odph_udphdr_t); +} + +/** + * Simple packet parser + */ + +int _odp_packet_parse(odp_packet_t pkt) +{ + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); + odph_ethhdr_t *eth; + odph_vlanhdr_t *vlan; + uint16_t ethtype; + uint8_t *parseptr; + uint32_t offset, seglen; + uint8_t ip_proto = 0; + + /* Reset parser metadata for new parse */ + pkt_hdr->error_flags.all = 0; + pkt_hdr->input_flags.all = 0; + pkt_hdr->output_flags.all = 0; + pkt_hdr->l2_offset = 0; + pkt_hdr->l3_offset = 0; + pkt_hdr->l4_offset = 0; + pkt_hdr->payload_offset = 0; + pkt_hdr->vlan_s_tag = 0; + pkt_hdr->vlan_c_tag = 0; + pkt_hdr->l3_protocol = 0; + pkt_hdr->l4_protocol = 0; + + /* We only support Ethernet for now */ + pkt_hdr->input_flags.eth = 1; + + /* Detect jumbo frames */ + if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX) + pkt_hdr->input_flags.jumbo = 1; + + /* Assume valid L2 header, no CRC/FCS check in SW */ + pkt_hdr->input_flags.l2 = 1; + + eth = (odph_ethhdr_t *)packet_offset_map(pkt_hdr, 0, &seglen); + offset = sizeof(odph_ethhdr_t); + parseptr = (uint8_t *)ð->type; + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + + /* Parse the VLAN header(s), if present */ + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { + pkt_hdr->input_flags.vlan_qinq = 1; + pkt_hdr->input_flags.vlan = 1; + vlan = (odph_vlanhdr_t *)(void *)parseptr; + pkt_hdr->vlan_s_tag = ((ethtype << 16) | + odp_be_to_cpu_16(vlan->tci)); + offset += sizeof(odph_vlanhdr_t); + parseptr += sizeof(odph_vlanhdr_t); + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + if (ethtype == ODPH_ETHTYPE_VLAN) { + pkt_hdr->input_flags.vlan = 1; + vlan = (odph_vlanhdr_t *)(void *)parseptr; + pkt_hdr->vlan_c_tag = ((ethtype << 16) | + odp_be_to_cpu_16(vlan->tci)); + offset += sizeof(odph_vlanhdr_t); + parseptr += sizeof(odph_vlanhdr_t); + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + /* Check for SNAP vs. DIX */ + if (ethtype < ODPH_ETH_LEN_MAX) { + pkt_hdr->input_flags.snap = 1; + if (ethtype > pkt_hdr->frame_len - offset) { + pkt_hdr->error_flags.snap_len = 1; + goto parse_exit; + } + offset += 8; + parseptr += 8; + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + /* Consume Ethertype for Layer 3 parse */ + parseptr += 2; + + /* Set l3_offset+flag only for known ethtypes */ + pkt_hdr->input_flags.l3 = 1; + pkt_hdr->l3_offset = offset; + pkt_hdr->l3_protocol = ethtype; + + /* Parse Layer 3 headers */ + switch (ethtype) { + case ODPH_ETHTYPE_IPV4: + pkt_hdr->input_flags.ipv4 = 1; + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_ETHTYPE_IPV6: + pkt_hdr->input_flags.ipv6 = 1; + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_ETHTYPE_ARP: + pkt_hdr->input_flags.arp = 1; + ip_proto = 255; /* Reserved invalid by IANA */ + break; + + default: + pkt_hdr->input_flags.l3 = 0; + ip_proto = 255; /* Reserved invalid by IANA */ + } + + /* Set l4_offset+flag only for known ip_proto */ + pkt_hdr->input_flags.l4 = 1; + pkt_hdr->l4_offset = offset; + pkt_hdr->l4_protocol = ip_proto; + + /* Parse Layer 4 headers */ + switch (ip_proto) { + case ODPH_IPPROTO_ICMP: + pkt_hdr->input_flags.icmp = 1; + break; + + case ODPH_IPPROTO_TCP: + pkt_hdr->input_flags.tcp = 1; + parse_tcp(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_IPPROTO_UDP: + pkt_hdr->input_flags.udp = 1; + parse_udp(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_IPPROTO_AH: + case ODPH_IPPROTO_ESP: + pkt_hdr->input_flags.ipsec = 1; + break; + + default: + pkt_hdr->input_flags.l4 = 0; + break; + } + + /* + * Anything beyond what we parse here is considered payload. + * Note: Payload is really only relevant for TCP and UDP. For + * all other protocols, the payload offset will point to the + * final header (ARP, ICMP, AH, ESP, or IP Fragment). + */ + pkt_hdr->payload_offset = offset; + +parse_exit: + return pkt_hdr->error_flags.all != 0; } diff --git a/platform/linux-generic/odp_packet_socket.c b/platform/linux-generic/odp_packet_socket.c index f96d5bf..05b40f0 100644 --- a/platform/linux-generic/odp_packet_socket.c +++ b/platform/linux-generic/odp_packet_socket.c @@ -34,6 +34,7 @@ #include <errno.h> #include <sys/syscall.h> +#include <odp.h> #include <odp_packet_socket.h> #include <odp_packet_internal.h> #include <odp_align_internal.h> @@ -42,7 +43,6 @@ #include <odph_eth.h> #include <odph_ip.h> -#include <odph_packet.h> /** Provide a sendmmsg wrapper for systems with no libc or kernel support. * As it is implemented as a weak symbol, it has zero effect on systems @@ -206,28 +206,19 @@ int setup_pkt_sock(pkt_sock_t *const pkt_sock, const char *netdev, unsigned int if_idx; struct ifreq ethreq; struct sockaddr_ll sa_ll; - odp_packet_t pkt; - uint8_t *pkt_buf; - uint8_t *l2_hdr; if (pool == ODP_BUFFER_POOL_INVALID) return -1; pkt_sock->pool = pool; - pkt = odph_packet_alloc(pool); - if (!odph_packet_is_valid(pkt)) - return -1; - - pkt_buf = odp_packet_addr(pkt); - l2_hdr = ETHBUF_ALIGN(pkt_buf); /* Store eth buffer offset for pkt buffers from this pool */ - pkt_sock->frame_offset = (uintptr_t)l2_hdr - (uintptr_t)pkt_buf; + pkt_sock->frame_offset = 0; /* pkt buffer size */ - pkt_sock->buf_size = odph_packet_buf_size(pkt); + pkt_sock->buf_size = odp_buffer_pool_segment_size(pool); /* max frame len taking into account the l2-offset */ - pkt_sock->max_frame_len = pkt_sock->buf_size - pkt_sock->frame_offset; - - odph_packet_free(pkt); + pkt_sock->max_frame_len = pkt_sock->buf_size - + odp_buffer_pool_headroom(pool) - + odp_buffer_pool_tailroom(pool); odp_spinlock_lock(&raw_sockets_lock); @@ -318,7 +309,6 @@ int recv_pkt_sock_basic(pkt_sock_t *const pkt_sock, int const sockfd = pkt_sock->sockfd; odp_packet_t pkt = ODP_PACKET_INVALID; uint8_t *pkt_buf; - uint8_t *l2_hdr; int nb_rx = 0; /* recvfrom: @@ -331,15 +321,14 @@ int recv_pkt_sock_basic(pkt_sock_t *const pkt_sock, for (i = 0; i < len; i++) { if (odp_likely(pkt == ODP_PACKET_INVALID)) { - pkt = odph_packet_alloc(pkt_sock->pool); + pkt = _odp_packet_alloc(pkt_sock->pool); if (odp_unlikely(pkt == ODP_PACKET_INVALID)) break; } - pkt_buf = odp_packet_addr(pkt); - l2_hdr = pkt_buf + pkt_sock->frame_offset; + pkt_buf = odp_packet_data(pkt); - recv_bytes = recvfrom(sockfd, l2_hdr, + recv_bytes = recvfrom(sockfd, pkt_buf, pkt_sock->max_frame_len, MSG_DONTWAIT, (struct sockaddr *)&sll, &addrlen); /* no data or error: free recv buf and break out of loop */ @@ -350,7 +339,8 @@ int recv_pkt_sock_basic(pkt_sock_t *const pkt_sock, continue; /* Parse and set packet header data */ - odp_packet_parse(pkt, recv_bytes, pkt_sock->frame_offset); + packet_set_len(pkt, recv_bytes); + _odp_packet_parse(pkt); pkt_table[nb_rx] = pkt; pkt = ODP_PACKET_INVALID; @@ -358,7 +348,7 @@ int recv_pkt_sock_basic(pkt_sock_t *const pkt_sock, } /* end for() */ if (odp_unlikely(pkt != ODP_PACKET_INVALID)) - odph_packet_free(pkt); + odp_packet_free(pkt); return nb_rx; } @@ -371,7 +361,7 @@ int send_pkt_sock_basic(pkt_sock_t *const pkt_sock, { odp_packet_t pkt; uint8_t *frame; - size_t frame_len; + uint32_t frame_len; unsigned i; unsigned flags; int sockfd; @@ -384,8 +374,7 @@ int send_pkt_sock_basic(pkt_sock_t *const pkt_sock, while (i < len) { pkt = pkt_table[i]; - frame = odp_packet_l2(pkt); - frame_len = odp_packet_get_len(pkt); + frame = odp_packet_l2_ptr(pkt, &frame_len); ret = send(sockfd, frame, frame_len, flags); if (odp_unlikely(ret == -1)) { @@ -402,7 +391,7 @@ int send_pkt_sock_basic(pkt_sock_t *const pkt_sock, nb_tx = i; for (i = 0; i < len; i++) - odph_packet_free(pkt_table[i]); + odp_packet_free(pkt_table[i]); return nb_tx; } @@ -429,11 +418,11 @@ int recv_pkt_sock_mmsg(pkt_sock_t *const pkt_sock, memset(msgvec, 0, sizeof(msgvec)); for (i = 0; i < (int)len; i++) { - pkt_table[i] = odph_packet_alloc(pkt_sock->pool); + pkt_table[i] = _odp_packet_alloc(pkt_sock->pool); if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID)) break; - pkt_buf = odp_packet_addr(pkt_table[i]); + pkt_buf = odp_packet_data(pkt_table[i]); l2_hdr = pkt_buf + pkt_sock->frame_offset; iovecs[i].iov_base = l2_hdr; iovecs[i].iov_len = pkt_sock->max_frame_len; @@ -451,13 +440,13 @@ int recv_pkt_sock_mmsg(pkt_sock_t *const pkt_sock, /* Don't receive packets sent by ourselves */ if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, eth_hdr->h_source))) { - odph_packet_free(pkt_table[i]); + odp_packet_free(pkt_table[i]); continue; } /* Parse and set packet header data */ - odp_packet_parse(pkt_table[i], msgvec[i].msg_len, - pkt_sock->frame_offset); + packet_set_len(pkt_table[i], msgvec[i].msg_len); + _odp_packet_parse(pkt_table[i]); pkt_table[nb_rx] = pkt_table[i]; nb_rx++; @@ -465,7 +454,7 @@ int recv_pkt_sock_mmsg(pkt_sock_t *const pkt_sock, /* Free unused pkt buffers */ for (; i < msgvec_len; i++) - odph_packet_free(pkt_table[i]); + odp_packet_free(pkt_table[i]); return nb_rx; } @@ -491,10 +480,9 @@ int send_pkt_sock_mmsg(pkt_sock_t *const pkt_sock, memset(msgvec, 0, sizeof(msgvec)); for (i = 0; i < len; i++) { - uint8_t *const frame = odp_packet_l2(pkt_table[i]); - const size_t frame_len = odp_packet_get_len(pkt_table[i]); - iovecs[i].iov_base = frame; - iovecs[i].iov_len = frame_len; + uint32_t len; + iovecs[i].iov_base = odp_packet_l2_ptr(pkt_table[i], &len); + iovecs[i].iov_len = len; msgvec[i].msg_hdr.msg_iov = &iovecs[i]; msgvec[i].msg_hdr.msg_iovlen = 1; } @@ -507,7 +495,7 @@ int send_pkt_sock_mmsg(pkt_sock_t *const pkt_sock, } for (i = 0; i < len; i++) - odph_packet_free(pkt_table[i]); + odp_packet_free(pkt_table[i]); return len; } @@ -571,7 +559,6 @@ static inline void mmap_tx_user_ready(struct tpacket2_hdr *hdr) static inline unsigned pkt_mmap_v2_rx(int sock, struct ring *ring, odp_packet_t pkt_table[], unsigned len, odp_buffer_pool_t pool, - size_t frame_offset, unsigned char if_mac[]) { union frame_map ppd; @@ -604,18 +591,18 @@ static inline unsigned pkt_mmap_v2_rx(int sock, struct ring *ring, continue; } - pkt_table[i] = odph_packet_alloc(pool); + pkt_table[i] = _odp_packet_alloc(pool); if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID)) break; - l2_hdr = odp_packet_addr(pkt_table[i]) - + frame_offset; + packet_set_len(pkt_table[i], pkt_len); + l2_hdr = odp_packet_data(pkt_table[i]); memcpy(l2_hdr, pkt_buf, pkt_len); mmap_rx_user_ready(ppd.raw); /* Parse and set packet header data */ - odp_packet_parse(pkt_table[i], pkt_len, frame_offset); + _odp_packet_parse(pkt_table[i]); frame_num = next_frame_num; i++; @@ -634,7 +621,7 @@ static inline unsigned pkt_mmap_v2_tx(int sock, struct ring *ring, { union frame_map ppd; uint8_t *pkt_buf; - size_t pkt_len; + uint32_t pkt_len; unsigned frame_num, next_frame_num; int ret; unsigned i = 0; @@ -647,8 +634,7 @@ static inline unsigned pkt_mmap_v2_tx(int sock, struct ring *ring, next_frame_num = (frame_num + 1) % ring->rd_num; - pkt_buf = odp_packet_l2(pkt_table[i]); - pkt_len = odp_packet_get_len(pkt_table[i]); + pkt_buf = odp_packet_l2_ptr(pkt_table[i], &pkt_len); ppd.v2->tp_h.tp_snaplen = pkt_len; ppd.v2->tp_h.tp_len = pkt_len; @@ -658,7 +644,7 @@ static inline unsigned pkt_mmap_v2_tx(int sock, struct ring *ring, mmap_tx_user_ready(ppd.raw); - odph_packet_free(pkt_table[i]); + odp_packet_free(pkt_table[i]); frame_num = next_frame_num; i++; } else { @@ -839,9 +825,6 @@ static int mmap_store_hw_addr(pkt_sock_mmap_t *const pkt_sock, int setup_pkt_sock_mmap(pkt_sock_mmap_t *const pkt_sock, const char *netdev, odp_buffer_pool_t pool, int fanout) { - odp_packet_t pkt; - uint8_t *pkt_buf; - uint8_t *l2_hdr; int if_idx; int ret = 0; @@ -850,16 +833,8 @@ int setup_pkt_sock_mmap(pkt_sock_mmap_t *const pkt_sock, const char *netdev, if (pool == ODP_BUFFER_POOL_INVALID) return -1; - pkt = odph_packet_alloc(pool); - if (!odph_packet_is_valid(pkt)) - return -1; - - pkt_buf = odp_packet_addr(pkt); - l2_hdr = ETHBUF_ALIGN(pkt_buf); /* Store eth buffer offset for pkt buffers from this pool */ - pkt_sock->frame_offset = (uintptr_t)l2_hdr - (uintptr_t)pkt_buf; - - odph_packet_free(pkt); + pkt_sock->frame_offset = 0; pkt_sock->pool = pool; pkt_sock->sockfd = mmap_pkt_socket(); @@ -926,7 +901,7 @@ int recv_pkt_sock_mmap(pkt_sock_mmap_t *const pkt_sock, { return pkt_mmap_v2_rx(pkt_sock->rx_ring.sock, &pkt_sock->rx_ring, pkt_table, len, pkt_sock->pool, - pkt_sock->frame_offset, pkt_sock->if_mac); + pkt_sock->if_mac); } /* -- 1.8.3.2 _______________________________________________ lng-odp mailing list lng-odp@lists.linaro.org http://lists.linaro.org/mailman/listinfo/lng-odp