Added support for multi-segmented packets. The first segments
is the packet descriptor, which contains all metadata and
pointers to other segments.
Signed-off-by: Petri Savolainen <petri.savolai...@nokia.com>
---
.../include/odp/api/plat/packet_types.h | 6 +-
.../linux-generic/include/odp_buffer_inlines.h | 11 -
.../linux-generic/include/odp_buffer_internal.h | 23 +-
.../linux-generic/include/odp_config_internal.h | 39 +-
.../linux-generic/include/odp_packet_internal.h | 80 +--
platform/linux-generic/include/odp_pool_internal.h | 3 -
platform/linux-generic/odp_buffer.c | 8 +-
platform/linux-generic/odp_crypto.c | 8 +-
platform/linux-generic/odp_packet.c | 712 +++++++++++++++++----
platform/linux-generic/odp_pool.c | 123 ++--
platform/linux-generic/pktio/netmap.c | 4 +-
platform/linux-generic/pktio/socket.c | 3 +-
12 files changed, 692 insertions(+), 328 deletions(-)
diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h
b/platform/linux-generic/include/odp/api/plat/packet_types.h
index b5345ed..864494d 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_types.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_types.h
@@ -32,9 +32,11 @@ typedef ODP_HANDLE_T(odp_packet_t);
#define ODP_PACKET_OFFSET_INVALID (0x0fffffff)
-typedef ODP_HANDLE_T(odp_packet_seg_t);
+/* A packet segment handle stores a small index. Strong type handles are
+ * pointers, which would be wasteful in this case. */
+typedef uint8_t odp_packet_seg_t;
-#define ODP_PACKET_SEG_INVALID _odp_cast_scalar(odp_packet_seg_t, 0xffffffff)
+#define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)-1)
/** odp_packet_color_t assigns names to the various pkt "colors" */
typedef enum {
diff --git a/platform/linux-generic/include/odp_buffer_inlines.h
b/platform/linux-generic/include/odp_buffer_inlines.h
index f8688f6..cf817d9 100644
--- a/platform/linux-generic/include/odp_buffer_inlines.h
+++ b/platform/linux-generic/include/odp_buffer_inlines.h
@@ -23,22 +23,11 @@ odp_event_type_t _odp_buffer_event_type(odp_buffer_t buf);
void _odp_buffer_event_type_set(odp_buffer_t buf, int ev);
int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf);
-void *buffer_map(odp_buffer_hdr_t *buf, uint32_t offset, uint32_t *seglen,
- uint32_t limit);
-
static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr)
{
return hdr->handle.handle;
}
-static inline uint32_t pool_id_from_buf(odp_buffer_t buf)
-{
- odp_buffer_bits_t handle;
-
- handle.handle = buf;
- return handle.pool_id;
-}
-
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include/odp_buffer_internal.h
b/platform/linux-generic/include/odp_buffer_internal.h
index 0ca13f8..4e75908 100644
--- a/platform/linux-generic/include/odp_buffer_internal.h
+++ b/platform/linux-generic/include/odp_buffer_internal.h
@@ -33,10 +33,6 @@ extern "C" {
#include <odp_schedule_if.h>
#include <stddef.h>
-ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_SEG_LEN_MIN >= 256,
- "ODP Segment size must be a minimum of 256 bytes");
-
-
typedef union odp_buffer_bits_t {
odp_buffer_t handle;
@@ -65,6 +61,20 @@ struct odp_buffer_hdr_t {
int burst_first;
struct odp_buffer_hdr_t *burst[BUFFER_BURST_SIZE];
+ struct {
+ void *hdr;
+ uint8_t *data;
+ uint32_t len;
+ } seg[CONFIG_PACKET_MAX_SEGS];
+
+ /* max data size */
+ uint32_t size;
+
+ /* Initial buffer data pointer and length */
+ void *base_data;
+ uint32_t base_len;
+ uint8_t *buf_end;
+
union {
uint32_t all;
struct {
@@ -75,7 +85,6 @@ struct odp_buffer_hdr_t {
int8_t type; /* buffer type */
odp_event_type_t event_type; /* for reuse as event */
- uint32_t size; /* max data size */
odp_pool_t pool_hdl; /* buffer pool handle */
union {
uint64_t buf_u64; /* user u64 */
@@ -86,8 +95,6 @@ struct odp_buffer_hdr_t {
uint32_t uarea_size; /* size of user area */
uint32_t segcount; /* segment count */
uint32_t segsize; /* segment size */
- /* block addrs */
- void *addr[ODP_CONFIG_PACKET_MAX_SEGS];
uint64_t order; /* sequence for ordered queues */
queue_entry_t *origin_qe; /* ordered queue origin */
union {
@@ -105,8 +112,6 @@ struct odp_buffer_hdr_t {
};
/* Forward declarations */
-int seg_alloc_head(odp_buffer_hdr_t *buf_hdr, int segcount);
-void seg_free_head(odp_buffer_hdr_t *buf_hdr, int segcount);
int seg_alloc_tail(odp_buffer_hdr_t *buf_hdr, int segcount);
void seg_free_tail(odp_buffer_hdr_t *buf_hdr, int segcount);
diff --git a/platform/linux-generic/include/odp_config_internal.h
b/platform/linux-generic/include/odp_config_internal.h
index e24d5ab..9a4e6eb 100644
--- a/platform/linux-generic/include/odp_config_internal.h
+++ b/platform/linux-generic/include/odp_config_internal.h
@@ -54,7 +54,7 @@ extern "C" {
* The default value (66) allows a 1500-byte packet to be received into a
single
* segment with Ethernet offset alignment and room for some header expansion.
*/
-#define ODP_CONFIG_PACKET_HEADROOM 66
+#define CONFIG_PACKET_HEADROOM 66
/*
* Default packet tailroom
@@ -65,21 +65,26 @@ extern "C" {
* without restriction. Note that most implementations will automatically
* consider any unused portion of the last segment of a packet as tailroom
*/
-#define ODP_CONFIG_PACKET_TAILROOM 0
+#define CONFIG_PACKET_TAILROOM 0
/*
* Maximum number of segments per packet
*/
-#define ODP_CONFIG_PACKET_MAX_SEGS 1
+#define CONFIG_PACKET_MAX_SEGS 1
/*
- * Maximum packet segment length
- *
- * This defines the maximum packet segment buffer length in bytes. The user
- * defined segment length (seg_len in odp_pool_param_t) must not be larger than
- * this.
+ * Maximum packet segment size including head- and tailrooms
*/
-#define ODP_CONFIG_PACKET_SEG_LEN_MAX (64 * 1024)
+#define CONFIG_PACKET_SEG_SIZE (64 * 1024)
+
+/* Maximum data length in a segment
+ *
+ * The user defined segment length (seg_len in odp_pool_param_t) must not
+ * be larger than this.
+*/
+#define CONFIG_PACKET_MAX_SEG_LEN (CONFIG_PACKET_SEG_SIZE - \
+ CONFIG_PACKET_HEADROOM - \
+ CONFIG_PACKET_TAILROOM)
/*
* Minimum packet segment length
@@ -88,21 +93,7 @@ extern "C" {
* defined segment length (seg_len in odp_pool_param_t) will be rounded up into
* this value.
*/
-#define ODP_CONFIG_PACKET_SEG_LEN_MIN ODP_CONFIG_PACKET_SEG_LEN_MAX
-
-/*
- * Maximum packet buffer length
- *
- * This defines the maximum number of bytes that can be stored into a packet
- * (maximum return value of odp_packet_buf_len(void)). Attempts to allocate
- * (including default head- and tailrooms) or extend packets to sizes larger
- * than this limit will fail.
- *
- * @internal In odp-linux implementation:
- * - The value MUST be an integral number of segments
- * - The value SHOULD be large enough to accommodate jumbo packets (9K)
- */
-#define ODP_CONFIG_PACKET_BUF_LEN_MAX ODP_CONFIG_PACKET_SEG_LEN_MAX
+#define CONFIG_PACKET_SEG_LEN_MIN CONFIG_PACKET_MAX_SEG_LEN
/* Maximum number of shared memory blocks.
*
diff --git a/platform/linux-generic/include/odp_packet_internal.h
b/platform/linux-generic/include/odp_packet_internal.h
index 0cdd5ca..d09231e 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -27,8 +27,6 @@ extern "C" {
#include <odp/api/crypto.h>
#include <odp_crypto_internal.h>
-#define PACKET_JUMBO_LEN (9 * 1024)
-
/** Minimum segment length expected by packet_parse_common() */
#define PACKET_PARSE_SEG_LEN 96
@@ -218,85 +216,13 @@ static inline void
copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr,
dst_hdr->op_result = src_hdr->op_result;
}
-static inline void *packet_map(odp_packet_hdr_t *pkt_hdr,
- uint32_t offset, uint32_t *seglen)
-{
- if (offset > pkt_hdr->frame_len)
- return NULL;
-
- return buffer_map(&pkt_hdr->buf_hdr,
- pkt_hdr->headroom + offset, seglen,
- pkt_hdr->headroom + pkt_hdr->frame_len);
-}
-
-static inline void push_head(odp_packet_hdr_t *pkt_hdr, size_t len)
-{
- pkt_hdr->headroom -= len;
- pkt_hdr->frame_len += len;
-}
-
-static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len)
-{
- pkt_hdr->headroom += len;
- pkt_hdr->frame_len -= len;
-}
-
-static inline int push_head_seg(odp_packet_hdr_t *pkt_hdr, size_t len)
-{
- uint32_t extrasegs =
- (len - pkt_hdr->headroom + pkt_hdr->buf_hdr.segsize - 1) /
- pkt_hdr->buf_hdr.segsize;
-
- if (pkt_hdr->buf_hdr.segcount + extrasegs >
- ODP_CONFIG_PACKET_MAX_SEGS ||
- seg_alloc_head(&pkt_hdr->buf_hdr, extrasegs))
- return -1;
-
- pkt_hdr->headroom += extrasegs * pkt_hdr->buf_hdr.segsize;
- return 0;
-}
-
-static inline void pull_head_seg(odp_packet_hdr_t *pkt_hdr)
-{
- uint32_t extrasegs = (pkt_hdr->headroom - 1) / pkt_hdr->buf_hdr.segsize;
-
- seg_free_head(&pkt_hdr->buf_hdr, extrasegs);
- pkt_hdr->headroom -= extrasegs * pkt_hdr->buf_hdr.segsize;
-}
-
-static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len)
-{
- pkt_hdr->tailroom -= len;
- pkt_hdr->frame_len += len;
-}
-
-static inline int push_tail_seg(odp_packet_hdr_t *pkt_hdr, size_t len)
+static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len)
{
- uint32_t extrasegs =
- (len - pkt_hdr->tailroom + pkt_hdr->buf_hdr.segsize - 1) /
- pkt_hdr->buf_hdr.segsize;
+ int last = pkt_hdr->buf_hdr.segcount - 1;
- if (pkt_hdr->buf_hdr.segcount + extrasegs >
- ODP_CONFIG_PACKET_MAX_SEGS ||
- seg_alloc_tail(&pkt_hdr->buf_hdr, extrasegs))
- return -1;
-
- pkt_hdr->tailroom += extrasegs * pkt_hdr->buf_hdr.segsize;
- return 0;
-}
-
-static inline void pull_tail_seg(odp_packet_hdr_t *pkt_hdr)
-{
- uint32_t extrasegs = pkt_hdr->tailroom / pkt_hdr->buf_hdr.segsize;
-
- seg_free_tail(&pkt_hdr->buf_hdr, extrasegs);
- pkt_hdr->tailroom -= extrasegs * pkt_hdr->buf_hdr.segsize;
-}
-
-static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len)
-{
pkt_hdr->tailroom += len;
pkt_hdr->frame_len -= len;
+ pkt_hdr->buf_hdr.seg[last].len -= len;
}
static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr)
diff --git a/platform/linux-generic/include/odp_pool_internal.h
b/platform/linux-generic/include/odp_pool_internal.h
index f7e951a..5d7b817 100644
--- a/platform/linux-generic/include/odp_pool_internal.h
+++ b/platform/linux-generic/include/odp_pool_internal.h
@@ -113,9 +113,6 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[],
odp_buffer_hdr_t *buf_hdr[], int num);
void buffer_free_multi(const odp_buffer_t buf[], int num_free);
-uint32_t pool_headroom(odp_pool_t pool);
-uint32_t pool_tailroom(odp_pool_t pool);
-
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/odp_buffer.c
b/platform/linux-generic/odp_buffer.c
index eed15c0..b791039 100644
--- a/platform/linux-generic/odp_buffer.c
+++ b/platform/linux-generic/odp_buffer.c
@@ -28,7 +28,7 @@ void *odp_buffer_addr(odp_buffer_t buf)
{
odp_buffer_hdr_t *hdr = buf_hdl_to_hdr(buf);
- return hdr->addr[0];
+ return hdr->seg[0].data;
}
uint32_t odp_buffer_size(odp_buffer_t buf)
@@ -56,11 +56,11 @@ int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t
buf)
" pool %" PRIu64 "\n",
odp_pool_to_u64(hdr->pool_hdl));
len += snprintf(&str[len], n-len,
- " addr %p\n", hdr->addr);
+ " addr %p\n", hdr->seg[0].data);
len += snprintf(&str[len], n-len,
- " size %" PRIu32 "\n", hdr->size);
+ " size %" PRIu32 "\n", hdr->size);
len += snprintf(&str[len], n-len,
- " type %i\n", hdr->type);
+ " type %i\n", hdr->type);
return len;
}
diff --git a/platform/linux-generic/odp_crypto.c
b/platform/linux-generic/odp_crypto.c
index 3ebabb7..7e686ff 100644
--- a/platform/linux-generic/odp_crypto.c
+++ b/platform/linux-generic/odp_crypto.c
@@ -754,9 +754,13 @@ odp_crypto_operation(odp_crypto_op_params_t *params,
ODP_POOL_INVALID != session->output_pool)
params->out_pkt = odp_packet_alloc(session->output_pool,
odp_packet_len(params->pkt));
+
+ if (odp_unlikely(ODP_PACKET_INVALID == params->out_pkt)) {
+ ODP_DBG("Alloc failed.\n");
+ return -1;
+ }
+
if (params->pkt != params->out_pkt) {
- if (odp_unlikely(ODP_PACKET_INVALID == params->out_pkt))
- ODP_ABORT();
(void)odp_packet_copy_from_pkt(params->out_pkt,
0,
params->pkt,
diff --git a/platform/linux-generic/odp_packet.c
b/platform/linux-generic/odp_packet.c
index 2eee775..a5c6ff4 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -20,12 +20,155 @@
#include <stdio.h>
#include <inttypes.h>
-/*
- *
- * Alloc and free
- * ********************************************************
- *
- */
+static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr)
+{
+ return (odp_packet_t)pkt_hdr->buf_hdr.handle.handle;
+}
+
+static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr)
+{
+ return pkt_hdr->buf_hdr.handle.handle;
+}
+
+static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr,
+ uint32_t seg_idx)
+{
+ return pkt_hdr->buf_hdr.seg[seg_idx].len;
+}
+
+static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t
seg_idx)
+{
+ return pkt_hdr->buf_hdr.seg[seg_idx].data;
+}
+
+static inline int packet_last_seg(odp_packet_hdr_t *pkt_hdr)
+{
+ if (CONFIG_PACKET_MAX_SEGS == 1)
+ return 0;
+ else
+ return pkt_hdr->buf_hdr.segcount - 1;
+}
+
+static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr)
+{
+ return packet_seg_len(pkt_hdr, 0);
+}
+
+static inline uint32_t packet_last_seg_len(odp_packet_hdr_t *pkt_hdr)
+{
+ int last = packet_last_seg(pkt_hdr);
+
+ return packet_seg_len(pkt_hdr, last);
+}
+
+static inline void *packet_data(odp_packet_hdr_t *pkt_hdr)
+{
+ return pkt_hdr->buf_hdr.seg[0].data;
+}
+
+static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr)
+{
+ int last = packet_last_seg(pkt_hdr);
+ uint32_t seg_len = pkt_hdr->buf_hdr.seg[last].len;
+
+ return pkt_hdr->buf_hdr.seg[last].data + seg_len;
+}
+
+static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len)
+{
+ pkt_hdr->headroom -= len;
+ pkt_hdr->frame_len += len;
+ pkt_hdr->buf_hdr.seg[0].data -= len;
+ pkt_hdr->buf_hdr.seg[0].len += len;
+}
+
+static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len)
+{
+ pkt_hdr->headroom += len;
+ pkt_hdr->frame_len -= len;
+ pkt_hdr->buf_hdr.seg[0].data += len;
+ pkt_hdr->buf_hdr.seg[0].len -= len;
+}
+
+static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len)
+{
+ int last = packet_last_seg(pkt_hdr);
+
+ pkt_hdr->tailroom -= len;
+ pkt_hdr->frame_len += len;
+ pkt_hdr->buf_hdr.seg[last].len += len;
+}
+
+/* Copy all metadata for segmentation modification. Segment data and lengths
+ * are not copied. */
+static inline void packet_seg_copy_md(odp_packet_hdr_t *dst,
+ odp_packet_hdr_t *src)
+{
+ dst->p = src->p;
+
+ /* lengths are not copied:
+ * .frame_len
+ * .headroom
+ * .tailroom
+ */
+
+ dst->input = src->input;
+ dst->dst_queue = src->dst_queue;
+ dst->flow_hash = src->flow_hash;
+ dst->timestamp = src->timestamp;
+ dst->op_result = src->op_result;
+
+ /* buffer header side packet metadata */
+ dst->buf_hdr.buf_u64 = src->buf_hdr.buf_u64;
+ dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr;
+ dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size;
+
+ /* segmentation data is not copied:
+ * buf_hdr.seg[]
+ * buf_hdr.segcount
+ */
+}
+
+static inline void *packet_map(odp_packet_hdr_t *pkt_hdr,
+ uint32_t offset, uint32_t *seg_len, int *seg_idx)
+{
+ void *addr;
+ uint32_t len;
+ int seg = 0;
+ int seg_count = pkt_hdr->buf_hdr.segcount;
+
+ if (odp_unlikely(offset >= pkt_hdr->frame_len))
+ return NULL;
+
+ if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) {
+ addr = pkt_hdr->buf_hdr.seg[0].data + offset;
+ len = pkt_hdr->buf_hdr.seg[0].len - offset;
+ } else {
+ int i;
+ uint32_t seg_start = 0, seg_end = 0;
+
+ for (i = 0; i < seg_count; i++) {
+ seg_end += pkt_hdr->buf_hdr.seg[i].len;
+
+ if (odp_likely(offset < seg_end))
+ break;
+
+ seg_start = seg_end;
+ }
+
+ addr = pkt_hdr->buf_hdr.seg[i].data + (offset - seg_start);
+ len = pkt_hdr->buf_hdr.seg[i].len - (offset - seg_start);
+ seg = i;
+ }
+
+ if (seg_len)
+ *seg_len = len;
+
+ if (seg_idx)
+ *seg_idx = seg;
+
+ return addr;
+}
static inline void packet_parse_disable(odp_packet_hdr_t *pkt_hdr)
{
@@ -48,11 +191,23 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr)
/**
* Initialize packet
*/
-static void packet_init(pool_t *pool, odp_packet_hdr_t *pkt_hdr,
- size_t size, int parse)
+static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len,
+ int parse)
{
- pkt_hdr->p.parsed_layers = LAYER_NONE;
+ uint32_t seg_len;
+ int num = pkt_hdr->buf_hdr.segcount;
+
+ if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num == 1)) {
+ seg_len = len;
+ pkt_hdr->buf_hdr.seg[0].len = len;
+ } else {
+ seg_len = len - ((num - 1) * CONFIG_PACKET_MAX_SEG_LEN);
+
+ /* Last segment data length */
+ pkt_hdr->buf_hdr.seg[num - 1].len = seg_len;
+ }
+ pkt_hdr->p.parsed_layers = LAYER_NONE;
pkt_hdr->p.input_flags.all = 0;
pkt_hdr->p.output_flags.all = 0;
pkt_hdr->p.error_flags.all = 0;
@@ -70,42 +225,260 @@ static void packet_init(pool_t *pool, odp_packet_hdr_t
*pkt_hdr,
* Packet tailroom is rounded up to fill the last
* segment occupied by the allocated length.
*/
- pkt_hdr->frame_len = size;
- pkt_hdr->headroom = pool->headroom;
- pkt_hdr->tailroom = pool->data_size - size + pool->tailroom;
+ pkt_hdr->frame_len = len;
+ pkt_hdr->headroom = CONFIG_PACKET_HEADROOM;
+ pkt_hdr->tailroom = CONFIG_PACKET_MAX_SEG_LEN - seg_len +
+ CONFIG_PACKET_TAILROOM;
pkt_hdr->input = ODP_PKTIO_INVALID;
}
-int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len,
- odp_packet_t pkt[], int max_num)
+static inline odp_packet_hdr_t *init_segments(odp_buffer_t buf[], int num)
{
- pool_t *pool = pool_entry_from_hdl(pool_hdl);
- int num, i;
- odp_packet_hdr_t *pkt_hdrs[max_num];
+ odp_packet_hdr_t *pkt_hdr;
+ int i;
+
+ /* First buffer is the packet descriptor */
+ pkt_hdr = odp_packet_hdr((odp_packet_t)buf[0]);
+
+ pkt_hdr->buf_hdr.seg[0].data = pkt_hdr->buf_hdr.base_data;
+ pkt_hdr->buf_hdr.seg[0].len = pkt_hdr->buf_hdr.base_len;
+
+ /* Link segments */
+ if (odp_unlikely(CONFIG_PACKET_MAX_SEGS != 1)) {
+ pkt_hdr->buf_hdr.segcount = num;
+
+ if (odp_unlikely(num > 1)) {
+ for (i = 1; i < num; i++) {
+ odp_packet_hdr_t *hdr;
+ odp_buffer_hdr_t *b_hdr;
+
+ hdr = odp_packet_hdr((odp_packet_t)buf[i]);
+ b_hdr = &hdr->buf_hdr;
+
+ pkt_hdr->buf_hdr.seg[i].hdr = hdr;
+ pkt_hdr->buf_hdr.seg[i].data = b_hdr->base_data;
+ pkt_hdr->buf_hdr.seg[i].len = b_hdr->base_len;
+ }
+ }
+ }
+
+ return pkt_hdr;
+}
+
+/* Calculate the number of segments */
+static inline int num_segments(uint32_t len)
+{
+ uint32_t max_seg_len;
+ int num;
- num = buffer_alloc_multi(pool, (odp_buffer_t *)pkt,
- (odp_buffer_hdr_t **)pkt_hdrs, max_num);
+ if (CONFIG_PACKET_MAX_SEGS == 1)
+ return 1;
+
+ num = 1;
+ max_seg_len = CONFIG_PACKET_MAX_SEG_LEN;
+
+ if (odp_unlikely(len > max_seg_len)) {
+ num = len / max_seg_len;
+
+ if (odp_likely((num * max_seg_len) != len))
+ num += 1;
+ }
+
+ return num;
+}
+
+static inline void copy_all_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from)
+{
+ int i;
+ int n = to->buf_hdr.segcount;
+ int num = from->buf_hdr.segcount;
for (i = 0; i < num; i++) {
- odp_packet_hdr_t *pkt_hdr = pkt_hdrs[i];
+ to->buf_hdr.seg[n + i].hdr = from->buf_hdr.seg[i].hdr;
+ to->buf_hdr.seg[n + i].data = from->buf_hdr.seg[i].data;
+ to->buf_hdr.seg[n + i].len = from->buf_hdr.seg[i].len;
+ }
+
+ to->buf_hdr.segcount = n + num;
+}
- packet_init(pool, pkt_hdr, len, 1 /* do parse */);
+static inline void copy_num_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from,
+ int num)
+{
+ int i;
- if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize)
- pull_tail_seg(pkt_hdr);
+ for (i = 0; i < num; i++) {
+ to->buf_hdr.seg[i].hdr = from->buf_hdr.seg[num + i].hdr;
+ to->buf_hdr.seg[i].data = from->buf_hdr.seg[num + i].data;
+ to->buf_hdr.seg[i].len = from->buf_hdr.seg[num + i].len;
}
+ to->buf_hdr.segcount = num;
+}
+
+static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr,
+ uint32_t len, int head)
+{
+ pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl);
+ odp_packet_hdr_t *new_hdr;
+ int num, ret;
+ uint32_t seg_len, offset;
+
+ num = num_segments(len);
+
+ if ((pkt_hdr->buf_hdr.segcount + num) > CONFIG_PACKET_MAX_SEGS)
+ return NULL;
+
+ {
+ odp_buffer_t buf[num];
+
+ ret = buffer_alloc_multi(pool, buf, NULL, num);
+ if (odp_unlikely(ret != num)) {
+ if (ret > 0)
+ buffer_free_multi(buf, ret);
+
+ return NULL;
+ }
+
+ new_hdr = init_segments(buf, num);
+ }
+
+ seg_len = len - ((num - 1) * pool->max_seg_len);
+ offset = pool->max_seg_len - seg_len;
+
+ if (head) {
+ /* add into the head*/
+ copy_all_segs(new_hdr, pkt_hdr);
+
+ /* adjust first segment length */
+ new_hdr->buf_hdr.seg[0].data += offset;
+ new_hdr->buf_hdr.seg[0].len = seg_len;
+
+ packet_seg_copy_md(new_hdr, pkt_hdr);
+ new_hdr->frame_len = pkt_hdr->frame_len + len;
+ new_hdr->headroom = pool->headroom + offset;
+ new_hdr->tailroom = pkt_hdr->tailroom;
+
+ pkt_hdr = new_hdr;
+ } else {
+ int last;
+
+ /* add into the tail */
+ copy_all_segs(pkt_hdr, new_hdr);
+
+ /* adjust last segment length */
+ last = packet_last_seg(pkt_hdr);
+ pkt_hdr->buf_hdr.seg[last].len = seg_len;
+
+ pkt_hdr->frame_len += len;
+ pkt_hdr->tailroom = pool->tailroom + offset;
+ }
+
+ return pkt_hdr;
+}
+
+static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
+ int num, uint32_t free_len,
+ uint32_t pull_len, int head)
+{
+ int i;
+ odp_packet_hdr_t *new_hdr;
+ odp_buffer_t buf[num];
+ int n = pkt_hdr->buf_hdr.segcount - num;
+
+ if (head) {
+ for (i = 0; i < num; i++)
+ buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[i].hdr);
+
+ /* First remaining segment is the new packet descriptor */
+ new_hdr = pkt_hdr->buf_hdr.seg[num].hdr;
+ copy_num_segs(new_hdr, pkt_hdr, n);
+ packet_seg_copy_md(new_hdr, pkt_hdr);
+
+ /* Tailroom not changed */
+ new_hdr->tailroom = pkt_hdr->tailroom;
+ /* No headroom in non-first segments */
+ new_hdr->headroom = 0;
+ new_hdr->frame_len = pkt_hdr->frame_len - free_len;
+
+ pull_head(new_hdr, pull_len);
+
+ pkt_hdr = new_hdr;
+ } else {
+ for (i = 0; i < num; i++)
+ buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[n + i].hdr);
+
+ /* Head segment remains, no need to copy or update majority
+ * of the metadata. */
+ pkt_hdr->buf_hdr.segcount = n;
+ pkt_hdr->frame_len -= free_len;
+ pkt_hdr->tailroom = pkt_hdr->buf_hdr.buf_end -
+ (uint8_t *)packet_tail(pkt_hdr);
+
+ pull_tail(pkt_hdr, pull_len);
+ }
+
+ buffer_free_multi(buf, num);
+
+ return pkt_hdr;
+}
+
+static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt,
+ int num_seg, odp_packet_t *pkt, int parse)
+{
+ int num_buf, i;
+ int num = max_pkt;
+ int max_buf = max_pkt * num_seg;
+ odp_buffer_t buf[max_buf];
+
+ num_buf = buffer_alloc_multi(pool, buf, NULL, max_buf);
+
+ /* Failed to allocate all segments */
+ if (odp_unlikely(num_buf != max_buf)) {
+ int num_free;
+
+ num = num_buf / num_seg;
+ num_free = num_buf - (num * num_seg);
+
+ if (num_free > 0)
+ buffer_free_multi(&buf[num_buf - num_free], num_free);
+
+ if (num == 0)
+ return 0;
+ }
+
+ for (i = 0; i < num; i++) {
+ odp_packet_hdr_t *pkt_hdr;
+
+ /* First buffer is the packet descriptor */
+ pkt[i] = (odp_packet_t)buf[i * num_seg];
+ pkt_hdr = init_segments(&buf[i * num_seg], num_seg);
+
+ packet_init(pkt_hdr, len, parse);
+ }
+
+ return num;
+}
+
+int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len,
+ odp_packet_t pkt[], int max_num)
+{
+ pool_t *pool = pool_entry_from_hdl(pool_hdl);
+ int num, num_seg;
+
+ num_seg = num_segments(len);
+ num = packet_alloc(pool, len, max_num, num_seg, pkt, 1);
+
return num;
}
odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len)
{
pool_t *pool = pool_entry_from_hdl(pool_hdl);
- size_t pkt_size = len ? len : pool->data_size;
odp_packet_t pkt;
- odp_packet_hdr_t *pkt_hdr;
- int ret;
+ int num, num_seg;
+ int zero_len = 0;
if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) {
__odp_errno = EINVAL;
@@ -115,28 +488,32 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl,
uint32_t len)
if (odp_unlikely(len > pool->max_len))
return ODP_PACKET_INVALID;
- ret = buffer_alloc_multi(pool, (odp_buffer_t *)&pkt, NULL, 1);
- if (ret != 1)
+ if (odp_unlikely(len == 0)) {
+ len = pool->data_size;
+ zero_len = 1;
+ }
+
+ num_seg = num_segments(len);
+ num = packet_alloc(pool, len, 1, num_seg, &pkt, 0);
+
+ if (odp_unlikely(num == 0))
return ODP_PACKET_INVALID;
- pkt_hdr = odp_packet_hdr(pkt);
- packet_init(pool, pkt_hdr, pkt_size, 0 /* do not parse */);
- if (len == 0)
- pull_tail(pkt_hdr, pkt_size);
+ if (odp_unlikely(zero_len)) {
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize)
- pull_tail_seg(pkt_hdr);
+ pull_tail(pkt_hdr, len);
+ }
return pkt;
}
int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len,
- odp_packet_t pkt[], int num)
+ odp_packet_t pkt[], int max_num)
{
pool_t *pool = pool_entry_from_hdl(pool_hdl);
- size_t pkt_size = len ? len : pool->data_size;
- int count, i;
- odp_packet_hdr_t *pkt_hdrs[num];
+ int num, num_seg;
+ int zero_len = 0;
if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) {
__odp_errno = EINVAL;
@@ -146,31 +523,75 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t
len,
if (odp_unlikely(len > pool->max_len))
return -1;
- count = buffer_alloc_multi(pool, (odp_buffer_t *)pkt,
- (odp_buffer_hdr_t **)pkt_hdrs, num);
+ if (odp_unlikely(len == 0)) {
+ len = pool->data_size;
+ zero_len = 1;
+ }
+
+ num_seg = num_segments(len);
+ num = packet_alloc(pool, len, max_num, num_seg, pkt, 0);
- for (i = 0; i < count; ++i) {
- odp_packet_hdr_t *pkt_hdr = pkt_hdrs[i];
+ if (odp_unlikely(zero_len)) {
+ int i;
- packet_init(pool, pkt_hdr, pkt_size, 0 /* do not parse */);
- if (len == 0)
- pull_tail(pkt_hdr, pkt_size);
+ for (i = 0; i < num; i++) {
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt[i]);
- if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize)
- pull_tail_seg(pkt_hdr);
+ pull_tail(pkt_hdr, len);
+ }
}
- return count;
+ return num;
}
void odp_packet_free(odp_packet_t pkt)
{
- buffer_free_multi((odp_buffer_t *)&pkt, 1);
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ int num_seg = pkt_hdr->buf_hdr.segcount;
+
+ if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) {
+ buffer_free_multi((odp_buffer_t *)&pkt, 1);
+ } else {
+ odp_buffer_t buf[num_seg];
+ int i;
+
+ buf[0] = (odp_buffer_t)pkt;
+
+ for (i = 1; i < num_seg; i++)
+ buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[i].hdr);
+
+ buffer_free_multi(buf, num_seg);
+ }
}
void odp_packet_free_multi(const odp_packet_t pkt[], int num)
{
- buffer_free_multi((const odp_buffer_t * const)pkt, num);
+ if (CONFIG_PACKET_MAX_SEGS == 1) {
+ buffer_free_multi((const odp_buffer_t * const)pkt, num);
+ } else {
+ odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS];
+ int i, j;
+ int bufs = 0;
+
+ for (i = 0; i < num; i++) {
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt[i]);
+ int num_seg = pkt_hdr->buf_hdr.segcount;
+ odp_buffer_hdr_t *buf_hdr = &pkt_hdr->buf_hdr;
+
+ buf[bufs] = (odp_buffer_t)pkt[i];
+ bufs++;
+
+ if (odp_likely(num_seg == 1))
+ continue;
+
+ for (j = 1; j < num_seg; j++) {
+ buf[bufs] = buffer_handle(buf_hdr->seg[j].hdr);
+ bufs++;
+ }
+ }
+
+ buffer_free_multi(buf, bufs);
+ }
}
int odp_packet_reset(odp_packet_t pkt, uint32_t len)
@@ -181,7 +602,7 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len)
if (len > pool->headroom + pool->data_size + pool->tailroom)
return -1;
- packet_init(pool, pkt_hdr, len, 0);
+ packet_init(pkt_hdr, len, 0);
return 0;
}
@@ -217,7 +638,7 @@ void *odp_packet_head(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- return buffer_map(&pkt_hdr->buf_hdr, 0, NULL, 0);
+ return pkt_hdr->buf_hdr.seg[0].data - pkt_hdr->headroom;
}
uint32_t odp_packet_buf_len(odp_packet_t pkt)
@@ -231,17 +652,14 @@ void *odp_packet_data(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- return packet_map(pkt_hdr, 0, NULL);
+ return packet_data(pkt_hdr);
}
uint32_t odp_packet_seg_len(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- uint32_t seglen;
- /* Call returns length of 1st data segment */
- packet_map(pkt_hdr, 0, &seglen);
- return seglen;
+ return packet_first_seg_len(pkt_hdr);
}
uint32_t odp_packet_len(odp_packet_t pkt)
@@ -263,7 +681,7 @@ void *odp_packet_tail(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL);
+ return packet_tail(pkt_hdr);
}
void *odp_packet_push_head(odp_packet_t pkt, uint32_t len)
@@ -274,21 +692,38 @@ void *odp_packet_push_head(odp_packet_t pkt, uint32_t len)
return NULL;
push_head(pkt_hdr, len);
- return packet_map(pkt_hdr, 0, NULL);
+ return packet_data(pkt_hdr);
}
int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len,
void **data_ptr, uint32_t *seg_len)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt);
+ odp_packet_hdr_t *new_hdr;
+ uint32_t headroom = pkt_hdr->headroom;
- if (len > pkt_hdr->headroom && push_head_seg(pkt_hdr, len))
- return -1;
+ if (len > headroom) {
+ push_head(pkt_hdr, headroom);
+ new_hdr = add_segments(pkt_hdr, len - headroom, 1);
- push_head(pkt_hdr, len);
+ if (new_hdr == NULL) {
+ /* segment alloc failed, rollback changes */
+ pull_head(pkt_hdr, headroom);
+ return -1;
+ }
+
+ *pkt = packet_handle(new_hdr);
+ pkt_hdr = new_hdr;
+ } else {
+ push_head(pkt_hdr, len);
+ }
if (data_ptr)
- *data_ptr = packet_map(pkt_hdr, 0, seg_len);
+ *data_ptr = packet_data(pkt_hdr);
+
+ if (seg_len)
+ *seg_len = packet_first_seg_len(pkt_hdr);
+
return 0;
}
@@ -300,51 +735,82 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len)
return NULL;
pull_head(pkt_hdr, len);
- return packet_map(pkt_hdr, 0, NULL);
+ return packet_data(pkt_hdr);
}
int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len,
- void **data_ptr, uint32_t *seg_len)
+ void **data_ptr, uint32_t *seg_len_out)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt);
+ uint32_t seg_len = packet_first_seg_len(pkt_hdr);
if (len > pkt_hdr->frame_len)
return -1;
- pull_head(pkt_hdr, len);
- if (pkt_hdr->headroom >= pkt_hdr->buf_hdr.segsize)
- pull_head_seg(pkt_hdr);
+ if (len < seg_len) {
+ pull_head(pkt_hdr, len);
+ } else if (CONFIG_PACKET_MAX_SEGS != 1) {
+ int num = 0;
+ uint32_t pull_len = 0;
+
+ while (seg_len <= len) {
+ pull_len = len - seg_len;
+ num++;
+ seg_len += packet_seg_len(pkt_hdr, num);
+ }
+
+ pkt_hdr = free_segments(pkt_hdr, num, len - pull_len,
+ pull_len, 1);
+ *pkt = packet_handle(pkt_hdr);
+ }
if (data_ptr)
- *data_ptr = packet_map(pkt_hdr, 0, seg_len);
+ *data_ptr = packet_data(pkt_hdr);
+
+ if (seg_len_out)
+ *seg_len_out = packet_first_seg_len(pkt_hdr);
+
return 0;
}
void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- uint32_t origin = pkt_hdr->frame_len;
+ void *old_tail;
if (len > pkt_hdr->tailroom)
return NULL;
+ old_tail = packet_tail(pkt_hdr);
push_tail(pkt_hdr, len);
- return packet_map(pkt_hdr, origin, NULL);
+
+ return old_tail;
}
int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len,
void **data_ptr, uint32_t *seg_len)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt);
- uint32_t origin = pkt_hdr->frame_len;
+ void *ret;
+ uint32_t tailroom = pkt_hdr->tailroom;
+ uint32_t tail_off = pkt_hdr->frame_len;
- if (len > pkt_hdr->tailroom && push_tail_seg(pkt_hdr, len))
- return -1;
+ if (len > tailroom) {
+ push_tail(pkt_hdr, tailroom);
+ ret = add_segments(pkt_hdr, len - tailroom, 0);
- push_tail(pkt_hdr, len);
+ if (ret == NULL) {
+ /* segment alloc failed, rollback changes */
+ pull_tail(pkt_hdr, tailroom);
+ return -1;
+ }
+ } else {
+ push_tail(pkt_hdr, len);
+ }
if (data_ptr)
- *data_ptr = packet_map(pkt_hdr, origin, seg_len);
+ *data_ptr = packet_map(pkt_hdr, tail_off, seg_len, NULL);
+
return 0;
}
@@ -352,27 +818,45 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (len > pkt_hdr->frame_len)
+ if (len > packet_last_seg_len(pkt_hdr))
return NULL;
pull_tail(pkt_hdr, len);
- return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL);
+
+ return packet_tail(pkt_hdr);
}
int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len,
void **tail_ptr, uint32_t *tailroom)
{
+ int last;
+ uint32_t seg_len;
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt);
if (len > pkt_hdr->frame_len)
return -1;
- pull_tail(pkt_hdr, len);
- if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize)
- pull_tail_seg(pkt_hdr);
+ last = packet_last_seg(pkt_hdr);
+ seg_len = packet_seg_len(pkt_hdr, last);
+
+ if (len < seg_len) {
+ pull_tail(pkt_hdr, len);
+ } else if (CONFIG_PACKET_MAX_SEGS != 1) {
+ int num = 0;
+ uint32_t pull_len = 0;
+
+ while (seg_len <= len) {
+ pull_len = len - seg_len;
+ num++;
+ seg_len += packet_seg_len(pkt_hdr, last - num);
+ }
+
+ free_segments(pkt_hdr, num, len - pull_len, pull_len, 0);
+ }
if (tail_ptr)
- *tail_ptr = packet_map(pkt_hdr, pkt_hdr->frame_len, NULL);
+ *tail_ptr = packet_tail(pkt_hdr);
+
if (tailroom)
*tailroom = pkt_hdr->tailroom;
return 0;
@@ -381,11 +865,12 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len,
void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len,
odp_packet_seg_t *seg)
{
+ int seg_idx;
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- void *addr = packet_map(pkt_hdr, offset, len);
+ void *addr = packet_map(pkt_hdr, offset, len, &seg_idx);
if (addr != NULL && seg != NULL)
- *seg = (odp_packet_seg_t)pkt;
+ *seg = seg_idx;
return addr;
}
@@ -445,7 +930,7 @@ void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len)
if (!packet_hdr_has_l2(pkt_hdr))
return NULL;
- return packet_map(pkt_hdr, pkt_hdr->p.l2_offset, len);
+ return packet_map(pkt_hdr, pkt_hdr->p.l2_offset, len, NULL);
}
uint32_t odp_packet_l2_offset(odp_packet_t pkt)
@@ -475,7 +960,7 @@ void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len)
if (pkt_hdr->p.parsed_layers < LAYER_L3)
packet_parse_layer(pkt_hdr, LAYER_L3);
- return packet_map(pkt_hdr, pkt_hdr->p.l3_offset, len);
+ return packet_map(pkt_hdr, pkt_hdr->p.l3_offset, len, NULL);
}
uint32_t odp_packet_l3_offset(odp_packet_t pkt)
@@ -506,7 +991,7 @@ void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len)
if (pkt_hdr->p.parsed_layers < LAYER_L4)
packet_parse_layer(pkt_hdr, LAYER_L4);
- return packet_map(pkt_hdr, pkt_hdr->p.l4_offset, len);
+ return packet_map(pkt_hdr, pkt_hdr->p.l4_offset, len, NULL);
}
uint32_t odp_packet_l4_offset(odp_packet_t pkt)
@@ -568,29 +1053,33 @@ int odp_packet_is_segmented(odp_packet_t pkt)
int odp_packet_num_segs(odp_packet_t pkt)
{
- return odp_packet_hdr(pkt)->buf_hdr.segcount;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+
+ return pkt_hdr->buf_hdr.segcount;
}
odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt)
{
- return (odp_packet_seg_t)pkt;
+ (void)pkt;
+
+ return 0;
}
odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt)
{
- (void)pkt;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- /* Only one segment */
- return (odp_packet_seg_t)pkt;
+ return packet_last_seg(pkt_hdr);
}
odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg)
{
- (void)pkt;
- (void)seg;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- /* Only one segment */
- return ODP_PACKET_SEG_INVALID;
+ if (odp_unlikely(seg >= (odp_packet_seg_t)packet_last_seg(pkt_hdr)))
+ return ODP_PACKET_SEG_INVALID;
+
+ return seg + 1;
}
/*
@@ -602,18 +1091,22 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt,
odp_packet_seg_t seg)
void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg)
{
- (void)seg;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- /* Only one segment */
- return odp_packet_data(pkt);
+ if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount))
+ return NULL;
+
+ return packet_seg_data(pkt_hdr, seg);
}
uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg)
{
- (void)seg;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- /* Only one segment */
- return odp_packet_seg_len(pkt);
+ if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount))
+ return 0;
+
+ return packet_seg_len(pkt_hdr, seg);
}
/*
@@ -688,7 +1181,7 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset,
uint32_t len,
uint32_t shift;
uint32_t seglen = 0; /* GCC */
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt);
- void *addr = packet_map(pkt_hdr, offset, &seglen);
+ void *addr = packet_map(pkt_hdr, offset, &seglen, NULL);
uint64_t uaddr = (uint64_t)(uintptr_t)addr;
uint64_t misalign;
@@ -733,6 +1226,7 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src)
src, 0, src_len);
if (src != *dst)
odp_packet_free(src);
+
return 0;
}
@@ -808,7 +1302,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t
offset,
return -1;
while (len > 0) {
- mapaddr = packet_map(pkt_hdr, offset, &seglen);
+ mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL);
cpylen = len > seglen ? seglen : len;
memcpy(dstaddr, mapaddr, cpylen);
offset += cpylen;
@@ -832,7 +1326,7 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t
offset,
return -1;
while (len > 0) {
- mapaddr = packet_map(pkt_hdr, offset, &seglen);
+ mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL);
cpylen = len > seglen ? seglen : len;
memcpy(mapaddr, srcaddr, cpylen);
offset += cpylen;
@@ -878,8 +1372,8 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t
dst_offset,
}
while (len > 0) {
- dst_map = packet_map(dst_hdr, dst_offset, &dst_seglen);
- src_map = packet_map(src_hdr, src_offset, &src_seglen);
+ dst_map = packet_map(dst_hdr, dst_offset, &dst_seglen, NULL);
+ src_map = packet_map(src_hdr, src_offset, &src_seglen, NULL);
minseg = dst_seglen > src_seglen ? src_seglen : dst_seglen;
cpylen = len > minseg ? minseg : len;
@@ -1364,8 +1858,8 @@ parse_exit:
*/
int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, layer_t layer)
{
- uint32_t seg_len;
- void *base = packet_map(pkt_hdr, 0, &seg_len);
+ uint32_t seg_len = packet_first_seg_len(pkt_hdr);
+ void *base = packet_data(pkt_hdr);
return packet_parse_common(&pkt_hdr->p, base, pkt_hdr->frame_len,
seg_len, layer);
diff --git a/platform/linux-generic/odp_pool.c
b/platform/linux-generic/odp_pool.c
index 364df97..7c462e5 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -32,6 +32,9 @@
ODP_STATIC_ASSERT(CONFIG_POOL_CACHE_SIZE > (2 * CACHE_BURST),
"cache_burst_size_too_large_compared_to_cache_size");
+ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_LEN_MIN >= 256,
+ "ODP Segment size must be a minimum of 256 bytes");
+
/* Thread local variables */
typedef struct pool_local_t {
pool_cache_t *cache[ODP_CONFIG_POOLS];
@@ -46,6 +49,14 @@ static inline odp_pool_t pool_index_to_handle(uint32_t
pool_idx)
return _odp_cast_scalar(odp_pool_t, pool_idx);
}
+static inline uint32_t pool_id_from_buf(odp_buffer_t buf)
+{
+ odp_buffer_bits_t handle;
+
+ handle.handle = buf;
+ return handle.pool_id;
+}
+
int odp_pool_init_global(void)
{
uint32_t i;
@@ -198,7 +209,7 @@ static void init_buffers(pool_t *pool)
ring_t *ring;
uint32_t mask;
int type;
- uint32_t size;
+ uint32_t seg_size;
ring = &pool->ring.hdr;
mask = pool->ring_mask;
@@ -223,12 +234,12 @@ static void init_buffers(pool_t *pool)
while (((uintptr_t)&data[offset]) % pool->align != 0)
offset++;
- memset(buf_hdr, 0, sizeof(odp_buffer_hdr_t));
+ memset(buf_hdr, 0, (uintptr_t)data - (uintptr_t)buf_hdr);
- size = pool->headroom + pool->data_size + pool->tailroom;
+ seg_size = pool->headroom + pool->data_size + pool->tailroom;
/* Initialize buffer metadata */
- buf_hdr->size = size;
+ buf_hdr->size = seg_size;
buf_hdr->type = type;
buf_hdr->event_type = type;
buf_hdr->pool_hdl = pool->pool_hdl;
@@ -236,10 +247,18 @@ static void init_buffers(pool_t *pool)
/* Show user requested size through API */
buf_hdr->uarea_size = pool->params.pkt.uarea_size;
buf_hdr->segcount = 1;
- buf_hdr->segsize = size;
+ buf_hdr->segsize = seg_size;
/* Pointer to data start (of the first segment) */
- buf_hdr->addr[0] = &data[offset];
+ buf_hdr->seg[0].hdr = buf_hdr;
+ buf_hdr->seg[0].data = &data[offset];
+ buf_hdr->seg[0].len = pool->data_size;
+
+ /* Store base values for fast init */
+ buf_hdr->base_data = buf_hdr->seg[0].data;
+ buf_hdr->base_len = buf_hdr->seg[0].len;
+ buf_hdr->buf_end = &data[offset + pool->data_size +
+ pool->tailroom];
buf_hdl = form_buffer_handle(pool->pool_idx, i);
buf_hdr->handle.handle = buf_hdl;
@@ -296,25 +315,13 @@ static odp_pool_t pool_create(const char *name,
odp_pool_param_t *params,
break;
case ODP_POOL_PACKET:
- headroom = ODP_CONFIG_PACKET_HEADROOM;
- tailroom = ODP_CONFIG_PACKET_TAILROOM;
- num = params->pkt.num;
- uarea_size = params->pkt.uarea_size;
-
- data_size = ODP_CONFIG_PACKET_SEG_LEN_MAX;
-
- if (data_size < ODP_CONFIG_PACKET_SEG_LEN_MIN)
- data_size = ODP_CONFIG_PACKET_SEG_LEN_MIN;
-
- if (data_size > ODP_CONFIG_PACKET_SEG_LEN_MAX) {
- ODP_ERR("Too large seg len requirement");
- return ODP_POOL_INVALID;
- }
-
- max_seg_len = ODP_CONFIG_PACKET_SEG_LEN_MAX -
- ODP_CONFIG_PACKET_HEADROOM -
- ODP_CONFIG_PACKET_TAILROOM;
- max_len = ODP_CONFIG_PACKET_MAX_SEGS * max_seg_len;
+ headroom = CONFIG_PACKET_HEADROOM;
+ tailroom = CONFIG_PACKET_TAILROOM;
+ num = params->pkt.num;
+ uarea_size = params->pkt.uarea_size;
+ data_size = CONFIG_PACKET_MAX_SEG_LEN;
+ max_seg_len = CONFIG_PACKET_MAX_SEG_LEN;
+ max_len = CONFIG_PACKET_MAX_SEGS * max_seg_len;
break;
case ODP_POOL_TIMEOUT:
@@ -470,31 +477,6 @@ void _odp_buffer_event_type_set(odp_buffer_t buf, int ev)
buf_hdl_to_hdr(buf)->event_type = ev;
}
-void *buffer_map(odp_buffer_hdr_t *buf,
- uint32_t offset,
- uint32_t *seglen,
- uint32_t limit)
-{
- int seg_index;
- int seg_offset;
-
- if (odp_likely(offset < buf->segsize)) {
- seg_index = 0;
- seg_offset = offset;
- } else {
- ODP_ERR("\nSEGMENTS NOT SUPPORTED\n");
- return NULL;
- }
-
- if (seglen != NULL) {
- uint32_t buf_left = limit - offset;
- *seglen = seg_offset + buf_left <= buf->segsize ?
- buf_left : buf->segsize - seg_offset;
- }
-
- return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]);
-}
-
odp_pool_t odp_pool_lookup(const char *name)
{
uint32_t i;
@@ -727,9 +709,7 @@ void odp_buffer_free_multi(const odp_buffer_t buf[], int
num)
int odp_pool_capability(odp_pool_capability_t *capa)
{
- uint32_t max_len = ODP_CONFIG_PACKET_SEG_LEN_MAX -
- ODP_CONFIG_PACKET_HEADROOM -
- ODP_CONFIG_PACKET_TAILROOM;
+ uint32_t max_seg_len = CONFIG_PACKET_MAX_SEG_LEN;
memset(capa, 0, sizeof(odp_pool_capability_t));
@@ -743,13 +723,13 @@ int odp_pool_capability(odp_pool_capability_t *capa)
/* Packet pools */
capa->pkt.max_pools = ODP_CONFIG_POOLS;
- capa->pkt.max_len = ODP_CONFIG_PACKET_MAX_SEGS * max_len;
+ capa->pkt.max_len = CONFIG_PACKET_MAX_SEGS * max_seg_len;
capa->pkt.max_num = CONFIG_POOL_MAX_NUM;
- capa->pkt.min_headroom = ODP_CONFIG_PACKET_HEADROOM;
- capa->pkt.min_tailroom = ODP_CONFIG_PACKET_TAILROOM;
- capa->pkt.max_segs_per_pkt = ODP_CONFIG_PACKET_MAX_SEGS;
- capa->pkt.min_seg_len = max_len;
- capa->pkt.max_seg_len = max_len;
+ capa->pkt.min_headroom = CONFIG_PACKET_HEADROOM;
+ capa->pkt.min_tailroom = CONFIG_PACKET_TAILROOM;
+ capa->pkt.max_segs_per_pkt = CONFIG_PACKET_MAX_SEGS;
+ capa->pkt.min_seg_len = max_seg_len;
+ capa->pkt.max_seg_len = max_seg_len;
capa->pkt.max_uarea_size = 0;
/* Timeout pools */
@@ -765,7 +745,7 @@ void odp_pool_print(odp_pool_t pool_hdl)
pool = pool_entry_from_hdl(pool_hdl);
- printf("Pool info\n");
+ printf("\nPool info\n");
printf("---------\n");
printf(" pool %" PRIu64 "\n",
odp_pool_to_u64(pool->pool_hdl));
@@ -812,19 +792,6 @@ uint64_t odp_pool_to_u64(odp_pool_t hdl)
return _odp_pri(hdl);
}
-int seg_alloc_head(odp_buffer_hdr_t *buf_hdr, int segcount)
-{
- (void)buf_hdr;
- (void)segcount;
- return 0;
-}
-
-void seg_free_head(odp_buffer_hdr_t *buf_hdr, int segcount)
-{
- (void)buf_hdr;
- (void)segcount;
-}
-
int seg_alloc_tail(odp_buffer_hdr_t *buf_hdr, int segcount)
{
(void)buf_hdr;
@@ -855,13 +822,3 @@ int odp_buffer_is_valid(odp_buffer_t buf)
return 1;
}
-
-uint32_t pool_headroom(odp_pool_t pool)
-{
- return pool_entry_from_hdl(pool)->headroom;
-}
-
-uint32_t pool_tailroom(odp_pool_t pool)
-{
- return pool_entry_from_hdl(pool)->tailroom;
-}
diff --git a/platform/linux-generic/pktio/netmap.c
b/platform/linux-generic/pktio/netmap.c
index cf67741..8eb8145 100644
--- a/platform/linux-generic/pktio/netmap.c
+++ b/platform/linux-generic/pktio/netmap.c
@@ -345,9 +345,7 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED,
pktio_entry_t *pktio_entry,
pkt_nm->pool = pool;
/* max frame len taking into account the l2-offset */
- pkt_nm->max_frame_len = ODP_CONFIG_PACKET_BUF_LEN_MAX -
- pool_headroom(pool) -
- pool_tailroom(pool);
+ pkt_nm->max_frame_len = CONFIG_PACKET_MAX_SEG_LEN;
/* allow interface to be opened with or without the 'netmap:' prefix */
prefix = "netmap:";
diff --git a/platform/linux-generic/pktio/socket.c
b/platform/linux-generic/pktio/socket.c
index ab25aab..9fe4a7e 100644
--- a/platform/linux-generic/pktio/socket.c
+++ b/platform/linux-generic/pktio/socket.c
@@ -46,7 +46,8 @@
#include <protocols/eth.h>
#include <protocols/ip.h>
-#define MAX_SEGS ODP_CONFIG_PACKET_MAX_SEGS
+#define MAX_SEGS CONFIG_PACKET_MAX_SEGS
+#define PACKET_JUMBO_LEN (9 * 1024)
static int disable_pktio; /** !0 this pktio disabled, 0 enabled */
--
2.8.1
