Add support for segment operation with references
Signed-off-by: Bill Fischofer <bill.fischo...@linaro.org>
---
.../include/odp/api/plat/packet_inlines.h | 39 ++---
.../include/odp/api/plat/packet_inlines_api.h | 16 --
.../linux-generic/include/odp_packet_internal.h | 55 ++++++-
platform/linux-generic/odp_packet.c | 161 ++++++++++++++++++---
4 files changed, 206 insertions(+), 65 deletions(-)
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h
b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
index 02295b08..8bfd85fe 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
@@ -40,7 +40,19 @@ static inline uint32_t _odp_packet_seg_len(odp_packet_t pkt)
/** @internal Inline function @param pkt @return */
static inline uint32_t _odp_packet_len(odp_packet_t pkt)
{
- return _odp_pkt_get(pkt, uint32_t, frame_len);
+ uint32_t pkt_len = _odp_pkt_get(pkt, uint32_t, frame_len);
+ void *ref_nxt = _odp_pkt_get(pkt, void *, ref_hdr);
+ void *ref_pkt = (void *)pkt;
+
+ while (ref_nxt) {
+ pkt_len += _odp_pkt_get(ref_pkt, uint32_t, ref_len) -
+ _odp_pkt_get(ref_pkt, uint32_t, ref_offset);
+
+ ref_pkt = ref_nxt;
+ ref_nxt = _odp_pkt_get(ref_nxt, void *, ref_hdr);
+ }
+
+ return pkt_len;
}
/** @internal Inline function @param pkt @return */
@@ -68,12 +80,6 @@ static inline odp_pktio_t _odp_packet_input(odp_packet_t pkt)
}
/** @internal Inline function @param pkt @return */
-static inline int _odp_packet_num_segs(odp_packet_t pkt)
-{
- return _odp_pkt_get(pkt, uint8_t, segcount);
-}
-
-/** @internal Inline function @param pkt @return */
static inline void *_odp_packet_user_ptr(odp_packet_t pkt)
{
return _odp_pkt_get(pkt, void *, user_ptr);
@@ -112,7 +118,8 @@ static inline void *_odp_packet_head(odp_packet_t pkt)
/** @internal Inline function @param pkt @return */
static inline int _odp_packet_is_segmented(odp_packet_t pkt)
{
- return _odp_pkt_get(pkt, uint8_t, segcount) > 1;
+ return _odp_pkt_get(pkt, uint8_t, segcount) > 1 ||
+ _odp_pkt_get(pkt, void *, ref_hdr) != NULL;
}
/** @internal Inline function @param pkt @return */
@@ -123,22 +130,6 @@ static inline odp_packet_seg_t
_odp_packet_first_seg(odp_packet_t pkt)
return 0;
}
-/** @internal Inline function @param pkt @return */
-static inline odp_packet_seg_t _odp_packet_last_seg(odp_packet_t pkt)
-{
- return _odp_packet_num_segs(pkt) - 1;
-}
-
-/** @internal Inline function @param pkt @param seg @return */
-static inline odp_packet_seg_t _odp_packet_next_seg(odp_packet_t pkt,
- odp_packet_seg_t seg)
-{
- if (odp_unlikely(seg >= _odp_packet_last_seg(pkt)))
- return ODP_PACKET_SEG_INVALID;
-
- return seg + 1;
-}
-
/** @internal Inline function @param pkt @param offset @param len */
static inline void _odp_packet_prefetch(odp_packet_t pkt, uint32_t offset,
uint32_t len)
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h
b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h
index 233bc876..f818f820 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h
@@ -48,11 +48,6 @@ _ODP_INLINE odp_pktio_t odp_packet_input(odp_packet_t pkt)
return _odp_packet_input(pkt);
}
-_ODP_INLINE int odp_packet_num_segs(odp_packet_t pkt)
-{
- return _odp_packet_num_segs(pkt);
-}
-
_ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt)
{
return _odp_packet_user_ptr(pkt);
@@ -93,17 +88,6 @@ _ODP_INLINE odp_packet_seg_t
odp_packet_first_seg(odp_packet_t pkt)
return _odp_packet_first_seg(pkt);
}
-_ODP_INLINE odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt)
-{
- return _odp_packet_last_seg(pkt);
-}
-
-_ODP_INLINE odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt,
- odp_packet_seg_t seg)
-{
- return _odp_packet_next_seg(pkt, seg);
-}
-
_ODP_INLINE void odp_packet_prefetch(odp_packet_t pkt, uint32_t offset,
uint32_t len)
{
diff --git a/platform/linux-generic/include/odp_packet_internal.h
b/platform/linux-generic/include/odp_packet_internal.h
index b6afd581..c99e9843 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -171,6 +171,50 @@ static inline odp_packet_hdr_t
*odp_packet_hdr(odp_packet_t pkt)
return (odp_packet_hdr_t *)(uintptr_t)pkt;
}
+static inline odp_packet_hdr_t *packet_last_hdr(odp_packet_t pkt,
+ uint32_t *offset)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ odp_packet_hdr_t *prev_hdr = pkt_hdr;
+ uint32_t ref_offset = 0;
+
+ while (pkt_hdr->ref_hdr) {
+ ref_offset = pkt_hdr->ref_offset;
+ prev_hdr = pkt_hdr;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ }
+
+ if (offset) {
+ if (prev_hdr != pkt_hdr)
+ ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len;
+ *offset = ref_offset;
+ }
+
+ return pkt_hdr;
+}
+
+static inline odp_packet_hdr_t *packet_prev_hdr(odp_packet_hdr_t *pkt_hdr,
+ odp_packet_hdr_t *cur_hdr,
+ uint32_t *offset)
+{
+ uint32_t ref_offset = 0;
+ odp_packet_hdr_t *prev_hdr = pkt_hdr;
+
+ while (pkt_hdr->ref_hdr != cur_hdr) {
+ ref_offset = pkt_hdr->ref_offset;
+ prev_hdr = pkt_hdr;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ }
+
+ if (offset) {
+ if (prev_hdr != pkt_hdr)
+ ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len;
+ *offset = ref_offset;
+ }
+
+ return pkt_hdr;
+}
+
static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr)
{
return (odp_packet_t)pkt_hdr;
@@ -194,7 +238,16 @@ static inline void
copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr,
static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr)
{
- return pkt_hdr->frame_len;
+ uint32_t pkt_len = pkt_hdr->frame_len;
+ odp_packet_hdr_t *ref_hdr = pkt_hdr->ref_hdr;
+
+ while (ref_hdr) {
+ pkt_len += (pkt_hdr->ref_len - pkt_hdr->ref_offset);
+ pkt_hdr = ref_hdr;
+ ref_hdr = ref_hdr->ref_hdr;
+ }
+
+ return pkt_len;
}
static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr)
diff --git a/platform/linux-generic/odp_packet.c
b/platform/linux-generic/odp_packet.c
index d4910214..c9cb3ef2 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -54,7 +54,7 @@ static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt)
static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr)
{
- return pkt_hdr->buf_hdr.handle.handle;
+ return odp_hdr_to_buf(&pkt_hdr->buf_hdr);
}
static inline uint32_t packet_ref_inc(odp_packet_hdr_t *pkt_hdr)
@@ -78,7 +78,8 @@ static inline uint32_t packet_seg_len(odp_packet_hdr_t
*pkt_hdr,
return pkt_hdr->buf_hdr.seg[seg_idx].len;
}
-static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t
seg_idx)
+static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr,
+ uint32_t seg_idx)
{
return pkt_hdr->buf_hdr.seg[seg_idx].data;
}
@@ -216,7 +217,15 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr,
int seg = 0;
int seg_count = pkt_hdr->buf_hdr.segcount;
- if (odp_unlikely(offset >= pkt_hdr->frame_len))
+ /* Special processing for references */
+ while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) {
+ offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset);
+ offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len);
+ pkt_hdr = pkt_hdr->ref_hdr;
+ seg_count = pkt_hdr->buf_hdr.segcount;
+ }
+
+ if (odp_unlikely(offset > pkt_hdr->frame_len))
return NULL;
if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) {
@@ -719,13 +728,47 @@ odp_event_t odp_packet_to_event(odp_packet_t pkt)
uint32_t odp_packet_buf_len(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ uint32_t buf_len = 0;
+
+ do {
+ buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ } while (pkt_hdr);
- return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount;
+ return buf_len;
}
-void *odp_packet_tail(odp_packet_t pkt)
+uint32_t odp_packet_unshared_len(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ uint32_t pkt_len = 0, offset = 0;
+
+ if (packet_ref_count(pkt_hdr) == 1)
+ pkt_hdr->unshared_len = pkt_hdr->frame_len;
+
+ do {
+ if (packet_ref_count(pkt_hdr) > 1) {
+ if (offset == 0)
+ pkt_len += pkt_hdr->unshared_len;
+ break;
+ }
+
+ pkt_len += pkt_hdr->frame_len - offset;
+ offset = pkt_hdr->ref_offset;
+
+ if (pkt_hdr->ref_hdr)
+ offset += (pkt_hdr->ref_hdr->frame_len -
+ pkt_hdr->ref_len);
+
+ pkt_hdr = pkt_hdr->ref_hdr;
+ } while (pkt_hdr);
+
+ return pkt_len;
+}
+
+void *odp_packet_tail(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL);
return packet_tail(pkt_hdr);
}
@@ -1360,6 +1403,45 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t
timestamp)
pkt_hdr->p.input_flags.timestamp = 1;
}
+int odp_packet_num_segs(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ uint32_t segcount = 0, i;
+ uint32_t seg_offset = 0, offset;
+
+ do {
+ segcount += pkt_hdr->buf_hdr.segcount - seg_offset;
+ offset = pkt_hdr->ref_offset;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ if (pkt_hdr) {
+ for (i = 0, seg_offset = 0;
+ i < pkt_hdr->buf_hdr.segcount;
+ i++, seg_offset++) {
+ if (offset < pkt_hdr->buf_hdr.seg[i].len)
+ break;
+ offset -= pkt_hdr->buf_hdr.seg[i].len;
+ }
+ }
+ } while (pkt_hdr);
+
+ return segcount;
+}
+
+odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt)
+{
+ return (odp_packet_seg_t)(odp_packet_num_segs(pkt) - 1);
+}
+
+odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg)
+{
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ if (odp_unlikely(seg >= packet_last_seg(pkt_hdr)))
+ return ODP_PACKET_SEG_INVALID;
+
+ return seg + 1;
+}
+
/*
*
* Segment level
@@ -1370,21 +1452,51 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t
timestamp)
void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ uint32_t seg_offset = 0, offset = 0, i;
+
+ while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset &&
+ pkt_hdr->ref_hdr) {
+ seg -= (pkt_hdr->buf_hdr.segcount - seg_offset);
+ offset = pkt_hdr->ref_offset;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ for (i = 0, seg_offset = 0;
+ i < pkt_hdr->buf_hdr.segcount;
+ i++, seg_offset++) {
+ if (offset < pkt_hdr->buf_hdr.seg[i].len)
+ break;
+ offset -= pkt_hdr->buf_hdr.seg[i].len;
+ }
+ }
- if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount))
+ if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount))
return NULL;
- return packet_seg_data(pkt_hdr, seg);
+ return packet_seg_data(pkt_hdr, seg + seg_offset) + offset;
}
uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ uint32_t seg_offset = 0, offset = 0, i;
+
+ while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset &&
+ pkt_hdr->ref_hdr) {
+ seg -= (pkt_hdr->buf_hdr.segcount - seg_offset);
+ offset = pkt_hdr->ref_offset;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ for (i = 0, seg_offset = 0;
+ i < pkt_hdr->buf_hdr.segcount;
+ i++, seg_offset++) {
+ if (offset < pkt_hdr->buf_hdr.seg[i].len)
+ break;
+ offset -= pkt_hdr->buf_hdr.seg[i].len;
+ }
+ }
- if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount))
+ if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount))
return 0;
- return packet_seg_len(pkt_hdr, seg);
+ return packet_seg_len(pkt_hdr, seg + seg_offset) - offset;
}
/*
@@ -1398,7 +1510,7 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t
offset, uint32_t len)
{
odp_packet_t pkt = *pkt_ptr;
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- uint32_t pktlen = pkt_hdr->frame_len;
+ uint32_t pktlen = packet_len(pkt_hdr);
odp_packet_t newpkt;
if (offset > pktlen)
@@ -1468,6 +1580,8 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset,
uint32_t len,
if (align > ODP_CACHE_LINE_SIZE)
return -1;
+ ODP_ASSERT(odp_packet_has_ref(*pkt) == 0);
+
if (seglen >= len) {
misalign = align <= 1 ? 0 :
ROUNDUP_ALIGN(uaddr, align) - uaddr;
@@ -1507,10 +1621,13 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t
src)
uint32_t dst_len = dst_hdr->frame_len;
uint32_t src_len = src_hdr->frame_len;
+ ODP_ASSERT(packet_ref_count(dst_hdr) == 1);
+
/* Do a copy if resulting packet would be out of segments or packets
- * are from different pools. */
+ * are from different pools or src is a reference. */
if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) ||
- odp_unlikely(dst_pool != src_pool)) {
+ odp_unlikely(dst_pool != src_pool) ||
+ odp_unlikely(packet_ref_count(src_hdr)) > 1) {
if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) {
(void)odp_packet_copy_from_pkt(*dst, dst_len,
src, 0, src_len);
@@ -1559,8 +1676,7 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len,
odp_packet_t *tail)
odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool)
{
- odp_packet_hdr_t *srchdr = packet_hdr(pkt);
- uint32_t pktlen = srchdr->frame_len;
+ uint32_t pktlen = odp_packet_len(pkt);
odp_packet_t newpkt = odp_packet_alloc(pool, pktlen);
if (newpkt != ODP_PACKET_INVALID) {
@@ -1599,7 +1715,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t
offset,
uint8_t *dstaddr = (uint8_t *)dst;
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- if (offset + len > pkt_hdr->frame_len)
+ if (offset + len > packet_len(pkt_hdr))
return -1;
while (len > 0) {
@@ -1623,7 +1739,7 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t
offset,
const uint8_t *srcaddr = (const uint8_t *)src;
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- if (offset + len > pkt_hdr->frame_len)
+ if (offset + len > packet_len(pkt_hdr))
return -1;
ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len);
@@ -1653,10 +1769,12 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t
dst_offset,
uint32_t src_seglen = 0; /* GCC */
int overlap;
- if (dst_offset + len > dst_hdr->frame_len ||
- src_offset + len > src_hdr->frame_len)
+ if (dst_offset + len > packet_len(dst_hdr) ||
+ src_offset + len > packet_len(src_hdr))
return -1;
+ ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len);
+
overlap = (dst_hdr == src_hdr &&
((dst_offset <= src_offset &&
dst_offset + len >= src_offset) ||
@@ -1741,7 +1859,7 @@ void odp_packet_print(odp_packet_t pkt)
len += snprintf(&str[len], n - len,
" l4_offset %" PRIu32 "\n", hdr->p.l4_offset);
len += snprintf(&str[len], n - len,
- " frame_len %" PRIu32 "\n", hdr->frame_len);
+ " frame_len %" PRIu32 "\n", packet_len(hdr));
len += snprintf(&str[len], n - len,
" input %" PRIu64 "\n",
odp_pktio_to_u64(hdr->input));
@@ -2268,11 +2386,6 @@ int odp_packet_has_ref(odp_packet_t pkt)
return 0;
}
-uint32_t odp_packet_unshared_len(odp_packet_t pkt)
-{
- return odp_packet_len(pkt);
-}
-
/* Include non-inlined versions of API functions */
#if ODP_ABI_COMPAT == 1
#include <odp/api/plat/packet_inlines_api.h>
--
2.12.0.rc1
