While reference counts are implemented as atomics, try to avoid
unnecessary atomic decrement operations in free paths. This streamlines
free processing in the expected case where packets do not contain
shared segments.
Signed-off-by: Bill Fischofer <bill.fischo...@linaro.org>
---
platform/linux-generic/include/odp_packet_internal.h | 6 +++++-
platform/linux-generic/odp_packet.c | 20 +++++++++++---------
2 files changed, 16 insertions(+), 10 deletions(-)
diff --git a/platform/linux-generic/include/odp_packet_internal.h
b/platform/linux-generic/include/odp_packet_internal.h
index 593f30dd..5864785a 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -257,7 +257,11 @@ static inline uint32_t packet_len(odp_packet_hdr_t
*pkt_hdr)
static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr)
{
- return odp_atomic_load_u32(&pkt_hdr->ref_count);
+ /* Breach the atomic type to do a peek at the ref count. This
+ * is used to bypass atomic operations if ref_count == 1 for
+ * performance reasons.
+ */
+ return pkt_hdr->ref_count.v;
}
static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n)
diff --git a/platform/linux-generic/odp_packet.c
b/platform/linux-generic/odp_packet.c
index 4d2cde12..81bbcedd 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -477,7 +477,7 @@ static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int
first, int num)
for (i = 0, nfree = 0; i < num; i++) {
odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr;
- if (packet_ref_dec(hdr) == 1)
+ if (packet_ref_count(hdr) == 1 || packet_ref_dec(hdr) == 1)
buf[nfree++] = buffer_handle(hdr);
}
@@ -499,7 +499,8 @@ static inline odp_packet_hdr_t
*free_segments(odp_packet_hdr_t *pkt_hdr,
for (i = 0, nfree = 0; i < num; i++) {
new_hdr = pkt_hdr->buf_hdr.seg[i].hdr;
- if (packet_ref_dec(new_hdr) == 1)
+ if (packet_ref_count(new_hdr) == 1 ||
+ packet_ref_dec(new_hdr) == 1)
buf[nfree++] = buffer_handle(new_hdr);
}
@@ -643,12 +644,12 @@ static inline odp_packet_hdr_t
*packet_free_to_list(odp_packet_hdr_t *pkt_hdr,
int num_seg, i;
do {
- ref_count = packet_ref_count(pkt_hdr) - 1;
+ ref_count = packet_ref_count(pkt_hdr);
num_seg = pkt_hdr->buf_hdr.segcount;
ref_hdr = pkt_hdr->ref_hdr;
if (odp_likely((CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1) &&
- ref_count == 0)) {
+ ref_count == 1)) {
if (*nfree >= nbufs)
break;
@@ -661,11 +662,12 @@ static inline odp_packet_hdr_t
*packet_free_to_list(odp_packet_hdr_t *pkt_hdr,
odp_packet_hdr_t *hdr =
pkt_hdr->buf_hdr.seg[i].hdr;
- if (packet_ref_dec(hdr) == 1)
+ if (packet_ref_count(hdr) == 1 ||
+ packet_ref_dec(hdr) == 1)
buf[(*nfree)++] = buffer_handle(hdr);
}
- if (ref_count == 1)
+ if (ref_count == 2)
pkt_hdr->unshared_len = pkt_hdr->frame_len;
}
@@ -682,18 +684,18 @@ static inline void packet_free(odp_packet_hdr_t *pkt_hdr)
int num_seg;
do {
- ref_count = packet_ref_count(pkt_hdr) - 1;
+ ref_count = packet_ref_count(pkt_hdr);
num_seg = pkt_hdr->buf_hdr.segcount;
ref_hdr = pkt_hdr->ref_hdr;
if (odp_likely((CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1) &&
- ref_count == 0)) {
+ ref_count == 1)) {
buffer_free_multi((odp_buffer_t *)
&pkt_hdr->buf_hdr.handle.handle, 1);
} else {
free_bufs(pkt_hdr, 0, num_seg);
- if (ref_count == 1)
+ if (ref_count == 2)
pkt_hdr->unshared_len = pkt_hdr->frame_len;
}
--
2.12.0.rc1
