Round up global pool allocations to a burst size. Cache any
extra buffers for future use. Prefetch buffers header which
very newly allocated from global pool and will be returned to
the caller.
Signed-off-by: Petri Savolainen <petri.savolai...@nokia.com>
---
.../linux-generic/include/odp_buffer_internal.h | 3 +-
platform/linux-generic/odp_packet.c | 16 +++--
platform/linux-generic/odp_pool.c | 74 ++++++++++++++++------
3 files changed, 68 insertions(+), 25 deletions(-)
diff --git a/platform/linux-generic/include/odp_buffer_internal.h
b/platform/linux-generic/include/odp_buffer_internal.h
index abe8591..64ba221 100644
--- a/platform/linux-generic/include/odp_buffer_internal.h
+++ b/platform/linux-generic/include/odp_buffer_internal.h
@@ -105,7 +105,8 @@ struct odp_buffer_hdr_t {
};
/* Forward declarations */
-int buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num);
+int buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[],
+ odp_buffer_hdr_t *buf_hdr[], int num);
void buffer_free_multi(const odp_buffer_t buf[], int num_free);
int seg_alloc_head(odp_buffer_hdr_t *buf_hdr, int segcount);
diff --git a/platform/linux-generic/odp_packet.c
b/platform/linux-generic/odp_packet.c
index 6df1c5b..6565a5d 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -80,14 +80,16 @@ static void packet_init(pool_t *pool, odp_packet_hdr_t
*pkt_hdr,
int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len,
odp_packet_t pkt[], int max_num)
{
- odp_packet_hdr_t *pkt_hdr;
pool_t *pool = odp_pool_to_entry(pool_hdl);
int num, i;
+ odp_packet_hdr_t *pkt_hdrs[max_num];
- num = buffer_alloc_multi(pool_hdl, (odp_buffer_t *)pkt, max_num);
+ num = buffer_alloc_multi(pool_hdl, (odp_buffer_t *)pkt,
+ (odp_buffer_hdr_t **)pkt_hdrs, max_num);
for (i = 0; i < num; i++) {
- pkt_hdr = odp_packet_hdr(pkt[i]);
+ odp_packet_hdr_t *pkt_hdr = pkt_hdrs[i];
+
packet_init(pool, pkt_hdr, len, 1 /* do parse */);
if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize)
@@ -113,7 +115,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t
len)
if (odp_unlikely(len > pool->max_len))
return ODP_PACKET_INVALID;
- ret = buffer_alloc_multi(pool_hdl, (odp_buffer_t *)&pkt, 1);
+ ret = buffer_alloc_multi(pool_hdl, (odp_buffer_t *)&pkt, NULL, 1);
if (ret != 1)
return ODP_PACKET_INVALID;
@@ -134,6 +136,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t
len,
pool_t *pool = odp_pool_to_entry(pool_hdl);
size_t pkt_size = len ? len : pool->data_size;
int count, i;
+ odp_packet_hdr_t *pkt_hdrs[num];
if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) {
__odp_errno = EINVAL;
@@ -143,10 +146,11 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t
len,
if (odp_unlikely(len > pool->max_len))
return -1;
- count = buffer_alloc_multi(pool_hdl, (odp_buffer_t *)pkt, num);
+ count = buffer_alloc_multi(pool_hdl, (odp_buffer_t *)pkt,
+ (odp_buffer_hdr_t **)pkt_hdrs, num);
for (i = 0; i < count; ++i) {
- odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt[i]);
+ odp_packet_hdr_t *pkt_hdr = pkt_hdrs[i];
packet_init(pool, pkt_hdr, pkt_size, 0 /* do not parse */);
if (len == 0)
diff --git a/platform/linux-generic/odp_pool.c
b/platform/linux-generic/odp_pool.c
index a2e5d54..7dc0938 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -562,14 +562,14 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t
*info)
return 0;
}
-int buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int max_num)
+int buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[],
+ odp_buffer_hdr_t *buf_hdr[], int max_num)
{
pool_t *pool;
ring_t *ring;
- uint32_t mask;
- int i;
+ uint32_t mask, i;
pool_cache_t *cache;
- uint32_t cache_num;
+ uint32_t cache_num, num_ch, num_deq, burst;
pool = pool_entry_from_hdl(pool_hdl);
ring = &pool->ring.hdr;
@@ -577,28 +577,66 @@ int buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t
buf[], int max_num)
cache = local.cache[_odp_typeval(pool_hdl)];
cache_num = cache->num;
+ num_ch = max_num;
+ num_deq = 0;
+ burst = CACHE_BURST;
- if (odp_likely((int)cache_num >= max_num)) {
- for (i = 0; i < max_num; i++)
- buf[i] = cache->buf[cache_num - max_num + i];
+ if (odp_unlikely(cache_num < (uint32_t)max_num)) {
+ /* Cache does not have enough buffers */
+ num_ch = cache_num;
+ num_deq = max_num - cache_num;
- cache->num = cache_num - max_num;
- return max_num;
+ if (odp_unlikely(num_deq > CACHE_BURST))
+ burst = num_deq;
}
- {
+ /* Get buffers from the cache */
+ for (i = 0; i < num_ch; i++)
+ buf[i] = cache->buf[cache_num - num_ch + i];
+
+ /* If needed, get more from the global pool */
+ if (odp_unlikely(num_deq)) {
/* Temporary copy needed since odp_buffer_t is uintptr_t
* and not uint32_t. */
- int num;
- uint32_t data[max_num];
+ uint32_t data[burst];
- num = ring_deq_multi(ring, mask, data, max_num);
+ burst = ring_deq_multi(ring, mask, data, burst);
+ cache_num = burst - num_deq;
- for (i = 0; i < num; i++)
- buf[i] = (odp_buffer_t)(uintptr_t)data[i];
+ if (odp_unlikely(burst < num_deq)) {
+ num_deq = burst;
+ cache_num = 0;
+ }
+
+ for (i = 0; i < num_deq; i++) {
+ uint32_t idx = num_ch + i;
+
+ buf[idx] = (odp_buffer_t)(uintptr_t)data[i];
+
+ if (buf_hdr) {
+ buf_hdr[idx] = odp_buf_to_hdr(buf[idx]);
+ /* Prefetch newly allocated and soon to be used
+ * buffer headers. */
+ odp_prefetch(buf_hdr[idx]);
+ }
+ }
+
+ /* Cache extra buffers. Cache is currently empty. */
+ for (i = 0; i < cache_num; i++)
+ cache->buf[i] = (odp_buffer_t)
+ (uintptr_t)data[num_deq + i];
+
+ cache->num = cache_num;
+ } else {
+ cache->num = cache_num - num_ch;
+ }
+
+ if (buf_hdr) {
+ for (i = 0; i < num_ch; i++)
+ buf_hdr[i] = odp_buf_to_hdr(buf[i]);
}
- return i;
+ return num_ch + num_deq;
}
static inline void buffer_free_to_pool(uint32_t pool_id,
@@ -694,7 +732,7 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl)
odp_buffer_t buf;
int ret;
- ret = buffer_alloc_multi(pool_hdl, &buf, 1);
+ ret = buffer_alloc_multi(pool_hdl, &buf, NULL, 1);
if (odp_likely(ret == 1))
return buf;
@@ -704,7 +742,7 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl)
int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num)
{
- return buffer_alloc_multi(pool_hdl, buf, num);
+ return buffer_alloc_multi(pool_hdl, buf, NULL, num);
}
void odp_buffer_free(odp_buffer_t buf)
--
2.8.1
