On Tue, Jan 27, 2015 at 4:44 PM, Geoffrey Blake <geoffrey.bl...@arm.com>
wrote:
> From: Geoffrey Blake <geoffrey.bl...@arm.com>
>
> Signed-off-by: Geoffrey Blake <geoffrey.bl...@arm.com>
>
Reviewed-and-tested-by: Bill Fischofer <bill.fischo...@linaro.org>
> ---
> (This document/code contribution attached is provided under the terms of
> agreement LES-LTM-21309)
> .../include/odp_buffer_pool_internal.h | 100
> ++++++++++-----------
> platform/linux-generic/odp_buffer_pool.c | 6 +-
> 2 files changed, 52 insertions(+), 54 deletions(-)
>
> diff --git a/platform/linux-generic/include/odp_buffer_pool_internal.h
> b/platform/linux-generic/include/odp_buffer_pool_internal.h
> index d6f44d9..1cc4898 100644
> --- a/platform/linux-generic/include/odp_buffer_pool_internal.h
> +++ b/platform/linux-generic/include/odp_buffer_pool_internal.h
> @@ -107,8 +107,8 @@ struct pool_entry_s {
> size_t pool_size;
> uint32_t buf_align;
> uint32_t buf_stride;
> - _odp_atomic_ptr_t buf_freelist;
> - _odp_atomic_ptr_t blk_freelist;
> + _odp_atomic_tptr_t buf_freelist;
> + _odp_atomic_tptr_t blk_freelist;
> odp_atomic_u32_t bufcount;
> odp_atomic_u32_t blkcount;
> odp_atomic_u64_t bufallocs;
> @@ -142,58 +142,55 @@ extern void *pool_entry_ptr[];
> #define pool_is_secure(pool) 0
> #endif
>
> -#define TAG_ALIGN ((size_t)16)
> +#define odp_retag_tptr(ptr) \
> + _odp_atomic_tptr_settag(&ptr, _odp_atomic_tptr_gettag(&ptr) + 1)
>
> -#define odp_cs(ptr, old, new) \
> - _odp_atomic_ptr_cmp_xchg_strong(&ptr, (void **)&old, (void *)new, \
> - _ODP_MEMMODEL_SC, \
> - _ODP_MEMMODEL_SC)
> -
> -/* Helper functions for pointer tagging to avoid ABA race conditions */
> -#define odp_tag(ptr) \
> - (((size_t)ptr) & (TAG_ALIGN - 1))
> +#define odp_get_tptr(ptr) \
> + _odp_atomic_tptr_getptr(&ptr)
>
> -#define odp_detag(ptr) \
> - ((void *)(((size_t)ptr) & -TAG_ALIGN))
> -
> -#define odp_retag(ptr, tag) \
> - ((void *)(((size_t)ptr) | odp_tag(tag)))
> +#define odp_set_tptr(ptr, newptr) \
> + _odp_atomic_tptr_setptr(&ptr, newptr)
>
> +#define odp_cs_tptr(ptr, old, new) \
> + _odp_atomic_tptr_cmp_xchg_strong(&ptr, &old, &new, \
> + _ODP_MEMMODEL_SC, \
> + _ODP_MEMMODEL_SC)
>
> static inline void *get_blk(struct pool_entry_s *pool)
> {
> - void *oldhead, *myhead, *newhead;
> + _odp_atomic_tptr_t oldhead, newhead;
>
> - oldhead = _odp_atomic_ptr_load(&pool->blk_freelist,
> _ODP_MEMMODEL_ACQ);
> + oldhead = _odp_atomic_tptr_load(&pool->blk_freelist,
> _ODP_MEMMODEL_ACQ);
>
> do {
> - size_t tag = odp_tag(oldhead);
> - myhead = odp_detag(oldhead);
> - if (odp_unlikely(myhead == NULL))
> + if (odp_unlikely(odp_get_tptr(oldhead) == NULL))
> break;
> - newhead = odp_retag(((odp_buf_blk_t *)myhead)->next, tag +
> 1);
> - } while (odp_cs(pool->blk_freelist, oldhead, newhead) == 0);
> + newhead = oldhead;
> + odp_set_tptr(newhead,
> + ((odp_buf_blk_t
> *)odp_get_tptr(oldhead))->next);
> + odp_retag_tptr(newhead);
> + } while (odp_cs_tptr(pool->blk_freelist, oldhead, newhead) == 0);
>
> - if (odp_unlikely(myhead == NULL))
> + if (odp_unlikely(odp_get_tptr(oldhead) == NULL))
> odp_atomic_inc_u64(&pool->blkempty);
> else
> odp_atomic_dec_u32(&pool->blkcount);
>
> - return (void *)myhead;
> + return (void *)odp_get_tptr(oldhead);
> }
>
> static inline void ret_blk(struct pool_entry_s *pool, void *block)
> {
> - void *oldhead, *myhead, *myblock;
> + _odp_atomic_tptr_t oldhead, myblock;
>
> - oldhead = _odp_atomic_ptr_load(&pool->blk_freelist,
> _ODP_MEMMODEL_ACQ);
> + oldhead = _odp_atomic_tptr_load(&pool->blk_freelist,
> _ODP_MEMMODEL_ACQ);
>
> do {
> - size_t tag = odp_tag(oldhead);
> - myhead = odp_detag(oldhead);
> - ((odp_buf_blk_t *)block)->next = myhead;
> - myblock = odp_retag(block, tag + 1);
> - } while (odp_cs(pool->blk_freelist, oldhead, myblock) == 0);
> + myblock = oldhead;
> + ((odp_buf_blk_t *)block)->next = odp_get_tptr(oldhead);
> + odp_set_tptr(myblock, block);
> + odp_retag_tptr(myblock);
> + } while (odp_cs_tptr(pool->blk_freelist, oldhead, myblock) == 0);
>
> odp_atomic_inc_u32(&pool->blkcount);
> odp_atomic_inc_u64(&pool->blkfrees);
> @@ -201,23 +198,26 @@ static inline void ret_blk(struct pool_entry_s
> *pool, void *block)
>
> static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool)
> {
> - odp_buffer_hdr_t *oldhead, *myhead, *newhead;
> + _odp_atomic_tptr_t oldhead, newhead;
> + odp_buffer_hdr_t *mybuf = NULL;
>
> - oldhead = _odp_atomic_ptr_load(&pool->buf_freelist,
> _ODP_MEMMODEL_ACQ);
> + oldhead = _odp_atomic_tptr_load(&pool->buf_freelist,
> _ODP_MEMMODEL_ACQ);
>
> do {
> - size_t tag = odp_tag(oldhead);
> - myhead = odp_detag(oldhead);
> - if (odp_unlikely(myhead == NULL))
> + newhead = oldhead;
> + if (odp_unlikely(odp_get_tptr(oldhead) == NULL))
> break;
> - newhead = odp_retag(myhead->next, tag + 1);
> - } while (odp_cs(pool->buf_freelist, oldhead, newhead) == 0);
> + odp_set_tptr(newhead,
> + ((odp_buffer_hdr_t
> *)odp_get_tptr(oldhead))->next);
> + odp_retag_tptr(newhead);
> + } while (odp_cs_tptr(pool->buf_freelist, oldhead, newhead) == 0);
>
> - if (odp_unlikely(myhead == NULL)) {
> + if (odp_unlikely(odp_get_tptr(oldhead) == NULL)) {
> odp_atomic_inc_u64(&pool->bufempty);
> } else {
> uint64_t bufcount =
> odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - 1;
> + mybuf = odp_get_tptr(oldhead);
>
> /* Check for low watermark condition */
> if (bufcount == pool->low_wm && !pool->low_wm_assert) {
> @@ -226,16 +226,16 @@ static inline odp_buffer_hdr_t *get_buf(struct
> pool_entry_s *pool)
> }
>
> odp_atomic_inc_u64(&pool->bufallocs);
> - myhead->next = myhead; /* Mark buffer allocated */
> - myhead->allocator = odp_thread_id();
> + mybuf->next = mybuf; /* Mark buffer allocated */
> + mybuf->allocator = odp_thread_id();
> }
>
> - return (void *)myhead;
> + return (void *)mybuf;
> }
>
> static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t
> *buf)
> {
> - odp_buffer_hdr_t *oldhead, *myhead, *mybuf;
> + _odp_atomic_tptr_t oldhead, newhead;
>
> buf->allocator = ODP_FREEBUF; /* Mark buffer free */
>
> @@ -249,14 +249,14 @@ static inline void ret_buf(struct pool_entry_s
> *pool, odp_buffer_hdr_t *buf)
> buf->size = 0;
> }
>
> - oldhead = _odp_atomic_ptr_load(&pool->buf_freelist,
> _ODP_MEMMODEL_ACQ);
> + oldhead = _odp_atomic_tptr_load(&pool->buf_freelist,
> _ODP_MEMMODEL_ACQ);
>
> do {
> - size_t tag = odp_tag(oldhead);
> - myhead = odp_detag(oldhead);
> - buf->next = myhead;
> - mybuf = odp_retag(buf, tag + 1);
> - } while (odp_cs(pool->buf_freelist, oldhead, mybuf) == 0);
> + newhead = oldhead;
> + buf->next = odp_get_tptr(oldhead);
> + odp_set_tptr(newhead, buf);
> + odp_retag_tptr(newhead);
> + } while (odp_cs_tptr(pool->buf_freelist, oldhead, newhead) == 0);
>
> uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, 1) +
> 1;
>
> diff --git a/platform/linux-generic/odp_buffer_pool.c
> b/platform/linux-generic/odp_buffer_pool.c
> index ac12057..6e35197 100644
> --- a/platform/linux-generic/odp_buffer_pool.c
> +++ b/platform/linux-generic/odp_buffer_pool.c
> @@ -291,10 +291,8 @@ odp_pool_t odp_pool_create(const char *name,
> pool->s.pool_mdata_addr = mdata_base_addr;
>
> pool->s.buf_stride = buf_stride;
> - _odp_atomic_ptr_store(&pool->s.buf_freelist, NULL,
> - _ODP_MEMMODEL_RLX);
> - _odp_atomic_ptr_store(&pool->s.blk_freelist, NULL,
> - _ODP_MEMMODEL_RLX);
> + _odp_atomic_tptr_init(&pool->s.buf_freelist, NULL);
> + _odp_atomic_tptr_init(&pool->s.blk_freelist, NULL);
>
> /* Initialization will increment these to their target
> vals */
> odp_atomic_store_u32(&pool->s.bufcount, 0);
> --
> 1.9.1
>
>
>
> _______________________________________________
> lng-odp mailing list
> lng-odp@lists.linaro.org
> http://lists.linaro.org/mailman/listinfo/lng-odp
>
_______________________________________________
lng-odp mailing list
lng-odp@lists.linaro.org
http://lists.linaro.org/mailman/listinfo/lng-odp
