On 2013-11-06 08:15:59 -0500, Robert Haas wrote: > > The API is described at: http://en.cppreference.com/w/c/atomic > > > > The fundamental difference to what I've suggested so far is that the > > atomic variable isn't a plain integer/type but a distinct datatype that > > can *only* be manipulated via the atomic operators. That has the nice > > property that it cannot be accidentally manipulated via plain operators. > > > > E.g. it wouldn't be > > uint32 pg_atomic_fetch_add_u32(uint32 *ptr, uint32 add_); > > but > > uint32 pg_atomic_fetch_add_u32(pg_atomic_uint32 *ptr, uint32 add_); > > > > where, for now, atomic_uint32 is something like: > > typedef struct pg_atomic_uint32 > > { > > volatile uint32 value; > > } pg_atomic_uint32; > > a future C11 implementation would just typedef C11's respective atomic > > type to pg_atomic_uint32. > > > > Comments? > > Sadly, I don't have a clear feeling for how well or poorly that's > going to work out.
I've implemented it that way and it imo looks sensible. I dislike the pg_atomic_init_(flag|u32|u64) calls that are forced on us by wanting to be compatible with C11, but I think that doesn't end up being too bad. So, attached is a very first draft of an atomics implementation. There's lots to be done, but this is how I think it should roughly look like and what things we should implement in the beginning. The API should be understandable from looking at src/include/storage/atomics.h I haven't tested the IBM xlc, HPUX IA64 acc, Sunpro fallback at all, but the important part is that those provide the necessary tools to implement everything. Also, even the gcc implementation falls back to compare_and_swap() based implementations for the operations, but that shouldn't matter for now. I haven't looked for other compilers yet. Questions: * Fundamental issues with the API? * should we split of compiler/arch specific parts of into include/ports or such? -0.5 from me. * should we add src/include/storage/atomics subdirectory? +0.5 from me. * Do we really want to continue to cater to compilers not supporting PG_USE_INLINE? I've tried to add support for them, but it does make things considerably more #ifdef-y. Afaik it's only HPUX's acc that has problems, and it seems to work but generate warnings, so maybe that's ok? * Which additional compilers do we want to support? icc (might be gcc compatible)? Greetings, Andres Freund -- Andres Freund http://www.2ndQuadrant.com/ PostgreSQL Development, 24x7 Support, Training & Services
>From 04a8dc8516323eb404a7a3392f950763497a76ec Mon Sep 17 00:00:00 2001 From: Andres Freund <and...@anarazel.de> Date: Wed, 6 Nov 2013 10:32:53 +0100 Subject: [PATCH 2/3] Very basic atomic ops implementation Only gcc has been tested, stub implementations for * msvc * HUPX acc * IBM xlc * Sun/Oracle Sunpro compiler are included. All implementations only provide the bare minimum of operations for now and rely on emulating more complex operations via compare_and_swap(). --- config/c-compiler.m4 | 77 +++++ configure.in | 19 +- src/backend/storage/lmgr/Makefile | 2 +- src/backend/storage/lmgr/atomics.c | 23 ++ src/include/c.h | 7 + src/include/pg_config.h.in | 11 +- src/include/storage/atomics-arch-arm.h | 29 ++ src/include/storage/atomics-generic-acc.h | 99 +++++++ src/include/storage/atomics-generic-gcc.h | 154 ++++++++++ src/include/storage/atomics-generic-msvc.h | 58 ++++ src/include/storage/atomics-generic-sunpro.h | 66 +++++ src/include/storage/atomics-generic-xlc.h | 84 ++++++ src/include/storage/atomics-generic.h | 290 +++++++++++++++++++ src/include/storage/atomics.h | 415 +++++++++++++++++++++++++++ src/include/storage/s_lock.h | 10 +- 15 files changed, 1325 insertions(+), 19 deletions(-) create mode 100644 src/backend/storage/lmgr/atomics.c create mode 100644 src/include/storage/atomics-arch-arm.h create mode 100644 src/include/storage/atomics-generic-acc.h create mode 100644 src/include/storage/atomics-generic-gcc.h create mode 100644 src/include/storage/atomics-generic-msvc.h create mode 100644 src/include/storage/atomics-generic-sunpro.h create mode 100644 src/include/storage/atomics-generic-xlc.h create mode 100644 src/include/storage/atomics-generic.h create mode 100644 src/include/storage/atomics.h diff --git a/config/c-compiler.m4 b/config/c-compiler.m4 index 4ba3236..ae92281 100644 --- a/config/c-compiler.m4 +++ b/config/c-compiler.m4 @@ -289,3 +289,80 @@ if test x"$Ac_cachevar" = x"yes"; then fi undefine([Ac_cachevar])dnl ])# PGAC_PROG_CC_LDFLAGS_OPT + +AC_DEFUN([PGAC_HAVE_GCC__SYNC_INT32_ATOMICS], +[AC_CACHE_CHECK(for __builtin_constant_p, pgac_cv__builtin_constant_p, +[AC_TRY_COMPILE([static int x; static int y[__builtin_constant_p(x) ? x : 1];], +[], +[pgac_cv__builtin_constant_p=yes], +[pgac_cv__builtin_constant_p=no])]) +if test x"$pgac_cv__builtin_constant_p" = xyes ; then +AC_DEFINE(HAVE__BUILTIN_CONSTANT_P, 1, + [Define to 1 if your compiler understands __builtin_constant_p.]) +fi])# PGAC_C_BUILTIN_CONSTANT_P + +# PGAC_HAVE_GCC__SYNC_INT32_TAS +# ------------------------- +# Check if the C compiler understands __sync_lock_test_and_set(), +# and define HAVE_GCC__SYNC_INT32_TAS +# +# NB: There are platforms where test_and_set is available but compare_and_swap +# is not, so test this separa +AC_DEFUN([PGAC_HAVE_GCC__SYNC_INT32_TAS], +[AC_CACHE_CHECK(for builtin locking functions, pgac_cv_gcc_sync_int32_tas, +[AC_TRY_LINK([], + [int lock = 0; + __sync_lock_test_and_set(&lock, 1); + __sync_lock_release(&lock);], + [pgac_cv_gcc_sync_int32_tas="yes"], + [pgac_cv_gcc_sync_int32_tas="no"])]) +if test x"$pgac_cv_gcc_sync_int32_tas" = x"yes"; then + AC_DEFINE(HAVE_GCC__SYNC_INT32_TAS, 1, [Define to 1 if you have __sync_lock_test_and_set(int *) and friends.]) +fi])# PGAC_HAVE_GCC__SYNC_INT32_TAS + +# PGAC_HAVE_GCC__SYNC_INT32_CAS +# ------------------------- +# Check if the C compiler understands __sync_compare_and_swap() for 32bit +# types, and define HAVE_GCC__SYNC_INT32_CAS if so. +AC_DEFUN([PGAC_HAVE_GCC__SYNC_INT32_CAS], +[AC_CACHE_CHECK(for builtin __sync int32 atomic operations, pgac_cv_gcc_sync_int32_cas, +[AC_TRY_LINK([], + [int val = 0; + __sync_val_compare_and_swap(&val, 0, 37);], + [pgac_cv_gcc_sync_int32_cas="yes"], + [pgac_cv_gcc_sync_int32_cas="no"])]) +if test x"$pgac_cv_gcc_sync_int32_cas" = x"yes"; then + AC_DEFINE(HAVE_GCC__SYNC_INT32_CAS, 1, [Define to 1 if you have __sync_compare_and_swap(int *, int, int).]) +fi])# PGAC_HAVE_GCC__SYNC_INT32_CAS + +# PGAC_HAVE_GCC__SYNC_INT32_CAS +# ------------------------- +# Check if the C compiler understands __sync_compare_and_swap() for 64bit +# types, and define HAVE_GCC__SYNC_INT64_CAS if so. +AC_DEFUN([PGAC_HAVE_GCC__SYNC_INT64_CAS], +[AC_CACHE_CHECK(for builtin __sync int64 atomic operations, pgac_cv_gcc_sync_int64_cas, +[AC_TRY_LINK([], + [PG_INT64_TYPE lock = 0; + __sync_val_compare_and_swap(&lock, 0, (PG_INT64_TYPE) 37);], + [pgac_cv_gcc_sync_int64_cas="yes"], + [pgac_cv_gcc_sync_int64_cas="no"])]) +if test x"$pgac_cv_gcc_sync_int64_cas" = x"yes"; then + AC_DEFINE(HAVE_GCC__SYNC_INT64_CAS, 1, [Define to 1 if you have __sync_compare_and_swap(int64 *, int64, int64).]) +fi])# PGAC_HAVE_GCC__SYNC_INT64_CAS + +# PGAC_HAVE_GCC__ATOMIC_INT32_CAS +# ------------------------- + +# Check if the C compiler understands __atomic_compare_exchange_n() for 32bit +# types, and define HAVE_GCC__ATOMIC_INT32_CAS if so. +AC_DEFUN([PGAC_HAVE_GCC__ATOMIC_INT32_CAS], +[AC_CACHE_CHECK(for builtin __atomic int32 atomic operations, pgac_cv_gcc_atomic_int32_cas, +[AC_TRY_LINK([], + [int val = 0; + int expect = 0; + __atomic_compare_exchange_n(&val, &expect, 37, 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED);], + [pgac_cv_gcc_atomic_int32_cas="yes"], + [pgac_cv_gcc_atomic_int32_cas="no"])]) +if test x"$pgac_cv_gcc_atomic_int32_cas" = x"yes"; then + AC_DEFINE(HAVE_GCC__ATOMIC_INT32_CAS, 1, [Define to 1 if you have __atomic_compare_exchange_n(int *, int *, int).]) +fi])# PGAC_HAVE_GCC__ATOMIC_INT32_CAS diff --git a/configure.in b/configure.in index a4baeaf..43c663a 100644 --- a/configure.in +++ b/configure.in @@ -19,7 +19,7 @@ m4_pattern_forbid(^PGAC_)dnl to catch undefined macros AC_INIT([PostgreSQL], [9.4devel], [pgsql-b...@postgresql.org]) -m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.63], [], [m4_fatal([Autoconf version 2.63 is required. +m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.63 is required. Untested combinations of 'autoconf' and PostgreSQL versions are not recommended. You can remove the check from 'configure.in' but it is then your responsibility whether the result works or not.])]) @@ -1453,17 +1453,6 @@ fi AC_CHECK_FUNCS([strtoll strtoq], [break]) AC_CHECK_FUNCS([strtoull strtouq], [break]) -AC_CACHE_CHECK([for builtin locking functions], pgac_cv_gcc_int_atomics, -[AC_TRY_LINK([], - [int lock = 0; - __sync_lock_test_and_set(&lock, 1); - __sync_lock_release(&lock);], - [pgac_cv_gcc_int_atomics="yes"], - [pgac_cv_gcc_int_atomics="no"])]) -if test x"$pgac_cv_gcc_int_atomics" = x"yes"; then - AC_DEFINE(HAVE_GCC_INT_ATOMICS, 1, [Define to 1 if you have __sync_lock_test_and_set(int *) and friends.]) -fi - # Lastly, restore full LIBS list and check for readline/libedit symbols LIBS="$LIBS_including_readline" @@ -1738,6 +1727,12 @@ AC_CHECK_TYPES([int8, uint8, int64, uint64], [], [], # C, but is missing on some old platforms. AC_CHECK_TYPES(sig_atomic_t, [], [], [#include <signal.h>]) +# Check for various atomic operations now that we have checked how to declare +# 64bit integers. +PGAC_HAVE_GCC__SYNC_INT32_TAS +PGAC_HAVE_GCC__SYNC_INT32_CAS +PGAC_HAVE_GCC__SYNC_INT64_CAS +PGAC_HAVE_GCC__ATOMIC_INT32_CAS if test "$PORTNAME" != "win32" then diff --git a/src/backend/storage/lmgr/Makefile b/src/backend/storage/lmgr/Makefile index e12a854..be95d50 100644 --- a/src/backend/storage/lmgr/Makefile +++ b/src/backend/storage/lmgr/Makefile @@ -12,7 +12,7 @@ subdir = src/backend/storage/lmgr top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = lmgr.o lock.o proc.o deadlock.o lwlock.o spin.o s_lock.o predicate.o +OBJS = atomics.o lmgr.o lock.o proc.o deadlock.o lwlock.o spin.o s_lock.o predicate.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/storage/lmgr/atomics.c b/src/backend/storage/lmgr/atomics.c new file mode 100644 index 0000000..a00f29a --- /dev/null +++ b/src/backend/storage/lmgr/atomics.c @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------- + * + * atomics.c + * Non-Inline parts of the atomics implementation + * + * Portions Copyright (c) 2013, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/storage/lmgr/atomics.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +/* + * We want the functions below to be inline; but if the compiler doesn't + * support that, fall back on providing them as regular functions. See + * STATIC_IF_INLINE in c.h. + */ +#define ATOMICS_INCLUDE_DEFINITIONS + +#include "storage/atomics.h" diff --git a/src/include/c.h b/src/include/c.h index 6e19c6d..347f394 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -866,6 +866,13 @@ typedef NameData *Name; #define STATIC_IF_INLINE #endif /* PG_USE_INLINE */ +#ifdef PG_USE_INLINE +#define STATIC_IF_INLINE_DECLARE static inline +#else +#define STATIC_IF_INLINE_DECLARE extern +#endif /* PG_USE_INLINE */ + + /* ---------------------------------------------------------------- * Section 8: random stuff * ---------------------------------------------------------------- diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 5eac52d..3dd76a2 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -173,8 +173,17 @@ /* Define to 1 if your compiler understands __FUNCTION__. */ #undef HAVE_FUNCNAME__FUNCTION +/* Define to 1 if you have __atomic_compare_exchange_n(int *, int *, int). */ +#undef HAVE_GCC__ATOMIC_INT32_CAS + +/* Define to 1 if you have __sync_compare_and_swap(int *, int, int). */ +#undef HAVE_GCC__SYNC_INT32_CAS + /* Define to 1 if you have __sync_lock_test_and_set(int *) and friends. */ -#undef HAVE_GCC_INT_ATOMICS +#undef HAVE_GCC__SYNC_INT32_TAS + +/* Define to 1 if you have __sync_compare_and_swap(int64 *, int64, int64). */ +#undef HAVE_GCC__SYNC_INT64_CAS /* Define to 1 if you have the `getaddrinfo' function. */ #undef HAVE_GETADDRINFO diff --git a/src/include/storage/atomics-arch-arm.h b/src/include/storage/atomics-arch-arm.h new file mode 100644 index 0000000..a00f8f5 --- /dev/null +++ b/src/include/storage/atomics-arch-arm.h @@ -0,0 +1,29 @@ +/*------------------------------------------------------------------------- + * + * atomics-arch-arm.h + * Atomic operations considerations specific to ARM + * + * Portions Copyright (c) 2013, PostgreSQL Global Development Group + * + * NOTES: + * + * src/include/storage/atomics-arch-arm.h + * + *------------------------------------------------------------------------- + */ + +/* intentionally no include guards, should only be included by atomics.h */ +#ifndef INSIDE_ATOMICS_H +#error "should be included via atomics.h" +#endif + + +/* + * FIXME: Disable 32bit atomics for ARMs before v6 or error out + */ + +/* + * 64 bit atomics on arm are implemented using kernel fallbacks and might be + * slow, so disable entirely for now. + */ +#define PG_DISABLE_64_BIT_ATOMICS diff --git a/src/include/storage/atomics-generic-acc.h b/src/include/storage/atomics-generic-acc.h new file mode 100644 index 0000000..680d621 --- /dev/null +++ b/src/include/storage/atomics-generic-acc.h @@ -0,0 +1,99 @@ +/*------------------------------------------------------------------------- + * + * atomics-generic-acc.h + * Atomic operations support when using HPs acc on HPUX + * + * Portions Copyright (c) 2013, PostgreSQL Global Development Group + * + * NOTES: + * + * Documentation: + * * inline assembly for Itanium-based HP-UX: + * http://h21007.www2.hp.com/portal/download/files/unprot/Itanium/inline_assem_ERS.pdf + * * Implementing Spinlocks on the Intel ® Itanium ® Architecture and PA-RISC + * http://h21007.www2.hp.com/portal/download/files/unprot/itanium/spinlocks.pdf + * + * Itanium only supports a small set of numbers (6, -8, -4, -1, 1, 4, 8, 16) + * for atomic add/sub, so we just implement everything but compare_exchange via + * the compare_exchange fallbacks in atomics-generic.h. + * + * src/include/storage/atomics-generic-acc.h + * + * ------------------------------------------------------------------------- + */ + +#include <machine/sys/inline.h> + +/* IA64 always has 32/64 bit atomics */ + +typedef struct atomic_uint32 +{ + volatile uint32 value; +} atomic_uint32; + +typedef atomic_uint32 atomic_flag; + +typedef struct atomic_uint64 +{ + volatile uint64 value; +} atomic_uint64; + +/* declare outside the defined(PG_USE_INLINE) for visibility */ +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 + + +#if defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) + +#define MINOR_FENCE (_Asm_fence) (_UP_CALL_FENCE | _UP_SYS_FENCE | \ + _DOWN_CALL_FENCE | _DOWN_SYS_FENCE ) + +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u32_impl(volatile atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + bool ret; + uint32 current; + + _Asm_mov_to_ar(_AREG_CCV, *expected, MINOR_FENCE); + /* + * We want a barrier, not just release/acquire semantics. + */ + _Asm_mf(); + /* + * Notes: + * DOWN_MEM_FENCE | _UP_MEM_FENCE prevents reordering by the compiler + */ + current = _Asm_cmpxchg(_SZ_W, /* word */ + _SEM_REL, + &ptr->value, + newval, _LDHINT_NONE, + _DOWN_MEM_FENCE | _UP_MEM_FENCE); + ret = current == *expected; + *expected = current; + return ret; +} + + +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u64_impl(volatile atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + bool ret; + uint64 current; + + _Asm_mov_to_ar(_AREG_CCV, *expected, MINOR_FENCE); + _Asm_mf(); + current = _Asm_cmpxchg(_SZ_D, /* doubleword */ + _SEM_REL, + &ptr->value, + newval, _LDHINT_NONE, + _DOWN_MEM_FENCE | _UP_MEM_FENCE); + ret = current == *expected; + *expected = current; + return ret; +} + +#undef MINOR_FENCE + +#endif /* defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) */ diff --git a/src/include/storage/atomics-generic-gcc.h b/src/include/storage/atomics-generic-gcc.h new file mode 100644 index 0000000..323c385 --- /dev/null +++ b/src/include/storage/atomics-generic-gcc.h @@ -0,0 +1,154 @@ +/*------------------------------------------------------------------------- + * + * atomics-generic-gcc.h + * Atomic operations, implemented using gcc (or compatible) intrinsics. + * + * Portions Copyright (c) 2013, PostgreSQL Global Development Group + * + * NOTES: + * + * Documentation: + * * Legacy __sync Built-in Functions for Atomic Memory Access + * http://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html + * * Built-in functions for memory model aware atomic operations + * http://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html + * + * src/include/storage/atomics-generic-gcc.h + * + *------------------------------------------------------------------------- + */ + +/* intentionally no include guards, should only be included by atomics.h */ +#ifndef INSIDE_ATOMICS_H +#error "should be included via atomics.h" +#endif + +/* declare outside the defined(PG_USE_INLINE) for visibility */ +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 + +typedef struct atomic_uint32 +{ + volatile uint32 value; +} atomic_uint32; + +typedef atomic_uint32 atomic_flag; + +#ifdef HAVE_GCC__ATOMIC_INT64_CAS +/* declare outside the defined(PG_USE_INLINE) for visibility */ +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 +typedef struct atomic_uint64 +{ + volatile uint64 value; +} atomic_uint64; +#endif + +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 + + +#if defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) + +#define PG_HAVE_ATOMIC_TEST_SET_FLAG +STATIC_IF_INLINE bool +pg_atomic_test_set_flag_impl(volatile atomic_flag *ptr) +{ + uint32 ret; + /* some platform only support a 1 here */ + ret = __sync_lock_test_and_set(&ptr->value, 1); + /* XXX: only a acquire barrier, make it a full one for now */ + pg_memory_barrier(); + return ret != 0; +} + +#define PG_HAVE_ATOMIC_CLEAR_FLAG +STATIC_IF_INLINE void +pg_atomic_clear_flag_impl(volatile atomic_flag *ptr) +{ + __sync_lock_release(&ptr->value); + /* XXX: only a release barrier, make it a full one for now */ + pg_memory_barrier(); +} + +STATIC_IF_INLINE uint32 +pg_atomic_read_u32(volatile atomic_uint32 *ptr) +{ + return ptr->value; +} + +STATIC_IF_INLINE void +pg_atomic_write_u32(volatile atomic_uint32 *ptr, uint32 val) +{ + ptr->value = val; +} + +#if defined(HAVE_GCC__ATOMIC_INT32_CAS) +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u32_impl(volatile atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +} + +#elif defined(HAVE_GCC__SYNC_INT32_CAS) +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u32_impl(volatile atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + bool ret; + uint32 current; + current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); + ret = current == *expected; + *expected = current; + return ret; +} + +#else +# error "strange configuration" +#endif + +#ifdef HAVE_GCC__ATOMIC_INT64_CAS + +STATIC_IF_INLINE uint64 +pg_atomic_read_u64(volatile atomic_uint64 *ptr) +{ + return ptr->value; +} + +STATIC_IF_INLINE void +pg_atomic_write_u64(volatile atomic_uint64 *ptr, uint64 val) +{ + ptr->value = val; +} + +/* if __atomic is supported for 32 it's also supported for 64bit */ +#if defined(HAVE_GCC__ATOMIC_INT32_CAS) +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u64_impl(volatile atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +} + +#elif defined(HAVE_GCC__SYNC_INT64_CAS) +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u64_impl(volatile atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + bool ret; + uint64 current; + current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); + ret = current == *expected; + *expected = current; + return ret; +} + +#else +# error "strange configuration" +#endif + +#endif /* HAVE_GCC__ATOMIC_INT64_CAS */ + +#endif /* defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) */ diff --git a/src/include/storage/atomics-generic-msvc.h b/src/include/storage/atomics-generic-msvc.h new file mode 100644 index 0000000..1fa17f3 --- /dev/null +++ b/src/include/storage/atomics-generic-msvc.h @@ -0,0 +1,58 @@ +/*------------------------------------------------------------------------- + * + * atomics-generic-msvc.h + * Atomic operations support when using MSVC + * + * Portions Copyright (c) 2013, PostgreSQL Global Development Group + * + * NOTES: + * + * Documentation: + * * Interlocked Variable Access + * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx + * + * src/include/storage/atomics-generic-msvc.h + * + *------------------------------------------------------------------------- + */ + +/* intentionally no include guards, should only be included by atomics.h */ +#ifndef INSIDE_ATOMICS_H +#error "should be included via atomics.h" +#endif + +#if defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) + +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u32_impl(volatile atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + bool ret; + uint32 current; + current = InterlockedCompareExchange(&ptr->value, newval, expected); + ret = current == *expected; + *expected = current; + return ret; +} + +/* + * FIXME: Only available on Vista/2003 onwards, add test for that. + * + * Only supported on >486, but we require XP as a minimum baseline, which + * doesn't support the 486, so we don't need to add checks for that case. + */ +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u64_impl(volatile atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + bool ret; + uint32 current; + current = InterlockedCompareExchange64(&ptr->value, newval, expected); + ret = current == *expected; + *expected = current; + return ret; +} + +#endif /* defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) */ diff --git a/src/include/storage/atomics-generic-sunpro.h b/src/include/storage/atomics-generic-sunpro.h new file mode 100644 index 0000000..c71014b --- /dev/null +++ b/src/include/storage/atomics-generic-sunpro.h @@ -0,0 +1,66 @@ +/*------------------------------------------------------------------------- + * + * atomics-generic-sunpro.h + * Atomic operations for solaris' CC + * + * Portions Copyright (c) 2013, PostgreSQL Global Development Group + * + * NOTES: + * + * Documentation: + * * manpage for atomic_cas(3C) + * http://www.unix.com/man-page/opensolaris/3c/atomic_cas/ + * http://docs.oracle.com/cd/E23824_01/html/821-1465/atomic-cas-3c.html + * + * src/include/storage/atomics-generic-sunpro.h + * + * ------------------------------------------------------------------------- + */ + +#include <atomic.h> + +typedef struct atomic_uint32 +{ + volatile uint32 value; +} atomic_uint32; + +typedef atomic_uint32 atomic_flag; + +typedef struct atomic_uint64 +{ + volatile uint64 value; +} atomic_uint64; + +/* declare outside the defined(PG_USE_INLINE) for visibility */ +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 + +#if defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) + +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u32_impl(volatile atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + bool ret; + uint64 current; + + current = atomic_cas_32(&ptr->value, *expected, newval); + ret = current == *expected; + *expected = current; + return ret; +} + +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u64_impl(volatile atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + bool ret; + uint64 current; + + current = atomic_cas_64(&ptr->value, *expected, newval); + ret = current == *expected; + *expected = current; + return ret; +} + +#endif diff --git a/src/include/storage/atomics-generic-xlc.h b/src/include/storage/atomics-generic-xlc.h new file mode 100644 index 0000000..c3b751c --- /dev/null +++ b/src/include/storage/atomics-generic-xlc.h @@ -0,0 +1,84 @@ +/*------------------------------------------------------------------------- + * + * atomics-generic-xlc.h + * Atomic operations for IBM's CC + * + * Portions Copyright (c) 2013, PostgreSQL Global Development Group + * + * NOTES: + * + * Documentation: + * * Synchronization and atomic built-in functions + * http://publib.boulder.ibm.com/infocenter/lnxpcomp/v8v101/topic/com.ibm.xlcpp8l.doc/compiler/ref/bif_sync.htm + * + * src/include/storage/atomics-generic-xlc.h + * + * ------------------------------------------------------------------------- + */ + +#include <atomic.h> + +typedef struct atomic_uint32 +{ + volatile uint32 value; +} atomic_uint32; + +typedef atomic_uint32 atomic_flag; + +typedef struct atomic_uint64 +{ + volatile uint64 value; +} atomic_uint64; + +/* declare outside the defined(PG_USE_INLINE) for visibility */ +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 + +/* FIXME: check for 64bit mode, only supported there */ +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 + +#if defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) + +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u32_impl(volatile atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + bool ret; + uint64 current; + + /* + * xlc's documentation tells us: + * "If __compare_and_swap is used as a locking primitive, insert a call to + * the __isync built-in function at the start of any critical sections." + */ + __isync(); + + /* + * XXX: __compare_and_swap is defined to take signed parameters, but that + * shouldn't matter since we don't perform any arithmetic operations. + */ + current = (uint32)__compare_and_swap((volatile int*)ptr->value, + (int)*expected, (int)newval); + ret = current == *expected; + *expected = current; + return ret; +} + +#ifdef PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u64_impl(volatile atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + bool ret; + uint64 current; + + __isync(); + + current = (uint64)__compare_and_swaplp((volatile long*)ptr->value, + (long)*expected, (long)newval); + ret = current == *expected; + *expected = current; + return ret; +} +#endif /* defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) */ + +#endif diff --git a/src/include/storage/atomics-generic.h b/src/include/storage/atomics-generic.h new file mode 100644 index 0000000..687dca6 --- /dev/null +++ b/src/include/storage/atomics-generic.h @@ -0,0 +1,290 @@ +/*------------------------------------------------------------------------- + * + * atomics-generic.h + * Atomic operations. + * + * Portions Copyright (c) 2013, PostgreSQL Global Development Group + * + * src/include/storage/atomics-generic.h + * + *------------------------------------------------------------------------- + */ + +/* intentionally no include guards, should only be included by atomics.h */ +#ifndef INSIDE_ATOMICS_H +#error "should be included via atomics.h" +#endif + +#ifndef PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 +#error "postgres requires 32bit CAS to be provided" +#endif + +/* provide fallback */ +#ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG +typedef atomic_uint32 atomic_flag; +#endif + +#if !defined(PG_DISABLE_64_BIT_ATOMICS) && defined(PG_HAS_ATOMIC_COMPARE_EXCHANGE_U64) +#define PG_HAVE_64_BIT_ATOMICS +#endif + +#if defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) + +#ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG +/* provide compare_exchange based TAS */ +#define PG_HAVE_ATOMIC_TEST_SET_FLAG + +STATIC_IF_INLINE bool +pg_atomic_test_set_flag_impl(volatile atomic_flag *ptr) +{ + uint32 value = 0; + pg_atomic_compare_exchange_u32_impl(ptr, value, 1); + return value == 0; +} + +#define PG_HAVE_ATOMIC_CLEAR_FLAG +STATIC_IF_INLINE void +pg_atomic_clear_flag_impl(volatile atomic_flag *ptr) +{ + uint32 value = 0; + pg_atomic_write_u32_impl(ptr, 1); + pg_barrier(); /* FIXME: recursive if no barriers provided */ +} +#endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */ + +#ifdef PG_HAVE_ATOMIC_INIT_FLAG +#define PG_HAVE_ATOMIC_INIT_FLAG +STATIC_IF_INLINE bool +pg_atomic_init_flag_impl(volatile atomic_flag *ptr) +{ + pg_atomic_clear_flag_impl(ptr); +} +#endif + +#ifndef PG_HAS_ATOMIC_INIT_U32 +#define PG_HAS_ATOMIC_INIT_U32 +STATIC_IF_INLINE void +pg_atomic_init_u32_impl(volatile atomic_uint32 *ptr, uint32 val_) +{ + pg_atomic_write_u32(ptr, val_); +} +#endif + +#ifndef PG_HAS_ATOMIC_EXCHANGE_U32 +#define PG_HAS_ATOMIC_EXCHANGE_U32 +STATIC_IF_INLINE uint32 +pg_atomic_exchange_u32_impl(volatile atomic_uint32 *ptr, uint32 xchg_) +{ + uint32 old; + while (true) + { + old = pg_atomic_read_u32(ptr); + if (pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_)) + break; + } + return old; +} +#endif + +#ifndef PG_HAS_ATOMIC_FETCH_ADD_U32 +#define PG_HAS_ATOMIC_FETCH_ADD_U32 +STATIC_IF_INLINE uint32 +pg_atomic_fetch_add_u32_impl(volatile atomic_uint32 *ptr, uint32 add_) +{ + uint32 old; + while (true) + { + old = pg_atomic_read_u32(ptr); + if (pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_)) + break; + } + return old; +} +#endif + +#ifndef PG_HAS_ATOMIC_FETCH_SUB_U32 +#define PG_HAS_ATOMIC_FETCH_SUB_U32 +STATIC_IF_INLINE uint32 +pg_atomic_fetch_sub_u32_impl(volatile atomic_uint32 *ptr, uint32 sub_) +{ + uint32 old; + while (true) + { + old = pg_atomic_read_u32(ptr); + if (pg_atomic_compare_exchange_u32_impl(ptr, &old, old - sub_)) + break; + } + return old; +} +#endif + +#ifndef PG_HAS_ATOMIC_FETCH_AND_U32 +#define PG_HAS_ATOMIC_FETCH_AND_U32 +STATIC_IF_INLINE uint32 +pg_atomic_fetch_and_u32_impl(volatile atomic_uint32 *ptr, uint32 and_) +{ + uint32 old; + while (true) + { + old = pg_atomic_read_u32(ptr); + if (pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_)) + break; + } + return old; +} +#endif + +#ifndef PG_HAS_ATOMIC_FETCH_OR_U32 +#define PG_HAS_ATOMIC_FETCH_OR_U32 +STATIC_IF_INLINE uint32 +pg_atomic_fetch_or_u32_impl(volatile atomic_uint32 *ptr, uint32 or_) +{ + uint32 old; + while (true) + { + old = pg_atomic_read_u32(ptr); + if (pg_atomic_compare_exchange_u32_impl(ptr, &old, old | or_)) + break; + } + return old; +} +#endif + +#ifndef PG_HAS_ATOMIC_ADD_FETCH_U32 +#define PG_HAS_ATOMIC_ADD_FETCH_U32 +STATIC_IF_INLINE uint32 +pg_atomic_add_fetch_u32_impl(volatile atomic_uint32 *ptr, uint32 add_) +{ + return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_; +} +#endif + +#ifndef PG_HAS_ATOMIC_SUB_FETCH_U32 +#define PG_HAS_ATOMIC_SUB_FETCH_U32 +STATIC_IF_INLINE uint32 +pg_atomic_sub_fetch_u32_impl(volatile atomic_uint32 *ptr, uint32 sub_) +{ + return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_; +} +#endif + +#ifndef PG_HAS_ATOMIC_FETCH_ADD_UNTIL_U32 +#define PG_HAS_ATOMIC_FETCH_ADD_UNTIL_U32 +STATIC_IF_INLINE uint32 +pg_atomic_fetch_add_until_u32_impl(volatile atomic_uint32 *ptr, uint32 add_, uint32 until) +{ + uint32 old; + while (true) + { + uint32 new_val; + old = pg_atomic_read_u32(ptr); + if (old >= until) + break; + + new_val = old + add_; + if (new_val > until) + new_val = until; + + if (pg_atomic_compare_exchange_u32_impl(ptr, &old, new_val)) + break; + } + return old; +} +#endif + +#ifdef PG_HAS_ATOMIC_COMPARE_EXCHANGE_U64 + +#ifndef PG_HAS_ATOMIC_INIT_U64 +#define PG_HAS_ATOMIC_INIT_U64 +STATIC_IF_INLINE void +pg_atomic_init_u64_impl(volatile atomic_uint64 *ptr, uint64 val_) +{ + pg_atomic_write_u64(ptr, val_); +} +#endif + +#ifndef PG_HAS_ATOMIC_FETCH_ADD_U64 +#define PG_HAS_ATOMIC_FETCH_ADD_U64 +STATIC_IF_INLINE uint64 +pg_atomic_fetch_add_u64_impl(volatile atomic_uint64 *ptr, uint64 add_) +{ + uint64 old; + while (true) + { + old = pg_atomic_read_u64(ptr); + if (pg_atomic_compare_exchange_64_impl(ptr, &old, old + add_)) + break; + } + return old; +} +#endif + +#ifndef PG_HAS_ATOMIC_FETCH_SUB_U64 +#define PG_HAS_ATOMIC_FETCH_SUB_U64 +STATIC_IF_INLINE uint64 +pg_atomic_fetch_sub_u64_impl(volatile atomic_uint64 *ptr, uint64 sub_) +{ + uint64 old; + while (true) + { + old = pg_atomic_read_u64(ptr); + if (pg_atomic_compare_exchange_64_impl(ptr, &old, old - sub_)) + break; + } + return old; +} +#endif + +#ifndef PG_HAS_ATOMIC_FETCH_AND_U64 +#define PG_HAS_ATOMIC_FETCH_AND_U64 +STATIC_IF_INLINE uint64 +pg_atomic_fetch_and_u64_impl(volatile atomic_uint64 *ptr, uint64 and_) +{ + uint64 old; + while (true) + { + old = pg_atomic_read_64_impl(ptr); + if (pg_atomic_compare_exchange_u64(ptr, &old, old & and_)) + break; + } + return old; +} +#endif + +#ifndef PG_HAS_ATOMIC_FETCH_OR_U64 +#define PG_HAS_ATOMIC_FETCH_OR_U64 +STATIC_IF_INLINE uint64 +pg_atomic_fetch_or_u64_impl(volatile atomic_uint64 *ptr, uint64 or_) +{ + uint64 old; + while (true) + { + old = pg_atomic_read_u64(ptr); + if (pg_atomic_compare_exchange_64_impl(ptr, &old, old | or_)) + break; + } + return old; +} +#endif + +#ifndef PG_HAS_ATOMIC_ADD_FETCH_U64 +#define PG_HAS_ATOMIC_ADD_FETCH_U64 +STATIC_IF_INLINE +pg_atomic_add_fetch_u64_impl(volatile atomic_uint64 *ptr, uint64 add_) +{ + return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_; +} +#endif + +#ifndef PG_HAS_ATOMIC_SUB_FETCH_U64 +#define PG_HAS_ATOMIC_SUB_FETCH_U64 +STATIC_IF_INLINE +pg_atomic_sub_fetch_u64_impl(volatile atomic_uint64 *ptr, uint64 sub_) +{ + return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_; +} +#endif + +#endif /* PG_HAS_ATOMIC_COMPARE_EXCHANGE_U64 */ + +#endif /* defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) */ diff --git a/src/include/storage/atomics.h b/src/include/storage/atomics.h new file mode 100644 index 0000000..574fc50c --- /dev/null +++ b/src/include/storage/atomics.h @@ -0,0 +1,415 @@ +/*------------------------------------------------------------------------- + * + * atomics.h + * Atomic operations. + * + * Portions Copyright (c) 2013, PostgreSQL Global Development Group + * + * src/include/storage/atomics.h + * + *------------------------------------------------------------------------- + */ +#ifndef ATOMICS_H +#define ATOMICS_H + +#include "storage/barrier.h" + +#define INSIDE_ATOMICS_H + +/* + * Architecture specific implementations/considerations. + */ +#if defined(__arm__) || defined(__arm) || \ + defined(__aarch64__) || defined(__aarch64) +# include "storage/atomics-arch-arm.h" +#endif + +/* + * Compiler specific, but architecture independent implementations + */ +/* gcc or compatible */ +#if defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS) +# include "storage/atomics-generic-gcc.h" +/* msvc FIXME: check against mingw */ +#elif defined(WIN32) +# include "storage/atomics-generic-msvc.h" +#elif defined(__hpux) && defined(__ia64) && !defined(__GNUC__) +# include "storage/atomics-generic-acc.h" +#elif defined(__SUNPRO_C) && defined(__ia64) && !defined(__GNUC__) +# include "storage/atomics-generic-sunpro.h" +#endif + +/* + * Provide fallback implementations for operations that aren't provided if + * possible. + */ +#include "storage/atomics-generic.h" + +/* + * pg_atomic_init_flag - initialize lock + * + * No barrier semantics. + */ +STATIC_IF_INLINE_DECLARE void +pg_atomic_init_flag(volatile atomic_flag *ptr); + +/* + * pg_atomic_test_and_set_flag - TAS() + * + * Full barrier semantics. (XXX? Only acquire?) + */ +STATIC_IF_INLINE_DECLARE bool +pg_atomic_test_set_flag(volatile atomic_flag *ptr); + +/* + * pg_atomic_clear_flag - release lock set by TAS() + * + * Full barrier semantics. (XXX? Only release?) + */ +STATIC_IF_INLINE_DECLARE void +pg_atomic_clear_flag(volatile atomic_flag *ptr); + +/* + * pg_atomic_init_u32 - initialize atomic variable + * + * Has to be done before any usage except when the atomic variable is declared + * statically in which case the variable is initialized to 0. + * + * No barrier semantics. + */ +STATIC_IF_INLINE_DECLARE void +pg_atomic_init_u32(volatile atomic_uint32 *ptr, uint32 val); + +/* + * pg_atomic_read_u32 - read from atomic variable + * + * No barrier semantics. + */ +STATIC_IF_INLINE_DECLARE uint32 +pg_atomic_read_u32(volatile atomic_uint32 *ptr); + +/* + * pg_atomic_write_u32 - write to atomic variable + * + * No barrier semantics. + */ +STATIC_IF_INLINE_DECLARE void +pg_atomic_write_u32(volatile atomic_uint32 *ptr, uint32 val); + +/* + * pg_atomic_exchange_u32 - exchange newval with current value + * + * Returns the old value of 'ptr' before the swap. + * + * Full barrier semantics. + */ +STATIC_IF_INLINE_DECLARE uint32 +pg_atomic_exchange_u32(volatile atomic_uint32 *ptr, uint32 newval); + +/* + * pg_atomic_compare_exchange_u32 - CAS operation + * + * Atomically compare the current value of ptr with *expected and store newval + * iff ptr and *expected have the same value. The current value of *ptr will + * always be stored in *expected. + * + * Return whether the values have been exchanged. + * + * Full barrier semantics. + */ +STATIC_IF_INLINE_DECLARE bool +pg_atomic_compare_exchange_u32(volatile atomic_uint32 *ptr, + uint32 *expected, uint32 newval); + +/* + * pg_atomic_fetch_add_u32 - atomically add to variable + * + * Returns the the value of ptr before the arithmetic operation. + * + * Full barrier semantics. + */ +STATIC_IF_INLINE_DECLARE uint32 +pg_atomic_fetch_add_u32(volatile atomic_uint32 *ptr, uint32 add_); + +/* + * pg_atomic_fetch_sub_u32 - atomically subtract from variable + * + * Returns the the value of ptr before the arithmetic operation. + * + * Full barrier semantics. + */ +STATIC_IF_INLINE_DECLARE uint32 +pg_atomic_fetch_sub_u32(volatile atomic_uint32 *ptr, uint32 sub_); + +/* + * pg_atomic_fetch_and_u32 - atomically bit-and and_ with variable + * + * Returns the the value of ptr before the arithmetic operation. + * + * Full barrier semantics. + */ +STATIC_IF_INLINE_DECLARE uint32 +pg_atomic_fetch_and_u32(volatile atomic_uint32 *ptr, uint32 and_); + +/* + * pg_atomic_fetch_or_u32 - atomically bit-or or_ with variable + * + * Returns the the value of ptr before the arithmetic operation. + * + * Full barrier semantics. + */ +STATIC_IF_INLINE_DECLARE uint32 +pg_atomic_fetch_or_u32(volatile atomic_uint32 *ptr, uint32 or_); + +/* + * pg_atomic_add_fetch_u32 - atomically add to variable + * + * Returns the the value of ptr after the arithmetic operation. + * + * Full barrier semantics. + */ +STATIC_IF_INLINE_DECLARE uint32 +pg_atomic_add_fetch_u32(volatile atomic_uint32 *ptr, uint32 add_); + +/* + * pg_atomic_sub_fetch_u32 - atomically subtract from variable + * + * Returns the the value of ptr after the arithmetic operation. + * + * Full barrier semantics. + */ +STATIC_IF_INLINE_DECLARE uint32 +pg_atomic_sub_fetch_u32(volatile atomic_uint32 *ptr, uint32 sub_); + +/* + * pg_fetch_add_until_u32 - saturated addition to variable + * + * Returns the the value of ptr after the arithmetic operation. + * + * Full barrier semantics. + */ +STATIC_IF_INLINE_DECLARE uint32 +pg_atomic_fetch_add_until_u32(volatile atomic_uint32 *ptr, uint32 add_, + uint32 until); + +/* ---- + * The 64 bit operations have the same semantics as their 32bit counterparts if + * they are available. + * --- + */ +#ifdef PG_HAVE_64_BIT_ATOMICS + +STATIC_IF_INLINE_DECLARE void +pg_atomic_init_u64(volatile atomic_uint64 *ptr, uint64 val_); + +STATIC_IF_INLINE_DECLARE uint32 +pg_atomic_read_u64(volatile atomic_uint64 *ptr); + +STATIC_IF_INLINE_DECLARE void +pg_atomic_write_u64(volatile atomic_uint64 *ptr, uint64 val); + +STATIC_IF_INLINE_DECLARE uint64 +pg_atomic_exchange_u64(volatile atomic_uint64 *ptr, uint64 newval); + +STATIC_IF_INLINE_DECLARE bool +pg_atomic_compare_exchange_u64(volatile atomic_uint64 *ptr, + uint32 *expected, uint64 newval); + +STATIC_IF_INLINE_DECLARE uint64 +pg_atomic_fetch_add_u64(volatile atomic_uint64 *ptr, uint64 add_); + +STATIC_IF_INLINE_DECLARE uint64 +pg_atomic_fetch_sub_u64(volatile atomic_uint64 *ptr, uint64 sub_); + +STATIC_IF_INLINE_DECLARE uint64 +pg_atomic_fetch_and_u64(volatile atomic_uint64 *ptr, uint64 and_); + +STATIC_IF_INLINE_DECLARE uint64 +pg_atomic_fetch_or_u64(volatile atomic_uint64 *ptr, uint64 or_); + +STATIC_IF_INLINE_DECLARE uint64 +pg_atomic_add_fetch_u64(volatile atomic_uint64 *ptr, uint64 add_); + +STATIC_IF_INLINE_DECLARE uint64 +pg_atomic_sub_fetch_u64(volatile atomic_uint64 *ptr, uint64 sub_); + +#endif /* PG_HAVE_64_BIT_ATOMICS */ + +/* + * The following functions are wrapper functions around the platform specific + * implementation of the atomic operations performing common checks. + */ +#if defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) + +#define CHECK_POINTER_ALIGNMENT(ptr, bndr) \ + Assert(TYPEALIGN((uintptr_t)(ptr), bndr)) + +STATIC_IF_INLINE_DECLARE bool +pg_atomic_test_set_flag(volatile atomic_flag *ptr) +{ + return pg_atomic_test_set_flag_impl(ptr); +} + +STATIC_IF_INLINE_DECLARE void +pg_atomic_clear_flag(volatile atomic_flag *ptr) +{ + pg_atomic_test_set_flag_impl(ptr); +} + +STATIC_IF_INLINE_DECLARE void +pg_atomic_init_flag(volatile atomic_flag *ptr) +{ + return pg_atomic_clear_flag_impl(ptr); +} + +STATIC_IF_INLINE_DECLARE void +pg_atomic_init_u32(volatile atomic_uint32 *ptr, uint32 val) +{ + CHECK_POINTER_ALIGNMENT(ptr, 4); + + pg_atomic_init_u32_impl(ptr, val); +} + +STATIC_IF_INLINE uint32 +pg_atomic_exchange_u32(volatile atomic_uint32 *ptr, uint32 newval) +{ + CHECK_POINTER_ALIGNMENT(ptr, 4); + + return pg_atomic_exchange_u32_impl(ptr, newval); +} + +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u32(volatile atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + CHECK_POINTER_ALIGNMENT(ptr, 4); + CHECK_POINTER_ALIGNMENT(expected, 4); + + return pg_atomic_compare_exchange_u32_impl(ptr, expected, newval); +} + +STATIC_IF_INLINE uint32 +pg_atomic_fetch_add_u32(volatile atomic_uint32 *ptr, uint32 add_) +{ + CHECK_POINTER_ALIGNMENT(ptr, 4); + return pg_atomic_fetch_add_u32_impl(ptr, add_); +} + +STATIC_IF_INLINE uint32 +pg_atomic_fetch_sub_u32(volatile atomic_uint32 *ptr, uint32 sub_) +{ + CHECK_POINTER_ALIGNMENT(ptr, 4); + return pg_atomic_fetch_sub_u32_impl(ptr, sub_); +} + +STATIC_IF_INLINE uint32 +pg_atomic_fetch_and_u32(volatile atomic_uint32 *ptr, uint32 and_) +{ + CHECK_POINTER_ALIGNMENT(ptr, 4); + return pg_atomic_fetch_and_u32_impl(ptr, and_); +} + +STATIC_IF_INLINE uint32 +pg_atomic_fetch_or_u32(volatile atomic_uint32 *ptr, uint32 or_) +{ + CHECK_POINTER_ALIGNMENT(ptr, 4); + return pg_atomic_fetch_or_u32_impl(ptr, or_); +} + +STATIC_IF_INLINE uint32 +pg_atomic_add_fetch_u32(volatile atomic_uint32 *ptr, uint32 add_) +{ + CHECK_POINTER_ALIGNMENT(ptr, 4); + return pg_atomic_add_fetch_u32_impl(ptr, add_); +} + +STATIC_IF_INLINE uint32 +pg_atomic_sub_fetch_u32(volatile atomic_uint32 *ptr, uint32 sub_) +{ + CHECK_POINTER_ALIGNMENT(ptr, 4); + return pg_atomic_sub_fetch_u32_impl(ptr, sub_); +} + +STATIC_IF_INLINE uint32 +pg_atomic_fetch_add_until_u32(volatile atomic_uint32 *ptr, uint32 add_, + uint32 until) +{ + CHECK_POINTER_ALIGNMENT(ptr, 4); + return pg_atomic_fetch_add_until_u32_impl(ptr, add_, until); +} + +#ifdef PG_HAVE_64_BIT_ATOMICS + +STATIC_IF_INLINE_DECLARE void +pg_atomic_init_u64(volatile atomic_uint64 *ptr, uint64 val) +{ + CHECK_POINTER_ALIGNMENT(ptr, 8); + + pg_atomic_init_u64_impl(ptr, val); +} + +STATIC_IF_INLINE uint64 +pg_atomic_read_u64(volatile atomic_uint64 *ptr) +{ + CHECK_POINTER_ALIGNMENT(ptr, 8); + return ptr->value; +} + +STATIC_IF_INLINE void +pg_atomic_write_u64(volatile atomic_uint64 *ptr, uint64 val) +{ + CHECK_POINTER_ALIGNMENT(ptr, 8); + ptr->value = val; +} + +STATIC_IF_INLINE uint64 +pg_atomic_exchange_u64(volatile atomic_uint64 *ptr, uint64 newval) +{ + CHECK_POINTER_ALIGNMENT(ptr, 8); + + return pg_atomic_exchange_u64_impl(ptr, newval); +} + +STATIC_IF_INLINE bool +pg_atomic_compare_exchange_u64(volatile atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + CHECK_POINTER_ALIGNMENT(ptr, 8); + CHECK_POINTER_ALIGNMENT(expected, 8); + return pg_atomic_compare_exchange_u64_impl(ptr, expected, newval); +} + +STATIC_IF_INLINE uint64 +pg_atomic_fetch_add_u64(volatile atomic_uint64 *ptr, uint64 add_) +{ + CHECK_POINTER_ALIGNMENT(ptr, 8); + return pg_atomic_fetch_add_u64_impl(ptr, add_); +} + +STATIC_IF_INLINE uint64 +pg_atomic_fetch_sub_u64(volatile atomic_uint64 *ptr, uint64 sub_) +{ + CHECK_POINTER_ALIGNMENT(ptr, 8); + return pg_atomic_fetch_sub_u64_impl(ptr, sub_); +} + +STATIC_IF_INLINE uint64 +pg_atomic_fetch_and_u64(volatile atomic_uint64 *ptr, uint64 and_) +{ + CHECK_POINTER_ALIGNMENT(ptr, 8); + return pg_atomic_fetch_and_u64_impl(ptr, and_); +} + +STATIC_IF_INLINE uint64 +pg_atomic_fetch_or_u64(volatile atomic_uint64 *ptr, uint64 or_) +{ + CHECK_POINTER_ALIGNMENT(ptr, 8); + return pg_atomic_fetch_or_u64_impl(ptr, or_); +} +#endif /* PG_HAVE_64_BIT_ATOMICS */ + +#endif /* defined(PG_USE_INLINE) || defined(ATOMICS_INCLUDE_DEFINITIONS) */ + +#undef INSIDE_ATOMICS_H + +#endif /* ATOMICS_H */ diff --git a/src/include/storage/s_lock.h b/src/include/storage/s_lock.h index 7dcd5d9..203e1f6 100644 --- a/src/include/storage/s_lock.h +++ b/src/include/storage/s_lock.h @@ -315,7 +315,7 @@ tas(volatile slock_t *lock) #define TAS(lock) tas(lock) -#ifdef HAVE_GCC_INT_ATOMICS +#ifdef HAVE_GCC__SYNC_INT32_TAS typedef int slock_t; @@ -327,7 +327,7 @@ tas(volatile slock_t *lock) #define S_UNLOCK(lock) __sync_lock_release(lock) -#else /* !HAVE_GCC_INT_ATOMICS */ +#else /* !HAVE_GCC__SYNC_INT32_TAS */ typedef unsigned char slock_t; @@ -344,7 +344,7 @@ tas(volatile slock_t *lock) return (int) _res; } -#endif /* HAVE_GCC_INT_ATOMICS */ +#endif /* HAVE_GCC__SYNC_INT32_TAS */ #endif /* __arm__ */ @@ -352,7 +352,7 @@ tas(volatile slock_t *lock) * On ARM64, we use __sync_lock_test_and_set(int *, int) if available. */ #if defined(__aarch64__) || defined(__aarch64) -#ifdef HAVE_GCC_INT_ATOMICS +#ifdef HAVE_GCC__SYNC_INT32_TAS #define HAS_TEST_AND_SET #define TAS(lock) tas(lock) @@ -367,7 +367,7 @@ tas(volatile slock_t *lock) #define S_UNLOCK(lock) __sync_lock_release(lock) -#endif /* HAVE_GCC_INT_ATOMICS */ +#endif /* HAVE_GCC__SYNC_INT32_TAS */ #endif /* __aarch64__ */ -- 1.8.3.251.g1462b67
-- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
