Author: Wenju He Date: 2025-12-05T10:12:25+08:00 New Revision: 79620d8ac094f2ab65b4c65376b710d04bba212f
URL: https://github.com/llvm/llvm-project/commit/79620d8ac094f2ab65b4c65376b710d04bba212f DIFF: https://github.com/llvm/llvm-project/commit/79620d8ac094f2ab65b4c65376b710d04bba212f.diff LOG: [libclc] Add OpenCL atomic_*_explicit builtins (#168318) Implement atomic_*_explicit (e.g. atomic_store_explicit) with memory_order plus optional memory_scope. OpenCL memory_order maps 1:1 to Clang (e.g. OpenCL memory_order_relaxed == Clang __ATOMIC_RELAXED), so we pass it unchanged to clc_atomic_* function which forwards to Clang _scoped_atomic* builtins. Other changes: * Add __opencl_get_clang_memory_scope helper in opencl/utils.h (OpenCL scope -> Clang scope). * Correct atomic_compare_exchange return type to bool. * Fix atomic_compare_exchange to return true when value stored in the pointer equals expected value. * Remove volatile from CLC functions so that volatile isn't present in LLVM IR. * Add '-fdeclare-opencl-builtins -finclude-default-header' flag to include declaration of memory_scope. Some constants in libclc are already provided by Clang’s OpenCL header; disable those in OpenCL library build and enable them only for CLC library build. Added: Modified: libclc/CMakeLists.txt libclc/clc/include/clc/atomic/atomic_decl.inc libclc/clc/include/clc/float/definitions.h libclc/clc/include/clc/integer/definitions.h libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.inc libclc/clc/lib/generic/atomic/clc_atomic_def.inc libclc/cmake/modules/AddLibclc.cmake libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h libclc/opencl/include/clc/opencl/atomic/atomic_load.h libclc/opencl/include/clc/opencl/atomic/atomic_store.h libclc/opencl/include/clc/opencl/explicit_fence/explicit_memory_fence.h libclc/opencl/include/clc/opencl/synchronization/barrier.h libclc/opencl/include/clc/opencl/synchronization/utils.h libclc/opencl/include/clc/opencl/utils.h libclc/opencl/lib/clspv/shared/vstore_half.cl libclc/opencl/lib/generic/atomic/atom_add.cl libclc/opencl/lib/generic/atomic/atom_and.cl libclc/opencl/lib/generic/atomic/atom_cmpxchg.cl libclc/opencl/lib/generic/atomic/atom_dec.cl libclc/opencl/lib/generic/atomic/atom_inc.cl libclc/opencl/lib/generic/atomic/atom_max.cl libclc/opencl/lib/generic/atomic/atom_min.cl libclc/opencl/lib/generic/atomic/atom_or.cl libclc/opencl/lib/generic/atomic/atom_sub.cl libclc/opencl/lib/generic/atomic/atom_xchg.cl libclc/opencl/lib/generic/atomic/atom_xor.cl libclc/opencl/lib/generic/atomic/atomic_add.cl libclc/opencl/lib/generic/atomic/atomic_and.cl libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl libclc/opencl/lib/generic/atomic/atomic_def.inc libclc/opencl/lib/generic/atomic/atomic_exchange.cl libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc libclc/opencl/lib/generic/atomic/atomic_load.cl libclc/opencl/lib/generic/atomic/atomic_max.cl libclc/opencl/lib/generic/atomic/atomic_min.cl libclc/opencl/lib/generic/atomic/atomic_or.cl libclc/opencl/lib/generic/atomic/atomic_store.cl libclc/opencl/lib/generic/atomic/atomic_sub.cl libclc/opencl/lib/generic/atomic/atomic_xchg.cl libclc/opencl/lib/generic/atomic/atomic_xor.cl Removed: libclc/opencl/include/clc/opencl/as_type.h libclc/opencl/include/clc/opencl/synchronization/cl_mem_fence_flags.h libclc/opencl/include/clc/opencl/types.h ################################################################################ diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt index 9a81f2658fe34..1d29adb0f3995 100644 --- a/libclc/CMakeLists.txt +++ b/libclc/CMakeLists.txt @@ -477,6 +477,7 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} ) ) list( APPEND build_flags + -Xclang -fdeclare-opencl-builtins -Xclang -finclude-default-header -I${CMAKE_CURRENT_SOURCE_DIR}/opencl/include ) diff --git a/libclc/clc/include/clc/atomic/atomic_decl.inc b/libclc/clc/include/clc/atomic/atomic_decl.inc index 5e0f456e34009..87ebe2b355e69 100644 --- a/libclc/clc/include/clc/atomic/atomic_decl.inc +++ b/libclc/clc/include/clc/atomic/atomic_decl.inc @@ -15,24 +15,23 @@ #ifdef __CLC_NO_VALUE_ARG #define __CLC_DECLARE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION( \ - volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder, \ - int MemoryScope); + ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder, int MemoryScope); #elif defined(__CLC_RETURN_VOID) #define __CLC_DECLARE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DECL void __CLC_FUNCTION( \ - volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ - int MemoryOrder, int MemoryScope); + ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value, int MemoryOrder, \ + int MemoryScope); #elif defined(__CLC_COMPARE_EXCHANGE) #define __CLC_DECLARE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION( \ - volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator, \ + ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator, \ __CLC_GENTYPE Value, int MemoryOrderEqual, int MemoryOrderUnequal, \ int MemoryScope); #else #define __CLC_DECLARE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION( \ - volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ - int MemoryOrder, int MemoryScope); + ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value, int MemoryOrder, \ + int MemoryScope); #endif __CLC_DECLARE_ATOMIC(global) diff --git a/libclc/clc/include/clc/float/definitions.h b/libclc/clc/include/clc/float/definitions.h index 93d2b5b391c5a..0317bf1bfc091 100644 --- a/libclc/clc/include/clc/float/definitions.h +++ b/libclc/clc/include/clc/float/definitions.h @@ -6,9 +6,15 @@ // //===----------------------------------------------------------------------===// +#ifndef MAXFLOAT #define MAXFLOAT 0x1.fffffep127f +#endif +#ifndef HUGE_VALF #define HUGE_VALF __builtin_huge_valf() +#endif +#ifndef INFINITY #define INFINITY __builtin_inff() +#endif #define FLT_DIG 6 #define FLT_MANT_DIG 24 @@ -17,33 +23,67 @@ #define FLT_MIN_10_EXP -37 #define FLT_MIN_EXP -125 #define FLT_RADIX 2 +#ifndef FLT_MAX #define FLT_MAX MAXFLOAT +#endif #define FLT_MIN 0x1.0p-126f #define FLT_EPSILON 0x1.0p-23f #define FLT_NAN __builtin_nanf("") +#ifndef FP_ILOGB0 #define FP_ILOGB0 (-2147483647 - 1) +#endif +#ifndef FP_ILOGBNAN #define FP_ILOGBNAN 2147483647 +#endif +#ifndef M_E_F #define M_E_F 0x1.5bf0a8p+1f +#endif +#ifndef M_LOG2E_F #define M_LOG2E_F 0x1.715476p+0f +#endif +#ifndef M_LOG10E_F #define M_LOG10E_F 0x1.bcb7b2p-2f +#endif +#ifndef M_LN2_F #define M_LN2_F 0x1.62e430p-1f +#endif +#ifndef M_LN10_F #define M_LN10_F 0x1.26bb1cp+1f +#endif +#ifndef M_PI_F #define M_PI_F 0x1.921fb6p+1f +#endif +#ifndef M_PI_2_F #define M_PI_2_F 0x1.921fb6p+0f +#endif +#ifndef M_PI_4_F #define M_PI_4_F 0x1.921fb6p-1f +#endif +#ifndef M_1_PI_F #define M_1_PI_F 0x1.45f306p-2f +#endif +#ifndef M_2_PI_F #define M_2_PI_F 0x1.45f306p-1f +#endif +#ifndef M_2_SQRTPI_F #define M_2_SQRTPI_F 0x1.20dd76p+0f +#endif +#ifndef M_SQRT2_F #define M_SQRT2_F 0x1.6a09e6p+0f +#endif +#ifndef M_SQRT1_2_F #define M_SQRT1_2_F 0x1.6a09e6p-1f +#endif #define M_LOG210_F 0x1.a934f0p+1f #ifdef cl_khr_fp64 +#ifndef HUGE_VAL #define HUGE_VAL __builtin_huge_val() +#endif #define DBL_DIG 15 #define DBL_MANT_DIG 53 @@ -82,11 +122,19 @@ #define HALF_MIN_EXP -13 #define HALF_RADIX 2 +#ifndef HALF_MAX #define HALF_MAX 0x1.ffcp15h +#endif +#ifndef HALF_MIN #define HALF_MIN 0x1.0p-14h +#endif +#ifndef HALF_EPSILON #define HALF_EPSILON 0x1.0p-10h +#endif #define HALF_NAN __builtin_nanf16("") +#ifndef M_LOG2E_H #define M_LOG2E_H 0x1.714p+0h +#endif #endif diff --git a/libclc/clc/include/clc/integer/definitions.h b/libclc/clc/include/clc/integer/definitions.h index ae2b52d21e781..3b9d57f208535 100644 --- a/libclc/clc/include/clc/integer/definitions.h +++ b/libclc/clc/include/clc/integer/definitions.h @@ -11,15 +11,23 @@ #define CHAR_BIT 8 #define INT_MAX 2147483647 +#ifndef INT_MIN #define INT_MIN (-2147483647 - 1) +#endif #define LONG_MAX 0x7fffffffffffffffL +#ifndef LONG_MIN #define LONG_MIN (-0x7fffffffffffffffL - 1) +#endif #define CHAR_MAX SCHAR_MAX #define CHAR_MIN SCHAR_MIN #define SCHAR_MAX 127 +#ifndef SCHAR_MIN #define SCHAR_MIN (-127 - 1) +#endif #define SHRT_MAX 32767 +#ifndef SHRT_MIN #define SHRT_MIN (-32767 - 1) +#endif #define UCHAR_MAX 255 #define UCHAR_MIN 0 #define USHRT_MAX 65535 diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.inc b/libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.inc index 74284fd61024c..07ef69d426768 100644 --- a/libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.inc +++ b/libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.inc @@ -25,7 +25,7 @@ #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_atomic_compare_exchange( \ - volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator, \ + ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator, \ __CLC_GENTYPE Value, int MemoryOrderEqual, int MemoryOrderUnequal, \ int MemoryScope) { \ __CLC_U_GENTYPE Comp = __CLC_AS_U_GENTYPE(Comparator); \ @@ -39,7 +39,7 @@ #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_atomic_compare_exchange( \ - volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator, \ + ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator, \ __CLC_GENTYPE Value, int MemoryOrderEqual, int MemoryOrderUnequal, \ int MemoryScope) { \ __scoped_atomic_compare_exchange_n(Ptr, &Comparator, Value, false, \ diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_def.inc b/libclc/clc/lib/generic/atomic/clc_atomic_def.inc index 6926b82248bf9..3a313defe0c03 100644 --- a/libclc/clc/lib/generic/atomic/clc_atomic_def.inc +++ b/libclc/clc/lib/generic/atomic/clc_atomic_def.inc @@ -36,32 +36,30 @@ #ifdef __CLC_NO_VALUE_ARG #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION( \ - volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder, \ - int MemoryScope) { \ + ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder, int MemoryScope) { \ return __CLC_AS_RETTYPE(__CLC_IMPL_FUNCTION( \ (ADDRSPACE __CLC_CASTTYPE *)Ptr, MemoryOrder, MemoryScope)); \ } #elif defined(__CLC_INC_DEC) #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION( \ - volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder, \ - int MemoryScope) { \ + ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder, int MemoryScope) { \ return __CLC_IMPL_FUNCTION(Ptr, (__CLC_U_GENTYPE)(-1), MemoryOrder, \ MemoryScope); \ } #elif defined(__CLC_RETURN_VOID) #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DEF void __CLC_FUNCTION( \ - volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ - int MemoryOrder, int MemoryScope) { \ + ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value, int MemoryOrder, \ + int MemoryScope) { \ __CLC_IMPL_FUNCTION((ADDRSPACE __CLC_CASTTYPE *)Ptr, \ __CLC_AS_CASTTYPE(Value), MemoryOrder, MemoryScope); \ } #else #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION( \ - volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ - int MemoryOrder, int MemoryScope) { \ + ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value, int MemoryOrder, \ + int MemoryScope) { \ return __CLC_AS_RETTYPE(__CLC_IMPL_FUNCTION( \ (ADDRSPACE __CLC_CASTTYPE *)Ptr, __CLC_AS_CASTTYPE(Value), \ MemoryOrder, MemoryScope)); \ diff --git a/libclc/cmake/modules/AddLibclc.cmake b/libclc/cmake/modules/AddLibclc.cmake index 3228926e765f4..b83ee7cf184eb 100644 --- a/libclc/cmake/modules/AddLibclc.cmake +++ b/libclc/cmake/modules/AddLibclc.cmake @@ -29,9 +29,13 @@ function(compile_to_bc) if( NOT ${FILE_EXT} STREQUAL ".ll" ) # Pass '-c' when not running the preprocessor set( PP_OPTS -c ) + set( EXTRA_OPTS ${ARG_EXTRA_OPTS} ) else() set( PP_OPTS -E;-P ) set( TMP_SUFFIX .tmp ) + string( REPLACE "-Xclang;-fdeclare-opencl-builtins;-Xclang;-finclude-default-header" + "" EXTRA_OPTS "${ARG_EXTRA_OPTS}" + ) endif() set( TARGET_ARG ) @@ -48,7 +52,7 @@ function(compile_to_bc) COMMAND ${clang_exe} ${TARGET_ARG} ${PP_OPTS} - ${ARG_EXTRA_OPTS} + ${EXTRA_OPTS} -MD -MF ${ARG_OUTPUT}.d -MT ${ARG_OUTPUT}${TMP_SUFFIX} # LLVM 13 enables standard includes by default - we don't want # those when pre-processing IR. We disable it unconditionally. diff --git a/libclc/opencl/include/clc/opencl/as_type.h b/libclc/opencl/include/clc/opencl/as_type.h deleted file mode 100644 index abe643df81046..0000000000000 --- a/libclc/opencl/include/clc/opencl/as_type.h +++ /dev/null @@ -1,92 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef __CLC_OPENCL_AS_TYPE_H__ -#define __CLC_OPENCL_AS_TYPE_H__ - -#include <clc/opencl/opencl-base.h> - -#define as_char(x) __builtin_astype(x, char) -#define as_uchar(x) __builtin_astype(x, uchar) -#define as_short(x) __builtin_astype(x, short) -#define as_ushort(x) __builtin_astype(x, ushort) -#define as_int(x) __builtin_astype(x, int) -#define as_uint(x) __builtin_astype(x, uint) -#define as_long(x) __builtin_astype(x, long) -#define as_ulong(x) __builtin_astype(x, ulong) -#define as_float(x) __builtin_astype(x, float) - -#define as_char2(x) __builtin_astype(x, char2) -#define as_uchar2(x) __builtin_astype(x, uchar2) -#define as_short2(x) __builtin_astype(x, short2) -#define as_ushort2(x) __builtin_astype(x, ushort2) -#define as_int2(x) __builtin_astype(x, int2) -#define as_uint2(x) __builtin_astype(x, uint2) -#define as_long2(x) __builtin_astype(x, long2) -#define as_ulong2(x) __builtin_astype(x, ulong2) -#define as_float2(x) __builtin_astype(x, float2) - -#define as_char3(x) __builtin_astype(x, char3) -#define as_uchar3(x) __builtin_astype(x, uchar3) -#define as_short3(x) __builtin_astype(x, short3) -#define as_ushort3(x) __builtin_astype(x, ushort3) -#define as_int3(x) __builtin_astype(x, int3) -#define as_uint3(x) __builtin_astype(x, uint3) -#define as_long3(x) __builtin_astype(x, long3) -#define as_ulong3(x) __builtin_astype(x, ulong3) -#define as_float3(x) __builtin_astype(x, float3) - -#define as_char4(x) __builtin_astype(x, char4) -#define as_uchar4(x) __builtin_astype(x, uchar4) -#define as_short4(x) __builtin_astype(x, short4) -#define as_ushort4(x) __builtin_astype(x, ushort4) -#define as_int4(x) __builtin_astype(x, int4) -#define as_uint4(x) __builtin_astype(x, uint4) -#define as_long4(x) __builtin_astype(x, long4) -#define as_ulong4(x) __builtin_astype(x, ulong4) -#define as_float4(x) __builtin_astype(x, float4) - -#define as_char8(x) __builtin_astype(x, char8) -#define as_uchar8(x) __builtin_astype(x, uchar8) -#define as_short8(x) __builtin_astype(x, short8) -#define as_ushort8(x) __builtin_astype(x, ushort8) -#define as_int8(x) __builtin_astype(x, int8) -#define as_uint8(x) __builtin_astype(x, uint8) -#define as_long8(x) __builtin_astype(x, long8) -#define as_ulong8(x) __builtin_astype(x, ulong8) -#define as_float8(x) __builtin_astype(x, float8) - -#define as_char16(x) __builtin_astype(x, char16) -#define as_uchar16(x) __builtin_astype(x, uchar16) -#define as_short16(x) __builtin_astype(x, short16) -#define as_ushort16(x) __builtin_astype(x, ushort16) -#define as_int16(x) __builtin_astype(x, int16) -#define as_uint16(x) __builtin_astype(x, uint16) -#define as_long16(x) __builtin_astype(x, long16) -#define as_ulong16(x) __builtin_astype(x, ulong16) -#define as_float16(x) __builtin_astype(x, float16) - -#ifdef cl_khr_fp64 -#define as_double(x) __builtin_astype(x, double) -#define as_double2(x) __builtin_astype(x, double2) -#define as_double3(x) __builtin_astype(x, double3) -#define as_double4(x) __builtin_astype(x, double4) -#define as_double8(x) __builtin_astype(x, double8) -#define as_double16(x) __builtin_astype(x, double16) -#endif - -#ifdef cl_khr_fp16 -#define as_half(x) __builtin_astype(x, half) -#define as_half2(x) __builtin_astype(x, half2) -#define as_half3(x) __builtin_astype(x, half3) -#define as_half4(x) __builtin_astype(x, half4) -#define as_half8(x) __builtin_astype(x, half8) -#define as_half16(x) __builtin_astype(x, half16) -#endif - -#endif // __CLC_OPENCL_AS_TYPE_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h index 59bfa0e87dd8f..cae033b17173a 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_compare_exchange_strong #define __CLC_COMPARE_EXCHANGE diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h index 7106c3e061d65..e8d9f8728bc6a 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_compare_exchange_weak #define __CLC_COMPARE_EXCHANGE diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc b/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc index 38d250f0693f7..a36e68bca86a2 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc @@ -25,32 +25,105 @@ #define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE) +#define __CLC_FUNCTION_EXPLICIT __CLC_XCONCAT(__CLC_FUNCTION, _explicit) + +#ifdef __CLC_NO_VALUE_ARG +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, memory_order Order, \ + memory_scope Scope); +#elif defined(__CLC_RETURN_VOID) +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL void __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ + memory_order Order, memory_scope Scope); +#elif defined(__CLC_COMPARE_EXCHANGE) +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL bool __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, \ + ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired, \ + memory_order Order, memory_scope Scope); +#else +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ + memory_order Order, memory_scope Scope); +#endif + +__CLC_DECL_ATOMIC(global) +__CLC_DECL_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DECL_ATOMIC() +#endif + +#undef __CLC_DECL_ATOMIC + +#if defined(__opencl_c_atomic_scope_device) + +#ifdef __CLC_NO_VALUE_ARG +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, memory_order Order); +#elif defined(__CLC_RETURN_VOID) +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL void __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ + memory_order Order); +#elif defined(__CLC_COMPARE_EXCHANGE) +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL bool __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, \ + ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired, \ + memory_order Success, memory_order Failure); +#else +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ + memory_order Order); +#endif + +__CLC_DECL_ATOMIC(global) +__CLC_DECL_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DECL_ATOMIC() +#endif + +#undef __CLC_DECL_ATOMIC + +#endif // defined(__opencl_c_atomic_scope_device) + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + #ifdef __CLC_NO_VALUE_ARG -#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION( \ volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr); #elif defined(__CLC_RETURN_VOID) -#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DECL void __CLC_FUNCTION( \ volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value); #elif defined(__CLC_COMPARE_EXCHANGE) -#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ - _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION( \ +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL bool __CLC_FUNCTION( \ volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, \ ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired); #else -#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION( \ volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value); #endif -__CLC_DEFINE_ATOMIC(global) -__CLC_DEFINE_ATOMIC(local) +__CLC_DECL_ATOMIC(global) +__CLC_DECL_ATOMIC(local) #if _CLC_GENERIC_AS_SUPPORTED -__CLC_DEFINE_ATOMIC() +__CLC_DECL_ATOMIC() #endif -#undef __CLC_DEFINE_ATOMIC +#undef __CLC_DECL_ATOMIC + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) #endif // __CLC_HAVE_FP_ATOMIC || __CLC_HAVE_INT_ATOMIC diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h b/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h index 9d949825b58c3..117355c216cc0 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_exchange #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h index bae5a7a7e19bb..bca0831d3063b 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_fetch_add #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h index 9f9d2225f910e..32919bf5ebe25 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_fetch_and #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h index bef102dc82f48..b6df519dcd016 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_fetch_max #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h index d7e346dc44368..fde0162427e43 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_fetch_min #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h index aa00982e15a56..10ed38ec660e5 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_fetch_or #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h index 3d04ed7ba34f8..afd56a62e990a 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_fetch_sub #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h index 2cdff08069025..372250ecf5df5 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_fetch_xor #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h index 2fcd3eef43a65..646fedde6f503 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h @@ -10,7 +10,6 @@ #define __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_CLEAR_H__ #include <clc/opencl/opencl-base.h> -#include <clc/opencl/types.h> #if defined(__opencl_c_atomic_order_seq_cst) && \ defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h index 6e3a8e403d5da..fe180b04632ba 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h @@ -10,7 +10,6 @@ #define __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_TEST_AND_SET_H__ #include <clc/opencl/opencl-base.h> -#include <clc/opencl/types.h> #if defined(__opencl_c_atomic_order_seq_cst) && \ defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_load.h b/libclc/opencl/include/clc/opencl/atomic/atomic_load.h index 7db259b136ec8..8d891098825ec 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_load.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_load.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_load #define __CLC_NO_VALUE_ARG diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_store.h b/libclc/opencl/include/clc/opencl/atomic/atomic_store.h index b3cdfc6ffaeae..2f9ab96f671d3 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_store.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_store.h @@ -9,6 +9,8 @@ #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__ #define __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__ +#include <clc/opencl/opencl-base.h> + #define __CLC_FUNCTION atomic_store #define __CLC_RETURN_VOID diff --git a/libclc/opencl/include/clc/opencl/explicit_fence/explicit_memory_fence.h b/libclc/opencl/include/clc/opencl/explicit_fence/explicit_memory_fence.h index 3ade970a33c61..7e0c7cabafda7 100644 --- a/libclc/opencl/include/clc/opencl/explicit_fence/explicit_memory_fence.h +++ b/libclc/opencl/include/clc/opencl/explicit_fence/explicit_memory_fence.h @@ -10,7 +10,6 @@ #define __CLC_OPENCL_EXPLICIT_FENCE_EXPLICIT_MEMORY_FENCE_H__ #include <clc/opencl/opencl-base.h> -#include <clc/opencl/synchronization/cl_mem_fence_flags.h> _CLC_DECL _CLC_OVERLOAD void mem_fence(cl_mem_fence_flags flags); _CLC_DECL _CLC_OVERLOAD void read_mem_fence(cl_mem_fence_flags flags); diff --git a/libclc/opencl/include/clc/opencl/synchronization/barrier.h b/libclc/opencl/include/clc/opencl/synchronization/barrier.h index 728ba4ae8a532..54295b9035ae8 100644 --- a/libclc/opencl/include/clc/opencl/synchronization/barrier.h +++ b/libclc/opencl/include/clc/opencl/synchronization/barrier.h @@ -10,7 +10,6 @@ #define __CLC_OPENCL_SYNCHRONIZATION_BARRIER_H__ #include <clc/opencl/opencl-base.h> -#include <clc/opencl/synchronization/cl_mem_fence_flags.h> _CLC_DECL _CLC_OVERLOAD void barrier(cl_mem_fence_flags flags); diff --git a/libclc/opencl/include/clc/opencl/synchronization/cl_mem_fence_flags.h b/libclc/opencl/include/clc/opencl/synchronization/cl_mem_fence_flags.h deleted file mode 100644 index 7b2f701c1ff99..0000000000000 --- a/libclc/opencl/include/clc/opencl/synchronization/cl_mem_fence_flags.h +++ /dev/null @@ -1,18 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef __CLC_OPENCL_SYNCHRONIZATION_CL_MEM_FENCE_FLAGS_H__ -#define __CLC_OPENCL_SYNCHRONIZATION_CL_MEM_FENCE_FLAGS_H__ - -typedef uint cl_mem_fence_flags; - -#define CLK_LOCAL_MEM_FENCE 1 -#define CLK_GLOBAL_MEM_FENCE 2 -#define CLK_IMAGE_MEM_FENCE 4 - -#endif // __CLC_OPENCL_SYNCHRONIZATION_CL_MEM_FENCE_FLAGS_H__ diff --git a/libclc/opencl/include/clc/opencl/synchronization/utils.h b/libclc/opencl/include/clc/opencl/synchronization/utils.h index a4f06ba8da6ac..016341a73ba37 100644 --- a/libclc/opencl/include/clc/opencl/synchronization/utils.h +++ b/libclc/opencl/include/clc/opencl/synchronization/utils.h @@ -11,7 +11,6 @@ #include <clc/internal/clc.h> #include <clc/mem_fence/clc_mem_semantic.h> -#include <clc/opencl/synchronization/cl_mem_fence_flags.h> static _CLC_INLINE int __opencl_get_memory_scope(cl_mem_fence_flags flag) { if (flag & CLK_GLOBAL_MEM_FENCE) diff --git a/libclc/opencl/include/clc/opencl/types.h b/libclc/opencl/include/clc/opencl/types.h deleted file mode 100644 index b1be88f21bdaa..0000000000000 --- a/libclc/opencl/include/clc/opencl/types.h +++ /dev/null @@ -1,48 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef __CLC_OPENCL_TYPES_H__ -#define __CLC_OPENCL_TYPES_H__ - -// Copied from clang/lib/Headers/opencl-c-base.h - -typedef enum memory_scope { - memory_scope_work_item = __OPENCL_MEMORY_SCOPE_WORK_ITEM, - memory_scope_work_group = __OPENCL_MEMORY_SCOPE_WORK_GROUP, - memory_scope_device = __OPENCL_MEMORY_SCOPE_DEVICE, -#if defined(__opencl_c_atomic_scope_all_devices) - memory_scope_all_svm_devices = __OPENCL_MEMORY_SCOPE_ALL_SVM_DEVICES, -#if (__OPENCL_C_VERSION__ >= CL_VERSION_3_0 || __OPENCL_CPP_VERSION__ >= 202100) - memory_scope_all_devices = memory_scope_all_svm_devices, -#endif // (__OPENCL_C_VERSION__ >= CL_VERSION_3_0 || __OPENCL_CPP_VERSION__ >= - // 202100) -#endif // defined(__opencl_c_atomic_scope_all_devices) -/** - * Subgroups have diff erent requirements on forward progress, so just test - * all the relevant macros. - * CL 3.0 sub-groups "they are not guaranteed to make independent forward - * progress" KHR subgroups "Subgroups within a workgroup are independent, make - * forward progress with respect to each other" - */ -#if defined(cl_intel_subgroups) || defined(cl_khr_subgroups) || \ - defined(__opencl_c_subgroups) - memory_scope_sub_group = __OPENCL_MEMORY_SCOPE_SUB_GROUP -#endif -} memory_scope; - -typedef enum memory_order { - memory_order_relaxed = __ATOMIC_RELAXED, - memory_order_acquire = __ATOMIC_ACQUIRE, - memory_order_release = __ATOMIC_RELEASE, - memory_order_acq_rel = __ATOMIC_ACQ_REL, -#if defined(__opencl_c_atomic_order_seq_cst) - memory_order_seq_cst = __ATOMIC_SEQ_CST -#endif -} memory_order; - -#endif // __CLC_OPENCL_TYPES_H__ diff --git a/libclc/opencl/include/clc/opencl/utils.h b/libclc/opencl/include/clc/opencl/utils.h index 42b948b8d30d2..7304ce30c5acd 100644 --- a/libclc/opencl/include/clc/opencl/utils.h +++ b/libclc/opencl/include/clc/opencl/utils.h @@ -10,9 +10,7 @@ #define __CLC_OPENCL_UTILS_H__ #include <clc/internal/clc.h> -#include <clc/opencl/types.h> -// INTEL_FEATURE_PISA static _CLC_INLINE int __opencl_get_clang_memory_scope(memory_scope scope) { switch (scope) { case __OPENCL_MEMORY_SCOPE_WORK_ITEM: @@ -30,6 +28,5 @@ static _CLC_INLINE int __opencl_get_clang_memory_scope(memory_scope scope) { return __MEMORY_SCOPE_SYSTEM; } } -// end INTEL_FEATURE_PISA #endif // __CLC_OPENCL_UTILS_H__ diff --git a/libclc/opencl/lib/clspv/shared/vstore_half.cl b/libclc/opencl/lib/clspv/shared/vstore_half.cl index 66973eb9ac4dc..b5f0ae75c6d7a 100644 --- a/libclc/opencl/lib/clspv/shared/vstore_half.cl +++ b/libclc/opencl/lib/clspv/shared/vstore_half.cl @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include <clc/float/definitions.h> -#include <clc/opencl/as_type.h> #include <clc/opencl/math/copysign.h> #include <clc/opencl/math/fabs.h> #include <clc/opencl/math/nextafter.h> diff --git a/libclc/opencl/lib/generic/atomic/atom_add.cl b/libclc/opencl/lib/generic/atomic/atom_add.cl index 368bbb790fd88..1628ea39de434 100644 --- a/libclc/opencl/lib/generic/atomic/atom_add.cl +++ b/libclc/opencl/lib/generic/atomic/atom_add.cl @@ -12,12 +12,12 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_add(volatile AS TYPE *p, TYPE val) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_add(AS TYPE *p, TYPE val) { \ return __clc_atomic_fetch_add(p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_add(AS TYPE *p, TYPE val) { \ - return atom_add((volatile AS TYPE *)p, val); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_add(volatile AS TYPE *p, TYPE val) { \ + return atom_add((AS TYPE *)p, val); \ } #ifdef cl_khr_global_int32_base_atomics diff --git a/libclc/opencl/lib/generic/atomic/atom_and.cl b/libclc/opencl/lib/generic/atomic/atom_and.cl index ffcc5bffaafac..bad85046c12e5 100644 --- a/libclc/opencl/lib/generic/atomic/atom_and.cl +++ b/libclc/opencl/lib/generic/atomic/atom_and.cl @@ -12,12 +12,12 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_and(volatile AS TYPE *p, TYPE val) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_and(AS TYPE *p, TYPE val) { \ return __clc_atomic_fetch_and(p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_and(AS TYPE *p, TYPE val) { \ - return atom_and((volatile AS TYPE *)p, val); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_and(volatile AS TYPE *p, TYPE val) { \ + return atom_and((AS TYPE *)p, val); \ } #ifdef cl_khr_global_int32_extended_atomics diff --git a/libclc/opencl/lib/generic/atomic/atom_cmpxchg.cl b/libclc/opencl/lib/generic/atomic/atom_cmpxchg.cl index 2e72ec529c45e..41ae9328e9480 100644 --- a/libclc/opencl/lib/generic/atomic/atom_cmpxchg.cl +++ b/libclc/opencl/lib/generic/atomic/atom_cmpxchg.cl @@ -12,14 +12,14 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_cmpxchg(volatile AS TYPE *p, TYPE cmp, \ - TYPE val) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_cmpxchg(AS TYPE *p, TYPE cmp, TYPE val) { \ return __clc_atomic_compare_exchange(p, cmp, val, __ATOMIC_RELAXED, \ __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_cmpxchg(AS TYPE *p, TYPE cmp, TYPE val) { \ - return atom_cmpxchg((volatile AS TYPE *)p, cmp, val); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_cmpxchg(volatile AS TYPE *p, TYPE cmp, \ + TYPE val) { \ + return atom_cmpxchg((AS TYPE *)p, cmp, val); \ } #ifdef cl_khr_global_int32_base_atomics diff --git a/libclc/opencl/lib/generic/atomic/atom_dec.cl b/libclc/opencl/lib/generic/atomic/atom_dec.cl index a1c7e58ef9e03..1f81f1d37b510 100644 --- a/libclc/opencl/lib/generic/atomic/atom_dec.cl +++ b/libclc/opencl/lib/generic/atomic/atom_dec.cl @@ -12,11 +12,11 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_dec(volatile AS TYPE *p) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_dec(AS TYPE *p) { \ return __clc_atomic_dec(p, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_dec(AS TYPE *p) { \ - return atom_dec((volatile AS TYPE *)p); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_dec(volatile AS TYPE *p) { \ + return atom_dec((AS TYPE *)p); \ } #ifdef cl_khr_global_int32_base_atomics diff --git a/libclc/opencl/lib/generic/atomic/atom_inc.cl b/libclc/opencl/lib/generic/atomic/atom_inc.cl index f3636d85693b8..5ae5bbb67e791 100644 --- a/libclc/opencl/lib/generic/atomic/atom_inc.cl +++ b/libclc/opencl/lib/generic/atomic/atom_inc.cl @@ -12,11 +12,11 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_inc(volatile AS TYPE *p) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_inc(AS TYPE *p) { \ return __clc_atomic_inc(p, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_inc(AS TYPE *p) { \ - return atom_inc((volatile AS TYPE *)p); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_inc(volatile AS TYPE *p) { \ + return atom_inc((AS TYPE *)p); \ } #ifdef cl_khr_global_int32_base_atomics diff --git a/libclc/opencl/lib/generic/atomic/atom_max.cl b/libclc/opencl/lib/generic/atomic/atom_max.cl index c2095ec36ba1e..249f5efbd4491 100644 --- a/libclc/opencl/lib/generic/atomic/atom_max.cl +++ b/libclc/opencl/lib/generic/atomic/atom_max.cl @@ -12,12 +12,12 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_max(volatile AS TYPE *p, TYPE val) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_max(AS TYPE *p, TYPE val) { \ return __clc_atomic_fetch_max(p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_max(AS TYPE *p, TYPE val) { \ - return atom_max((volatile AS TYPE *)p, val); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_max(volatile AS TYPE *p, TYPE val) { \ + return atom_max((AS TYPE *)p, val); \ } #ifdef cl_khr_global_int32_extended_atomics diff --git a/libclc/opencl/lib/generic/atomic/atom_min.cl b/libclc/opencl/lib/generic/atomic/atom_min.cl index 6360d018d1e90..029c601dd53af 100644 --- a/libclc/opencl/lib/generic/atomic/atom_min.cl +++ b/libclc/opencl/lib/generic/atomic/atom_min.cl @@ -12,12 +12,12 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_min(volatile AS TYPE *p, TYPE val) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_min(AS TYPE *p, TYPE val) { \ return __clc_atomic_fetch_min(p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_min(AS TYPE *p, TYPE val) { \ - return atom_min((volatile AS TYPE *)p, val); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_min(volatile AS TYPE *p, TYPE val) { \ + return atom_min((AS TYPE *)p, val); \ } #ifdef cl_khr_global_int32_extended_atomics diff --git a/libclc/opencl/lib/generic/atomic/atom_or.cl b/libclc/opencl/lib/generic/atomic/atom_or.cl index ad28aa436de8c..91f745bdda61a 100644 --- a/libclc/opencl/lib/generic/atomic/atom_or.cl +++ b/libclc/opencl/lib/generic/atomic/atom_or.cl @@ -12,12 +12,12 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_or(volatile AS TYPE *p, TYPE val) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_or(AS TYPE *p, TYPE val) { \ return __clc_atomic_fetch_or(p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_or(AS TYPE *p, TYPE val) { \ - return atom_or((volatile AS TYPE *)p, val); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_or(volatile AS TYPE *p, TYPE val) { \ + return atom_or((AS TYPE *)p, val); \ } #ifdef cl_khr_global_int32_extended_atomics diff --git a/libclc/opencl/lib/generic/atomic/atom_sub.cl b/libclc/opencl/lib/generic/atomic/atom_sub.cl index 9daaa1b3ce154..7eeabd51dad48 100644 --- a/libclc/opencl/lib/generic/atomic/atom_sub.cl +++ b/libclc/opencl/lib/generic/atomic/atom_sub.cl @@ -12,12 +12,12 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_sub(volatile AS TYPE *p, TYPE val) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_sub(AS TYPE *p, TYPE val) { \ return __clc_atomic_fetch_sub(p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_sub(AS TYPE *p, TYPE val) { \ - return atom_sub((volatile AS TYPE *)p, val); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_sub(volatile AS TYPE *p, TYPE val) { \ + return atom_sub((AS TYPE *)p, val); \ } #ifdef cl_khr_global_int32_base_atomics diff --git a/libclc/opencl/lib/generic/atomic/atom_xchg.cl b/libclc/opencl/lib/generic/atomic/atom_xchg.cl index 5b75873f29760..770e4366de834 100644 --- a/libclc/opencl/lib/generic/atomic/atom_xchg.cl +++ b/libclc/opencl/lib/generic/atomic/atom_xchg.cl @@ -12,12 +12,12 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_xchg(volatile AS TYPE *p, TYPE val) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_xchg(AS TYPE *p, TYPE val) { \ return __clc_atomic_exchange(p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_xchg(AS TYPE *p, TYPE val) { \ - return atom_xchg((volatile AS TYPE *)p, val); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_xchg(volatile AS TYPE *p, TYPE val) { \ + return atom_xchg((AS TYPE *)p, val); \ } #ifdef cl_khr_global_int32_base_atomics diff --git a/libclc/opencl/lib/generic/atomic/atom_xor.cl b/libclc/opencl/lib/generic/atomic/atom_xor.cl index 21aba01267e18..3fcf54e1c8832 100644 --- a/libclc/opencl/lib/generic/atomic/atom_xor.cl +++ b/libclc/opencl/lib/generic/atomic/atom_xor.cl @@ -12,12 +12,12 @@ // Non-volatile overloads are for backward compatibility with OpenCL 1.0. #define __CLC_IMPL(AS, TYPE) \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_xor(volatile AS TYPE *p, TYPE val) { \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_xor(AS TYPE *p, TYPE val) { \ return __clc_atomic_fetch_xor(p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } \ - _CLC_OVERLOAD _CLC_DEF TYPE atom_xor(AS TYPE *p, TYPE val) { \ - return atom_xor((volatile AS TYPE *)p, val); \ + _CLC_OVERLOAD _CLC_DEF TYPE atom_xor(volatile AS TYPE *p, TYPE val) { \ + return atom_xor((AS TYPE *)p, val); \ } #ifdef cl_khr_global_int32_extended_atomics diff --git a/libclc/opencl/lib/generic/atomic/atomic_add.cl b/libclc/opencl/lib/generic/atomic/atomic_add.cl index 5501d30544e7c..2f6606eb33338 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_add.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_add.cl @@ -11,7 +11,7 @@ #define __CLC_IMPL(TYPE, AS) \ _CLC_OVERLOAD _CLC_DEF TYPE atomic_add(volatile AS TYPE *p, TYPE val) { \ - return __clc_atomic_fetch_add(p, val, __ATOMIC_RELAXED, \ + return __clc_atomic_fetch_add((AS TYPE *)p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } diff --git a/libclc/opencl/lib/generic/atomic/atomic_and.cl b/libclc/opencl/lib/generic/atomic/atomic_and.cl index ce1adbb6f8235..3bb1f7a6af273 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_and.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_and.cl @@ -11,7 +11,7 @@ #define __CLC_IMPL(TYPE, AS) \ _CLC_OVERLOAD _CLC_DEF TYPE atomic_and(volatile AS TYPE *p, TYPE val) { \ - return __clc_atomic_fetch_and(p, val, __ATOMIC_RELAXED, \ + return __clc_atomic_fetch_and((AS TYPE *)p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } diff --git a/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl b/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl index 16a8db43e9374..135813b8b879c 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl @@ -12,8 +12,8 @@ #define __CLC_IMPL(TYPE, AS) \ _CLC_OVERLOAD _CLC_DEF TYPE atomic_cmpxchg(volatile AS TYPE *p, TYPE cmp, \ TYPE val) { \ - return __clc_atomic_compare_exchange(p, cmp, val, __ATOMIC_RELAXED, \ - __ATOMIC_RELAXED, \ + return __clc_atomic_compare_exchange((AS TYPE *)p, cmp, val, \ + __ATOMIC_RELAXED, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } diff --git a/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl index 2c1f07d8ca485..f113e82285a02 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl @@ -6,13 +6,12 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_compare_exchange.h> #include <clc/opencl/atomic/atomic_compare_exchange_strong.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_compare_exchange_strong +#define __CLC_IMPL_FUNCTION __clc_atomic_compare_exchange #define __CLC_COMPARE_EXCHANGE #define __CLC_BODY <atomic_def.inc> @@ -20,6 +19,3 @@ #define __CLC_BODY <atomic_def.inc> #include <clc/math/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl index 69bdf37250f70..f5c2899bb18ed 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl @@ -6,13 +6,12 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_compare_exchange.h> #include <clc/opencl/atomic/atomic_compare_exchange_weak.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_compare_exchange_weak +#define __CLC_IMPL_FUNCTION __clc_atomic_compare_exchange #define __CLC_COMPARE_EXCHANGE #define __CLC_BODY <atomic_def.inc> @@ -20,6 +19,3 @@ #define __CLC_BODY <atomic_def.inc> #include <clc/math/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_def.inc b/libclc/opencl/lib/generic/atomic/atomic_def.inc index e6b7c831e10d3..d13ec24db087d 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_def.inc +++ b/libclc/opencl/lib/generic/atomic/atomic_def.inc @@ -25,41 +25,134 @@ #define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE) +#define __CLC_FUNCTION_EXPLICIT __CLC_XCONCAT(__CLC_FUNCTION, _explicit) + +#ifdef __CLC_NO_VALUE_ARG +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, memory_order Order, \ + memory_scope Scope) { \ + return __CLC_IMPL_FUNCTION((ADDRSPACE __CLC_GENTYPE *)Ptr, Order, \ + __opencl_get_clang_memory_scope(Scope)); \ + } +#elif defined(__CLC_RETURN_VOID) +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ + memory_order Order, memory_scope Scope) { \ + __CLC_IMPL_FUNCTION((ADDRSPACE __CLC_GENTYPE *)Ptr, Value, Order, \ + __opencl_get_clang_memory_scope(Scope)); \ + } +#elif defined(__CLC_COMPARE_EXCHANGE) +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF bool __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, \ + ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired, \ + memory_order Success, memory_order Failure, memory_scope Scope) { \ + __CLC_GENTYPE Comparator = *Expected; \ + __CLC_GENTYPE RetValue = __CLC_IMPL_FUNCTION( \ + (ADDRSPACE __CLC_GENTYPE *)Ptr, Comparator, Desired, Success, Failure, \ + __opencl_get_clang_memory_scope(Scope)); \ + if (Comparator != RetValue) { \ + *Expected = RetValue; \ + return false; \ + } \ + return true; \ + } +#else +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ + memory_order Order, memory_scope Scope) { \ + return __CLC_IMPL_FUNCTION((ADDRSPACE __CLC_GENTYPE *)Ptr, Value, Order, \ + __opencl_get_clang_memory_scope(Scope)); \ + } +#endif + +__CLC_DEFINE_ATOMIC(global) +__CLC_DEFINE_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC() +#endif + +#undef __CLC_DEFINE_ATOMIC + +#if defined(__opencl_c_atomic_scope_device) + +#ifdef __CLC_NO_VALUE_ARG +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, memory_order Order) { \ + return __CLC_FUNCTION_EXPLICIT(Ptr, Order, __OPENCL_MEMORY_SCOPE_DEVICE); \ + } +#elif defined(__CLC_RETURN_VOID) +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ + memory_order Order) { \ + __CLC_FUNCTION_EXPLICIT(Ptr, Value, Order, __OPENCL_MEMORY_SCOPE_DEVICE); \ + } +#elif defined(__CLC_COMPARE_EXCHANGE) +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF bool __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, \ + ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired, \ + memory_order Success, memory_order Failure) { \ + return __CLC_FUNCTION_EXPLICIT(Ptr, Expected, Desired, Success, Failure, \ + __OPENCL_MEMORY_SCOPE_DEVICE); \ + } +#else +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION_EXPLICIT( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ + memory_order Order) { \ + return __CLC_FUNCTION_EXPLICIT(Ptr, Value, Order, \ + __OPENCL_MEMORY_SCOPE_DEVICE); \ + } +#endif + +__CLC_DEFINE_ATOMIC(global) +__CLC_DEFINE_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC() +#endif + +#undef __CLC_DEFINE_ATOMIC + +#endif // defined(__opencl_c_atomic_scope_device) + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + #ifdef __CLC_NO_VALUE_ARG #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION( \ volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr) { \ - return __CLC_IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, \ - __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE); \ + return __CLC_FUNCTION_EXPLICIT(Ptr, __ATOMIC_SEQ_CST, \ + __OPENCL_MEMORY_SCOPE_DEVICE); \ } #elif defined(__CLC_RETURN_VOID) #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DEF void __CLC_FUNCTION( \ volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value) { \ - __CLC_IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Value, \ - __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE); \ + __CLC_FUNCTION_EXPLICIT(Ptr, Value, __ATOMIC_SEQ_CST, \ + __OPENCL_MEMORY_SCOPE_DEVICE); \ } #elif defined(__CLC_COMPARE_EXCHANGE) #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ - _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION( \ + _CLC_OVERLOAD _CLC_DEF bool __CLC_FUNCTION( \ volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, \ ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired) { \ - __CLC_GENTYPE Comparator = *Expected; \ - __CLC_GENTYPE RetValue = __clc_atomic_compare_exchange( \ - (volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Comparator, Desired, \ - __ATOMIC_SEQ_CST, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); \ - if (Comparator != RetValue) { \ - *Expected = RetValue; \ - return true; \ - } \ - return false; \ + return __CLC_FUNCTION_EXPLICIT(Ptr, Expected, Desired, __ATOMIC_SEQ_CST, \ + __ATOMIC_SEQ_CST, \ + __OPENCL_MEMORY_SCOPE_DEVICE); \ } #else #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION( \ volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value) { \ - return __CLC_IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Value, \ - __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE); \ + return __CLC_FUNCTION_EXPLICIT(Ptr, Value, __ATOMIC_SEQ_CST, \ + __OPENCL_MEMORY_SCOPE_DEVICE); \ } #endif @@ -71,6 +164,9 @@ __CLC_DEFINE_ATOMIC() #undef __CLC_DEFINE_ATOMIC +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) + #endif // __CLC_HAVE_FP_ATOMIC || __CLC_HAVE_INT_ATOMIC #undef __CLC_HAVE_INT_ATOMIC diff --git a/libclc/opencl/lib/generic/atomic/atomic_exchange.cl b/libclc/opencl/lib/generic/atomic/atomic_exchange.cl index 5f7e2fa593e3f..f7568f6ace38c 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_exchange.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_exchange.cl @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_exchange.h> #include <clc/opencl/atomic/atomic_exchange.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_exchange #define __CLC_IMPL_FUNCTION __clc_atomic_exchange @@ -20,6 +18,3 @@ #define __CLC_BODY <atomic_def.inc> #include <clc/math/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl index 0362ff89d1d78..d27cc7120ccce 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_fetch_add.h> #include <clc/opencl/atomic/atomic_fetch_add.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_fetch_add #define __CLC_IMPL_FUNCTION __clc_atomic_fetch_add @@ -20,6 +18,3 @@ #define __CLC_BODY <atomic_def.inc> #include <clc/math/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl index a1796f20c6e44..b8531722911cf 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl @@ -6,17 +6,12 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_fetch_and.h> #include <clc/opencl/atomic/atomic_fetch_and.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_fetch_and #define __CLC_IMPL_FUNCTION __clc_atomic_fetch_and #define __CLC_BODY <atomic_def.inc> #include <clc/integer/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl index 03b5d1d8ae7bd..b644ca336437a 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_fetch_max.h> #include <clc/opencl/atomic/atomic_fetch_max.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_fetch_max #define __CLC_IMPL_FUNCTION __clc_atomic_fetch_max @@ -20,6 +18,3 @@ #define __CLC_BODY <atomic_def.inc> #include <clc/math/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl index 60ffeff04cc6a..f24fcf329b6f2 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_fetch_min.h> #include <clc/opencl/atomic/atomic_fetch_min.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_fetch_min #define __CLC_IMPL_FUNCTION __clc_atomic_fetch_min @@ -20,6 +18,3 @@ #define __CLC_BODY <atomic_def.inc> #include <clc/math/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl index 8f4100bb150e3..1f6fe4cac090a 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl @@ -6,17 +6,12 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_fetch_or.h> #include <clc/opencl/atomic/atomic_fetch_or.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_fetch_or #define __CLC_IMPL_FUNCTION __clc_atomic_fetch_or #define __CLC_BODY <atomic_def.inc> #include <clc/integer/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl index ecb5b4315ee86..94323a2c0fcb6 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_fetch_sub.h> #include <clc/opencl/atomic/atomic_fetch_sub.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_fetch_sub #define __CLC_IMPL_FUNCTION __clc_atomic_fetch_sub @@ -20,6 +18,3 @@ #define __CLC_BODY <atomic_def.inc> #include <clc/math/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl index c49a55820c8d4..13e1db1124f9a 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl @@ -6,17 +6,12 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_fetch_xor.h> #include <clc/opencl/atomic/atomic_fetch_xor.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_fetch_xor #define __CLC_IMPL_FUNCTION __clc_atomic_fetch_xor #define __CLC_BODY <atomic_def.inc> #include <clc/integer/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc b/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc index dd8dafb38c883..fe517ef7a7de7 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc +++ b/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc @@ -13,7 +13,8 @@ #define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION( \ volatile ADDRSPACE __CLC_GENTYPE *Ptr) { \ - return __CLC_IMPL_FUNCTION(Ptr, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); \ + return __CLC_IMPL_FUNCTION((ADDRSPACE __CLC_GENTYPE *)Ptr, \ + __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); \ } __CLC_DEFINE_ATOMIC(global) diff --git a/libclc/opencl/lib/generic/atomic/atomic_load.cl b/libclc/opencl/lib/generic/atomic/atomic_load.cl index e904330be0064..1b93ce84ea863 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_load.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_load.cl @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_load.h> #include <clc/opencl/atomic/atomic_load.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_load #define __CLC_IMPL_FUNCTION __clc_atomic_load @@ -21,6 +19,3 @@ #define __CLC_BODY <atomic_def.inc> #include <clc/math/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_max.cl b/libclc/opencl/lib/generic/atomic/atomic_max.cl index 362a0ed90ca0e..fe0efdbb55c82 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_max.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_max.cl @@ -9,14 +9,14 @@ #include <clc/atomic/clc_atomic_fetch_max.h> #include <clc/opencl/atomic/atomic_max.h> -#define __CLC_IMPL(TYPE, AS, OP) \ +#define __CLC_IMPL(TYPE, AS) \ _CLC_OVERLOAD _CLC_DEF TYPE atomic_max(volatile AS TYPE *p, TYPE val) { \ - return __clc_atomic_fetch_max(p, val, __ATOMIC_RELAXED, \ + return __clc_atomic_fetch_max((AS TYPE *)p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } -__CLC_IMPL(int, global, max) -__CLC_IMPL(unsigned int, global, umax) -__CLC_IMPL(int, local, max) -__CLC_IMPL(unsigned int, local, umax) +__CLC_IMPL(int, global) +__CLC_IMPL(unsigned int, global) +__CLC_IMPL(int, local) +__CLC_IMPL(unsigned int, local) #undef __CLC_IMPL diff --git a/libclc/opencl/lib/generic/atomic/atomic_min.cl b/libclc/opencl/lib/generic/atomic/atomic_min.cl index 1976be0014d70..6a4586b5d26b6 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_min.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_min.cl @@ -9,14 +9,14 @@ #include <clc/atomic/clc_atomic_fetch_min.h> #include <clc/opencl/atomic/atomic_min.h> -#define __CLC_IMPL(TYPE, AS, OP) \ +#define __CLC_IMPL(TYPE, AS) \ _CLC_OVERLOAD _CLC_DEF TYPE atomic_min(volatile AS TYPE *p, TYPE val) { \ - return __clc_atomic_fetch_min(p, val, __ATOMIC_RELAXED, \ + return __clc_atomic_fetch_min((AS TYPE *)p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } -__CLC_IMPL(int, global, min) -__CLC_IMPL(unsigned int, global, umin) -__CLC_IMPL(int, local, min) -__CLC_IMPL(unsigned int, local, umin) +__CLC_IMPL(int, global) +__CLC_IMPL(unsigned int, global) +__CLC_IMPL(int, local) +__CLC_IMPL(unsigned int, local) #undef __CLC_IMPL diff --git a/libclc/opencl/lib/generic/atomic/atomic_or.cl b/libclc/opencl/lib/generic/atomic/atomic_or.cl index ef8bc00f45593..1720b66b87fff 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_or.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_or.cl @@ -11,7 +11,7 @@ #define __CLC_IMPL(TYPE, AS) \ _CLC_OVERLOAD _CLC_DEF TYPE atomic_or(volatile AS TYPE *p, TYPE val) { \ - return __clc_atomic_fetch_or(p, val, __ATOMIC_RELAXED, \ + return __clc_atomic_fetch_or((AS TYPE *)p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } diff --git a/libclc/opencl/lib/generic/atomic/atomic_store.cl b/libclc/opencl/lib/generic/atomic/atomic_store.cl index 584e29ef99a5f..fcaa4d3128f7d 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_store.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_store.cl @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// -#if defined(__opencl_c_atomic_order_seq_cst) && \ - defined(__opencl_c_atomic_scope_device) - #include <clc/atomic/clc_atomic_store.h> #include <clc/opencl/atomic/atomic_store.h> +#include <clc/opencl/utils.h> #define __CLC_FUNCTION atomic_store #define __CLC_IMPL_FUNCTION __clc_atomic_store @@ -21,6 +19,3 @@ #define __CLC_BODY <atomic_def.inc> #include <clc/math/gentype.inc> - -#endif // defined(__opencl_c_atomic_order_seq_cst) && - // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_sub.cl b/libclc/opencl/lib/generic/atomic/atomic_sub.cl index 397737d113c0d..ddb7cc29d3bb0 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_sub.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_sub.cl @@ -11,7 +11,7 @@ #define __CLC_IMPL(TYPE, AS) \ _CLC_OVERLOAD _CLC_DEF TYPE atomic_sub(volatile AS TYPE *p, TYPE val) { \ - return __clc_atomic_fetch_sub(p, val, __ATOMIC_RELAXED, \ + return __clc_atomic_fetch_sub((AS TYPE *)p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } diff --git a/libclc/opencl/lib/generic/atomic/atomic_xchg.cl b/libclc/opencl/lib/generic/atomic/atomic_xchg.cl index 2b4bbf06d9400..8e0b87cc9343c 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_xchg.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_xchg.cl @@ -11,7 +11,7 @@ #define __CLC_IMPL(TYPE, AS) \ _CLC_OVERLOAD _CLC_DEF TYPE atomic_xchg(volatile AS TYPE *p, TYPE val) { \ - return __clc_atomic_exchange(p, val, __ATOMIC_RELAXED, \ + return __clc_atomic_exchange((AS TYPE *)p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } diff --git a/libclc/opencl/lib/generic/atomic/atomic_xor.cl b/libclc/opencl/lib/generic/atomic/atomic_xor.cl index 1f200c58edbff..46dc6c3e9111f 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_xor.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_xor.cl @@ -11,7 +11,7 @@ #define __CLC_IMPL(TYPE, AS) \ _CLC_OVERLOAD _CLC_DEF TYPE atomic_xor(volatile AS TYPE *p, TYPE val) { \ - return __clc_atomic_fetch_xor(p, val, __ATOMIC_RELAXED, \ + return __clc_atomic_fetch_xor((AS TYPE *)p, val, __ATOMIC_RELAXED, \ __MEMORY_SCOPE_DEVICE); \ } _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
