diff options
Diffstat (limited to 'include/atomic.h')
| -rw-r--r-- | include/atomic.h | 448 |
1 files changed, 287 insertions, 161 deletions
diff --git a/include/atomic.h b/include/atomic.h index 4adb7a94..8eb6820b 100644 --- a/include/atomic.h +++ b/include/atomic.h @@ -2,204 +2,330 @@ #define AL_ATOMIC_H #include "static_assert.h" +#include "bool.h" #ifdef __cplusplus extern "C" { #endif -typedef void *volatile XchgPtr; +/* Atomics using C11 */ +#ifdef HAVE_C11_ATOMIC -typedef unsigned int uint; -typedef union { - uint value; -} RefCount; - -#define STATIC_REFCOUNT_INIT(V) {(V)} - -#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) && !defined(__QNXNTO__) - -inline void InitRef(volatile RefCount *ptr, uint value) -{ ptr->value = value; } -inline uint ReadRef(volatile RefCount *ptr) -{ __sync_synchronize(); return ptr->value; } -inline uint IncrementRef(volatile RefCount *ptr) -{ return __sync_add_and_fetch(&ptr->value, 1); } -inline uint DecrementRef(volatile RefCount *ptr) -{ return __sync_sub_and_fetch(&ptr->value, 1); } -inline uint ExchangeRef(volatile RefCount *ptr, uint newval) -{ return __sync_lock_test_and_set(&ptr->value, newval); } -inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval) -{ return __sync_val_compare_and_swap(&ptr->value, oldval, newval); } +#include <stdatomic.h> -inline int ExchangeInt(volatile int *ptr, int newval) -{ return __sync_lock_test_and_set(ptr, newval); } -inline void *ExchangePtr(XchgPtr *ptr, void *newval) -{ return __sync_lock_test_and_set(ptr, newval); } -inline int CompExchangeInt(volatile int *ptr, int oldval, int newval) -{ return __sync_val_compare_and_swap(ptr, oldval, newval); } -inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval) -{ return __sync_val_compare_and_swap(ptr, oldval, newval); } +#define almemory_order memory_order +#define almemory_order_relaxed memory_order_relaxed +#define almemory_order_consume memory_order_consume +#define almemory_order_acquire memory_order_acquire +#define almemory_order_release memory_order_release +#define almemory_order_acq_rel memory_order_acq_rel +#define almemory_order_seq_cst memory_order_seq_cst + +#define ATOMIC(T) T _Atomic + +#define ATOMIC_INIT(_val, _newval) atomic_init((_val), (_newval)) +#define ATOMIC_INIT_STATIC(_newval) ATOMIC_VAR_INIT(_newval) + +#define PARAM2(f, a, b, ...) (f((a), (b))) +#define PARAM3(f, a, b, c, ...) (f((a), (b), (c))) +#define PARAM5(f, a, b, c, d, e, ...) (f((a), (b), (c), (d), (e))) + +#define ATOMIC_LOAD(...) PARAM2(atomic_load_explicit, __VA_ARGS__, memory_order_seq_cst) +#define ATOMIC_STORE(...) PARAM3(atomic_store_explicit, __VA_ARGS__, memory_order_seq_cst) + +#define ATOMIC_ADD(T, ...) PARAM3(atomic_fetch_add_explicit, __VA_ARGS__, memory_order_seq_cst) +#define ATOMIC_SUB(T, ...) PARAM3(atomic_fetch_sub_explicit, __VA_ARGS__, memory_order_seq_cst) + +#define ATOMIC_EXCHANGE(T, ...) PARAM3(atomic_exchange_explicit, __VA_ARGS__, memory_order_seq_cst) +#define ATOMIC_COMPARE_EXCHANGE_STRONG(T, ...) \ + PARAM5(atomic_compare_exchange_strong_explicit, __VA_ARGS__, memory_order_seq_cst, memory_order_seq_cst) +#define ATOMIC_COMPARE_EXCHANGE_WEAK(T, ...) \ + PARAM5(atomic_compare_exchange_weak_explicit, __VA_ARGS__, memory_order_seq_cst, memory_order_seq_cst) +/* Atomics using GCC intrinsics */ +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) && !defined(__QNXNTO__) + +enum almemory_order { + almemory_order_relaxed, + almemory_order_consume, + almemory_order_acquire, + almemory_order_release, + almemory_order_acq_rel, + almemory_order_seq_cst +}; + +#define ATOMIC(T) struct { T volatile value; } + +#define ATOMIC_INIT(_val, _newval) do { (_val)->value = (_newval); } while(0) +#define ATOMIC_INIT_STATIC(_newval) {(_newval)} + +#define ATOMIC_LOAD(_val, ...) __extension__({ \ + __typeof((_val)->value) _r = (_val)->value; \ + __asm__ __volatile__("" ::: "memory"); \ + _r; \ +}) +#define ATOMIC_STORE(_val, _newval, ...) do { \ + __asm__ __volatile__("" ::: "memory"); \ + (_val)->value = (_newval); \ +} while(0) + +#define ATOMIC_ADD(T, _val, _incr, ...) __extension__({ \ + static_assert(sizeof(T)==sizeof((_val)->value), "Type "#T" has incorrect size!"); \ + __sync_fetch_and_add(&(_val)->value, (_incr)); \ +}) +#define ATOMIC_SUB(T, _val, _decr, ...) __extension__({ \ + static_assert(sizeof(T)==sizeof((_val)->value), "Type "#T" has incorrect size!"); \ + __sync_fetch_and_sub(&(_val)->value, (_decr)); \ +}) + +#define ATOMIC_EXCHANGE(T, _val, _newval, ...) __extension__({ \ + static_assert(sizeof(T)==sizeof((_val)->value), "Type "#T" has incorrect size!"); \ + __sync_lock_test_and_set(&(_val)->value, (_newval)); \ +}) +#define ATOMIC_COMPARE_EXCHANGE_STRONG(T, _val, _oldval, _newval, ...) __extension__({ \ + static_assert(sizeof(T)==sizeof((_val)->value), "Type "#T" has incorrect size!"); \ + T _o = *(_oldval); \ + *(_oldval) = __sync_val_compare_and_swap(&(_val)->value, _o, (_newval)); \ + *(_oldval) == _o; \ +}) + +/* Atomics using x86/x86-64 GCC inline assembly */ #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) -inline uint xaddl(volatile uint *dest, int incr) -{ - unsigned int ret; - __asm__ __volatile__("lock; xaddl %0,(%1)" - : "=r" (ret) - : "r" (dest), "0" (incr) - : "memory"); - return ret; -} +#define WRAP_ADD(ret, dest, incr) __asm__ __volatile__( \ + "lock; xaddl %0,(%1)" \ + : "=r" (ret) \ + : "r" (dest), "0" (incr) \ + : "memory" \ +) +#define WRAP_SUB(ret, dest, decr) __asm__ __volatile__( \ + "lock; xaddl %0,(%1)" \ + : "=r" (ret) \ + : "r" (dest), "0" (-(decr)) \ + : "memory" \ +) -inline void InitRef(volatile RefCount *ptr, uint value) -{ ptr->value = value; } -inline uint ReadRef(volatile RefCount *ptr) -{ __asm__ __volatile__("" ::: "memory"); return ptr->value; } -inline uint IncrementRef(volatile RefCount *ptr) -{ return xaddl(&ptr->value, 1)+1; } -inline uint DecrementRef(volatile RefCount *ptr) -{ return xaddl(&ptr->value, -1)-1; } -inline uint ExchangeRef(volatile RefCount *ptr, uint newval) -{ - int ret; - __asm__ __volatile__("lock; xchgl %0,(%1)" - : "=r" (ret) - : "r" (&ptr->value), "0" (newval) - : "memory"); - return ret; -} -inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval) -{ - int ret; - __asm__ __volatile__("lock; cmpxchgl %2,(%1)" - : "=a" (ret) - : "r" (&ptr->value), "r" (newval), "0" (oldval) - : "memory"); - return ret; -} +#define WRAP_XCHG(S, ret, dest, newval) __asm__ __volatile__( \ + "lock; xchg"S" %0,(%1)" \ + : "=r" (ret) \ + : "r" (dest), "0" (newval) \ + : "memory" \ +) +#define WRAP_CMPXCHG(S, ret, dest, oldval, newval) __asm__ __volatile__( \ + "lock; cmpxchg"S" %2,(%1)" \ + : "=a" (ret) \ + : "r" (dest), "r" (newval), "0" (oldval) \ + : "memory" \ +) -inline int ExchangeInt(volatile int *dest, int newval) -{ - int ret; - __asm__ __volatile__("lock; xchgl %0,(%1)" - : "=r" (ret) - : "r" (dest), "0" (newval) - : "memory"); - return ret; -} -inline void *ExchangePtr(XchgPtr *dest, void *newval) -{ - void *ret; - __asm__ __volatile__( -#ifdef __i386__ - "lock; xchgl %0,(%1)" -#else - "lock; xchgq %0,(%1)" -#endif - : "=r" (ret) - : "r" (dest), "0" (newval) - : "memory" - ); - return ret; -} -inline int CompExchangeInt(volatile int *dest, int oldval, int newval) -{ - int ret; - __asm__ __volatile__("lock; cmpxchgl %2,(%1)" - : "=a" (ret) - : "r" (dest), "r" (newval), "0" (oldval) - : "memory"); - return ret; -} -inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval) -{ - void *ret; - __asm__ __volatile__( -#ifdef __i386__ - "lock; cmpxchgl %2,(%1)" -#else - "lock; cmpxchgq %2,(%1)" -#endif - : "=a" (ret) - : "r" (dest), "r" (newval), "0" (oldval) - : "memory" - ); - return ret; -} +enum almemory_order { + almemory_order_relaxed, + almemory_order_consume, + almemory_order_acquire, + almemory_order_release, + almemory_order_acq_rel, + almemory_order_seq_cst +}; + +#define ATOMIC(T) struct { T volatile value; } + +#define ATOMIC_INIT(_val, _newval) do { (_val)->value = (_newval); } while(0) +#define ATOMIC_INIT_STATIC(_newval) {(_newval)} + +#define ATOMIC_LOAD(_val, ...) __extension__({ \ + __typeof((_val)->value) _r = (_val)->value; \ + __asm__ __volatile__("" ::: "memory"); \ + _r; \ +}) +#define ATOMIC_STORE(_val, _newval, ...) do { \ + __asm__ __volatile__("" ::: "memory"); \ + (_val)->value = (_newval); \ +} while(0) + +#define ATOMIC_ADD(T, _val, _incr, ...) __extension__({ \ + static_assert(sizeof(T)==4, "Type "#T" has incorrect size!"); \ + static_assert(sizeof(T)==sizeof((_val)->value), "Type "#T" has incorrect size!"); \ + T _r; \ + WRAP_ADD(_r, &(_val)->value, (T)(_incr)); \ + _r; \ +}) +#define ATOMIC_SUB(T, _val, _decr, ...) __extension__({ \ + static_assert(sizeof(T)==4, "Type "#T" has incorrect size!"); \ + static_assert(sizeof(T)==sizeof((_val)->value), "Type "#T" has incorrect size!"); \ + T _r; \ + WRAP_SUB(_r, &(_val)->value, (T)(_decr)); \ + _r; \ +}) + +#define ATOMIC_EXCHANGE(T, _val, _newval, ...) __extension__({ \ + static_assert(sizeof(T)==4 || sizeof(T)==8, "Type "#T" has incorrect size!"); \ + static_assert(sizeof(T)==sizeof((_val)->value), "Type "#T" has incorrect size!"); \ + T _r; \ + if(sizeof(T) == 4) WRAP_XCHG("l", _r, &(_val)->value, (T)(_newval)); \ + else if(sizeof(T) == 8) WRAP_XCHG("q", _r, &(_val)->value, (T)(_newval)); \ + _r; \ +}) +#define ATOMIC_COMPARE_EXCHANGE_STRONG(T, _val, _oldval, _newval, ...) __extension__({ \ + static_assert(sizeof(T)==4 || sizeof(T)==8, "Type "#T" has incorrect size!"); \ + static_assert(sizeof(T)==sizeof((_val)->value), "Type "#T" has incorrect size!"); \ + T _old = *(_oldval); \ + if(sizeof(T) == 4) WRAP_CMPXCHG("l", *(_oldval), &(_val)->value, _old, (T)(_newval)); \ + else if(sizeof(T) == 8) WRAP_CMPXCHG("q", *(_oldval), &(_val)->value, _old, (T)(_newval)); \ + *(_oldval) == _old; \ +}) + +/* Atomics using Windows methods */ #elif defined(_WIN32) #define WIN32_LEAN_AND_MEAN #include <windows.h> -static_assert(sizeof(LONG)==sizeof(uint), "sizeof LONG does not match sizeof uint"); +/* NOTE: This mess is *extremely* noisy, at least on GCC. It works by wrapping + * Windows' 32-bit and 64-bit atomic methods, which are then casted to use the + * given type based on its size (e.g. int and float use 32-bit atomics). This + * is fine for the swap and compare-and-swap methods, although the add and + * subtract methods only work properly for integer types. + * + * Despite how noisy it is, it's unfortunately the only way that doesn't rely + * on C99 (damn MSVC). + */ -inline void InitRef(volatile RefCount *ptr, uint value) -{ ptr->value = value; } -inline uint ReadRef(volatile RefCount *ptr) -{ _ReadBarrier(); return ptr->value; } -inline uint IncrementRef(volatile RefCount *ptr) +inline LONG AtomicAdd32(volatile LONG *dest, LONG incr) { - union { - volatile uint *u; - volatile LONG *l; - } u = { &ptr->value }; - return InterlockedIncrement(u.l); + return InterlockedExchangeAdd(dest, incr); } -inline uint DecrementRef(volatile RefCount *ptr) +inline LONG AtomicSub32(volatile LONG *dest, LONG decr) { - union { - volatile uint *u; - volatile LONG *l; - } u = { &ptr->value }; - return InterlockedDecrement(u.l); + return InterlockedExchangeAdd(dest, -decr); } -inline uint ExchangeRef(volatile RefCount *ptr, uint newval) + +inline LONG AtomicSwap32(volatile LONG *dest, LONG newval) { - union { - volatile uint *i; - volatile LONG *l; - } u = { &ptr->value }; - return InterlockedExchange(u.l, newval); + return InterlockedExchange(dest, newval); } -inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval) +inline LONGLONG AtomicSwap64(volatile LONGLONG *dest, LONGLONG newval) { - union { - volatile uint *i; - volatile LONG *l; - } u = { &ptr->value }; - return InterlockedCompareExchange(u.l, newval, oldval); + return InterlockedExchange64(dest, newval); } -inline int ExchangeInt(volatile int *ptr, int newval) +inline bool CompareAndSwap32(volatile LONG *dest, LONG newval, LONG *oldval) { - union { - volatile int *i; - volatile LONG *l; - } u = { ptr }; - return InterlockedExchange(u.l, newval); + LONG old = *oldval; + *oldval = InterlockedCompareExchange(dest, newval, *oldval); + return old == *oldval; } -inline void *ExchangePtr(XchgPtr *ptr, void *newval) +inline bool CompareAndSwap64(volatile LONGLONG *dest, LONGLONG newval, LONGLONG *oldval) { - return InterlockedExchangePointer(ptr, newval); + LONGLONG old = *oldval; + *oldval = InterlockedCompareExchange64(dest, newval, *oldval); + return old == *oldval; } -inline int CompExchangeInt(volatile int *ptr, int oldval, int newval) + +#define WRAP_ADDSUB(T, _func, _ptr, _amnt) ((T(*)(T volatile*,T))_func)((_ptr), (_amnt)) +#define WRAP_XCHG(T, _func, _ptr, _newval) ((T(*)(T volatile*,T))_func)((_ptr), (_newval)) +#define WRAP_CMPXCHG(T, _func, _ptr, _newval, _oldval) ((bool(*)(T volatile*,T,T*))_func)((_ptr), (_newval), (_oldval)) + + +enum almemory_order { + almemory_order_relaxed, + almemory_order_consume, + almemory_order_acquire, + almemory_order_release, + almemory_order_acq_rel, + almemory_order_seq_cst +}; + +#define ATOMIC(T) struct { T volatile value; } + +#define ATOMIC_INIT(_val, _newval) do { (_val)->value = (_newval); } while(0) +#define ATOMIC_INIT_STATIC(_newval) {(_newval)} + +#define ATOMIC_LOAD(_val, ...) ((_val)->value) +#define ATOMIC_STORE(_val, _newval, ...) do { \ + (_val)->value = (_newval); \ +} while(0) + +int _al_invalid_atomic_size(); /* not defined */ + +#define ATOMIC_ADD(T, _val, _incr, ...) \ + ((sizeof(T)==4) ? WRAP_ADDSUB(T, AtomicAdd32, &(_val)->value, (_incr)) : \ + (T)_al_invalid_atomic_size()) +#define ATOMIC_SUB(T, _val, _decr, ...) \ + ((sizeof(T)==4) ? WRAP_ADDSUB(T, AtomicSub32, &(_val)->value, (_decr)) : \ + (T)_al_invalid_atomic_size()) + +#define ATOMIC_EXCHANGE(T, _val, _newval, ...) \ + ((sizeof(T)==4) ? WRAP_XCHG(T, AtomicSwap32, &(_val)->value, (_newval)) : \ + (sizeof(T)==8) ? WRAP_XCHG(T, AtomicSwap64, &(_val)->value, (_newval)) : \ + (T)_al_invalid_atomic_size()) +#define ATOMIC_COMPARE_EXCHANGE_STRONG(T, _val, _oldval, _newval, ...) \ + ((sizeof(T)==4) ? WRAP_CMPXCHG(T, CompareAndSwap32, &(_val)->value, (_newval), (_oldval)) : \ + (sizeof(T)==8) ? WRAP_CMPXCHG(T, CompareAndSwap64, &(_val)->value, (_newval), (_oldval)) : \ + (bool)_al_invalid_atomic_size()) + +#else + +#error "No atomic functions available on this platform!" + +#define ATOMIC(T) T + +#define ATOMIC_INIT_STATIC(_newval) (0) + +#define ATOMIC_LOAD_UNSAFE(_val) (0) +#define ATOMIC_STORE_UNSAFE(_val, _newval) ((void)0) + +#define ATOMIC_LOAD(_val, ...) (0) +#define ATOMIC_STORE(_val, _newval, ...) ((void)0) + +#define ATOMIC_ADD(T, _val, _incr, ...) (0) +#define ATOMIC_SUB(T, _val, _decr, ...) (0) + +#define ATOMIC_EXCHANGE(T, _val, _newval, ...) (0) +#define ATOMIC_COMPARE_EXCHANGE_STRONG(T, _val, _oldval, _newval, ...) (0) +#endif + +/* If no weak cmpxchg is provided (not all systems will have one), substitute a + * strong cmpxchg. */ +#ifndef ATOMIC_COMPARE_EXCHANGE_WEAK +#define ATOMIC_COMPARE_EXCHANGE_WEAK ATOMIC_COMPARE_EXCHANGE_STRONG +#endif + + +typedef unsigned int uint; +typedef ATOMIC(uint) RefCount; + +inline void InitRef(RefCount *ptr, uint value) +{ ATOMIC_INIT(ptr, value); } +inline uint ReadRef(RefCount *ptr) +{ return ATOMIC_LOAD(ptr); } +inline uint IncrementRef(RefCount *ptr) +{ return ATOMIC_ADD(uint, ptr, 1)+1; } +inline uint DecrementRef(RefCount *ptr) +{ return ATOMIC_SUB(uint, ptr, 1)-1; } + + +/* NOTE: Not atomic! */ +inline int ExchangeInt(volatile int *ptr, int newval) { - union { - volatile int *i; - volatile LONG *l; - } u = { ptr }; - return InterlockedCompareExchange(u.l, newval, oldval); + int old = *ptr; + *ptr = newval; + return old; } -inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval) + +typedef void *volatile XchgPtr; +/* NOTE: Not atomic! */ +inline void *ExchangePtr(XchgPtr *ptr, void *newval) { - return InterlockedCompareExchangePointer(ptr, newval, oldval); + void *old = *ptr; + *ptr = newval; + return old; } -#else -#error "No atomic functions available on this platform!" -#endif +/* This is *NOT* atomic, but is a handy utility macro to compare-and-swap non- + * atomic variables. */ +#define COMPARE_EXCHANGE(_val, _oldval, _newval) ((*(_val) == *(_oldval)) ? ((*(_val)=(_newval)),true) : ((*(_oldval)=*(_val)),false)) + #ifdef __cplusplus } |