diff options
| -rw-r--r-- | common/atomic.c | 13 | ||||
| -rw-r--r-- | include/atomic.h | 260 | ||||
| -rw-r--r-- | include/rwlock.h | 2 |
3 files changed, 116 insertions, 159 deletions
diff --git a/common/atomic.c b/common/atomic.c index 634587e4..ab258e8e 100644 --- a/common/atomic.c +++ b/common/atomic.c @@ -4,12 +4,13 @@ #include "atomic.h" -extern inline void InitRef(volatile RefCount *ptr, uint value); -extern inline uint ReadRef(volatile RefCount *ptr); -extern inline uint IncrementRef(volatile RefCount *ptr); -extern inline uint DecrementRef(volatile RefCount *ptr); -extern inline uint ExchangeRef(volatile RefCount *ptr, uint newval); -extern inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval); +extern inline void InitRef(RefCount *ptr, uint value); +extern inline uint ReadRef(RefCount *ptr); +extern inline uint IncrementRef(RefCount *ptr); +extern inline uint DecrementRef(RefCount *ptr); +extern inline uint ExchangeRef(RefCount *ptr, uint newval); +extern inline uint CompExchangeRef(RefCount *ptr, uint oldval, uint newval); + extern inline int ExchangeInt(volatile int *ptr, int newval); extern inline void *ExchangePtr(XchgPtr *ptr, void *newval); extern inline int CompExchangeInt(volatile int *ptr, int oldval, int newval); diff --git a/include/atomic.h b/include/atomic.h index 174aebe9..a35d9be0 100644 --- a/include/atomic.h +++ b/include/atomic.h @@ -10,27 +10,10 @@ extern "C" { typedef void *volatile XchgPtr; typedef unsigned int uint; -typedef union { - uint value; -} RefCount; - -#define STATIC_REFCOUNT_INIT(V) {(V)} +/* Atomics using GCC intrinsics */ #if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) && !defined(__QNXNTO__) -inline void InitRef(volatile RefCount *ptr, uint value) -{ ptr->value = value; } -inline uint ReadRef(volatile RefCount *ptr) -{ __sync_synchronize(); return ptr->value; } -inline uint IncrementRef(volatile RefCount *ptr) -{ return __sync_add_and_fetch(&ptr->value, 1); } -inline uint DecrementRef(volatile RefCount *ptr) -{ return __sync_sub_and_fetch(&ptr->value, 1); } -inline uint ExchangeRef(volatile RefCount *ptr, uint newval) -{ return __sync_lock_test_and_set(&ptr->value, newval); } -inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval) -{ return __sync_val_compare_and_swap(&ptr->value, oldval, newval); } - inline int ExchangeInt(volatile int *ptr, int newval) { return __sync_lock_test_and_set(ptr, newval); } inline void *ExchangePtr(XchgPtr *ptr, void *newval) @@ -56,6 +39,17 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval) __sync_synchronize(); \ } while(0) +#define ATOMIC_ADD(T, _val, _incr) __extension__({ \ + static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \ + T _r = __sync_fetch_and_add(&(_val).value, (_incr)); \ + _r; \ +}) +#define ATOMIC_SUB(T, _val, _decr) __extension__({ \ + static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \ + T _r = __sync_fetch_and_sub(&(_val).value, (_decr)); \ + _r; \ +}) + #define ATOMIC_EXCHANGE(T, _val, _newval) __extension__({ \ static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \ T _r = __sync_lock_test_and_set(&(_val).value, (_newval)); \ @@ -67,52 +61,29 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval) _r; \ }) +/* Atomics using x86/x86-64 GCC inline assembly */ #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) -inline uint xaddl(volatile uint *dest, int incr) -{ - unsigned int ret; - __asm__ __volatile__("lock; xaddl %0,(%1)" - : "=r" (ret) - : "r" (dest), "0" (incr) - : "memory"); - return ret; -} - -inline void InitRef(volatile RefCount *ptr, uint value) -{ ptr->value = value; } -inline uint ReadRef(volatile RefCount *ptr) -{ __asm__ __volatile__("" ::: "memory"); return ptr->value; } -inline uint IncrementRef(volatile RefCount *ptr) -{ return xaddl(&ptr->value, 1)+1; } -inline uint DecrementRef(volatile RefCount *ptr) -{ return xaddl(&ptr->value, -1)-1; } -inline uint ExchangeRef(volatile RefCount *ptr, uint newval) -{ - int ret; - __asm__ __volatile__("lock; xchgl %0,(%1)" - : "=r" (ret) - : "r" (&ptr->value), "0" (newval) - : "memory"); - return ret; -} -inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval) -{ - int ret; - __asm__ __volatile__("lock; cmpxchgl %2,(%1)" - : "=a" (ret) - : "r" (&ptr->value), "r" (newval), "0" (oldval) - : "memory"); - return ret; -} +#define WRAP_ADD(ret, dest, incr) __asm__ __volatile__( \ + "lock; xaddl %0,(%1)" \ + : "=r" (ret) \ + : "r" (dest), "0" (incr) \ + : "memory" \ +) +#define WRAP_SUB(ret, dest, decr) __asm__ __volatile__( \ + "lock; xaddl %0,(%1)" \ + : "=r" (ret) \ + : "r" (dest), "0" (-(decr)) \ + : "memory" \ +) -#define EXCHANGE(S, ret, dest, newval) __asm__ __volatile__( \ +#define WRAP_XCHG(S, ret, dest, newval) __asm__ __volatile__( \ "lock; xchg"S" %0,(%1)" \ : "=r" (ret) \ : "r" (dest), "0" (newval) \ : "memory" \ ) -#define COMP_EXCHANGE(S, ret, dest, oldval, newval) __asm__ __volatile__( \ +#define WRAP_CMPXCHG(S, ret, dest, oldval, newval) __asm__ __volatile__( \ "lock; cmpxchg"S" %2,(%1)" \ : "=a" (ret) \ : "r" (dest), "r" (newval), "0" (oldval) \ @@ -123,13 +94,13 @@ inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval) inline int ExchangeInt(volatile int *dest, int newval) { int ret; - EXCHANGE("l", ret, dest, newval); + WRAP_XCHG("l", ret, dest, newval); return ret; } inline int CompExchangeInt(volatile int *dest, int oldval, int newval) { int ret; - COMP_EXCHANGE("l", ret, dest, oldval, newval); + WRAP_CMPXCHG("l", ret, dest, oldval, newval); return ret; } @@ -137,26 +108,26 @@ inline int CompExchangeInt(volatile int *dest, int oldval, int newval) inline void *ExchangePtr(XchgPtr *dest, void *newval) { void *ret; - EXCHANGE("l", ret, dest, newval); + WRAP_XCHG("l", ret, dest, newval); return ret; } inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval) { void *ret; - COMP_EXCHANGE("l", ret, dest, oldval, newval); + WRAP_CMPXCHG("l", ret, dest, oldval, newval); return ret; } #else inline void *ExchangePtr(XchgPtr *dest, void *newval) { void *ret; - EXCHANGE("q", ret, dest, newval); + WRAP_XCHG("q", ret, dest, newval); return ret; } inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval) { void *ret; - COMP_EXCHANGE("q", ret, dest, oldval, newval); + WRAP_CMPXCHG("q", ret, dest, oldval, newval); return ret; } #endif @@ -171,117 +142,73 @@ inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval) (_val).value = (_newval); \ } while(0) -#define ATOMIC_LOAD(_val) (__asm__ __volatile__("" ::: "memory"),(_val).value) +inline void _al_mem_barrier(void) +{ __asm__ __volatile__("" ::: "memory"); } + +#define ATOMIC_LOAD(_val) (_al_mem_barrier(),(_val).value) #define ATOMIC_STORE(_val, _newval) do { \ (_val).value = (_newval); \ - __asm__ __volatile__("" ::: "memory"); \ + _al_mem_barrier(); \ } while(0) +#define ATOMIC_ADD(T, _val, _incr) __extension__({ \ + T _r; \ + static_assert(sizeof(T)==4, "Type "#T" has incorrect size!"); \ + static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \ + WRAP_ADD(_r, &(_val).value, (_incr)); \ + _r; \ +}) +#define ATOMIC_SUB(T, _val, _decr) __extension__({ \ + T _r; \ + static_assert(sizeof(T)==4, "Type "#T" has incorrect size!"); \ + static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \ + WRAP_SUB(_r, &(_val).value, (_decr)); \ + _r; \ +}) + #define ATOMIC_EXCHANGE(T, _val, _newval) __extension__({ \ T _r; \ static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \ - if(sizeof(T) == 4) EXCHANGE("l", _r, &(_val).value, (_newval)); \ - else if(sizeof(T) == 8) EXCHANGE("q", _r, &(_val).value, (_newval)); \ + if(sizeof(T) == 4) WRAP_XCHG("l", _r, &(_val).value, (_newval)); \ + else if(sizeof(T) == 8) WRAP_XCHG("q", _r, &(_val).value, (_newval)); \ _r; \ }) #define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) __extension__({ \ T _r; \ static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \ - if(sizeof(T) == 4) COMP_EXCHANGE("l", _r, &(_val).value, (_oldval), (_newval)); \ - else if(sizeof(T) == 8) COMP_EXCHANGE("q", _r, &(_val).value, (_oldval), (_newval)); \ + if(sizeof(T) == 4) WRAP_CMPXCHG("l", _r, &(_val).value, (_oldval), (_newval)); \ + else if(sizeof(T) == 8) WRAP_CMPXCHG("q", _r, &(_val).value, (_oldval), (_newval)); \ _r; \ }) +/* Atomics using Windows methods */ #elif defined(_WIN32) #define WIN32_LEAN_AND_MEAN #include <windows.h> -static_assert(sizeof(LONG)==sizeof(uint), "sizeof LONG does not match sizeof uint"); - -inline void InitRef(volatile RefCount *ptr, uint value) -{ ptr->value = value; } -inline uint ReadRef(volatile RefCount *ptr) -{ _ReadBarrier(); return ptr->value; } -inline uint IncrementRef(volatile RefCount *ptr) -{ - union { - volatile uint *u; - volatile LONG *l; - } u = { &ptr->value }; - return InterlockedIncrement(u.l); -} -inline uint DecrementRef(volatile RefCount *ptr) -{ - union { - volatile uint *u; - volatile LONG *l; - } u = { &ptr->value }; - return InterlockedDecrement(u.l); -} -inline uint ExchangeRef(volatile RefCount *ptr, uint newval) -{ - union { - volatile uint *i; - volatile LONG *l; - } u = { &ptr->value }; - return InterlockedExchange(u.l, newval); -} -inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval) -{ - union { - volatile uint *i; - volatile LONG *l; - } u = { &ptr->value }; - return InterlockedCompareExchange(u.l, newval, oldval); -} - -inline int ExchangeInt32(volatile int *ptr, int newval) -{ - union { - volatile int *i; - volatile LONG *l; - } u = { ptr }; - return InterlockedExchange(u.l, newval); -} -inline int CompExchangeInt32(volatile int *ptr, int oldval, int newval) -{ - union { - volatile int *i; - volatile LONG *l; - } u = { ptr }; - return InterlockedCompareExchange(u.l, newval, oldval); -} -inline __int64 ExchangeInt64(volatile __int64 *ptr, __int64 newval) -{ - union { - volatile __int64 *i; - volatile LONGLONG *l; - } u = { ptr }; - return InterlockedExchange64(u.l, newval); -} -inline __int64 CompExchangeInt64(volatile __int64 *ptr, __int64 oldval, __int64 newval) -{ - union { - volatile __int64 *i; - volatile LONGLONG *l; - } u = { ptr }; - return InterlockedCompareExchange64(u.l, newval, oldval); -} +#define RAW_CAST(T1, T2, _val) (((union{T2 from; T1 to;}){.from=(_val)}).to) +#define WRAP_ADD(T1, T2, _func, _ptr, _incr) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_incr)))) +#define WRAP_SUB(T1, T2, _func, _ptr, _decr) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), -RAW_CAST(T1,T2,(_decr)))) +#define WRAP_XCHG(T1, T2, _func, _ptr, _newval) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_newval)))) +#define WRAP_CMPXCHG(T1, T2, _func, _ptr, _oldval, _newval) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_oldval)), RAW_CAST(T1,T2,(_newval)))) inline int ExchangeInt(volatile int *ptr, int newval) -{ return ExchangeInt32(ptr, newval); } +{ return WRAP_XCHG(LONG,int,InterlockedExchange,ptr,newval); } inline int CompExchangeInt(volatile int *ptr, int oldval, int newval) -{ return CompExchangeInt32(ptr, oldval, newval); } +{ return WRAP_CMPXCHG(LONG,int,InterlockedCompareExchange,ptr,newval, oldval); } +#ifdef _WIN64 inline void *ExchangePtr(XchgPtr *ptr, void *newval) -{ - return InterlockedExchangePointer(ptr, newval); -} +{ return WRAP_XCHG(LONGLONG,void*,InterlockedExchange64,ptr,newval); } inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval) -{ - return InterlockedCompareExchangePointer(ptr, newval, oldval); -} +{ return WRAP_CMPXCHG(LONGLONG,void*,InterlockedCompareExchange64,ptr,newval,oldval); } +#else +inline void *ExchangePtr(XchgPtr *ptr, void *newval) +{ return WRAP_XCHG(LONG,void*,InterlockedExchange,ptr,newval); } +inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval) +{ return WRAP_CMPXCHG(LONG,void*,InterlockedCompareExchange,ptr,newval,oldval); } +#endif #define ATOMIC(T) struct { T volatile value; } @@ -293,7 +220,9 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval) (_val).value = (_newval); \ } while(0) -#define ATOMIC_LOAD(_val) (_ReadBarrier(),(_val).value) +inline void _al_mem_barrier(void) { _ReadBarrier(); } + +#define ATOMIC_LOAD(_val) (_al_mem_barrier(),(_val).value) #define ATOMIC_STORE(_val, _newval) do { \ (_val).value = (_newval); \ _WriteBarrier(); \ @@ -301,15 +230,42 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval) int _al_invalid_atomic_size(); /* not defined */ -#define ATOMIC_FUNC_SELECT(T, C, F32, F64) ((sizeof(T) == 4) ? (C)F32 : ((sizeof(T) == 8) ? (C)F64 : (C)_al_invalid_atomic_size)) - -#define ATOMIC_EXCHANGE(T, _val, _newval) (ATOMIC_FUNC_SELECT(T, T(*)(volatile T*,T), ExchangeInt32, ExchangeInt64)(&(_val).value, (_newval))) -#define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) (ATOMIC_FUNC_SELECT(T, T(*)(volatile T*,T,T), CompExchangeInt32, CompExchangeInt64)(&(_val).value, (_oldval), (_newval))) +#define ATOMIC_ADD(T, _val, _incr) \ + ((sizeof(T)==4) ? WRAP_ADD(LONG, T, InterlockedExchangeAdd, &(_val).value, (_incr)) : \ + (T)_al_invalid_atomic_size()) +#define ATOMIC_SUB(T, _val, _decr) \ + ((sizeof(T)==4) ? WRAP_SUB(LONG, T, InterlockedExchangeAdd, &(_val).value, (_decr)) : \ + (T)_al_invalid_atomic_size()) + +#define ATOMIC_EXCHANGE(T, _val, _newval) \ + ((sizeof(T)==4) ? WRAP_XCHG(LONG, T, InterlockedExchange, &(_val).value, (_newval)) : \ + (sizeof(T)==8) ? WRAP_XCHG(LONGLONG, T, InterlockedExchange64, &(_val).value, (_newval)) : \ + (T)_al_invalid_atomic_size()) +#define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) \ + ((sizeof(T)==4) ? WRAP_CMPXCHG(LONG, T, InterlockedCompareExchange, &(_val).value, (_newval), (_oldval)) : \ + (sizeof(T)==8) ? WRAP_CMPXCHG(LONGLONG, T, InterlockedCompareExchange64, &(_val).value, (_newval), (_oldval)) : \ + (T)_al_invalid_atomic_size()) #else #error "No atomic functions available on this platform!" #endif + +typedef ATOMIC(uint) RefCount; + +inline void InitRef(RefCount *ptr, uint value) +{ ATOMIC_STORE_UNSAFE(*ptr, value); } +inline uint ReadRef(RefCount *ptr) +{ return ATOMIC_LOAD(*ptr); } +inline uint IncrementRef(RefCount *ptr) +{ return ATOMIC_ADD(uint, *ptr, 1)+1; } +inline uint DecrementRef(RefCount *ptr) +{ return ATOMIC_SUB(uint, *ptr, 1)-1; } +inline uint ExchangeRef(RefCount *ptr, uint newval) +{ return ATOMIC_EXCHANGE(uint, *ptr, newval); } +inline uint CompExchangeRef(RefCount *ptr, uint oldval, uint newval) +{ return ATOMIC_COMPARE_EXCHANGE(uint, *ptr, oldval, newval); } + #ifdef __cplusplus } #endif diff --git a/include/rwlock.h b/include/rwlock.h index 03482515..f2a8feab 100644 --- a/include/rwlock.h +++ b/include/rwlock.h @@ -15,7 +15,7 @@ typedef struct { volatile int read_entry_lock; volatile int write_lock; } RWLock; -#define RWLOCK_STATIC_INITIALIZE { STATIC_REFCOUNT_INIT(0), STATIC_REFCOUNT_INIT(0), false, false, false } +#define RWLOCK_STATIC_INITIALIZE { ATOMIC_INIT_STATIC(0), ATOMIC_INIT_STATIC(0), false, false, false } void RWLockInit(RWLock *lock); void ReadLock(RWLock *lock); |