Implement RefCount as a generic atomic type

3 files changed, 116 insertions, 159 deletions

diff --git a/common/atomic.c b/common/atomic.c
index 634587e4..ab258e8e 100644
--- a/common/atomic.c
+++ b/common/atomic.c
@@ -4,12 +4,13 @@
 #include "atomic.h"
 
 
-extern inline void InitRef(volatile RefCount *ptr, uint value);
-extern inline uint ReadRef(volatile RefCount *ptr);
-extern inline uint IncrementRef(volatile RefCount *ptr);
-extern inline uint DecrementRef(volatile RefCount *ptr);
-extern inline uint ExchangeRef(volatile RefCount *ptr, uint newval);
-extern inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval);
+extern inline void InitRef(RefCount *ptr, uint value);
+extern inline uint ReadRef(RefCount *ptr);
+extern inline uint IncrementRef(RefCount *ptr);
+extern inline uint DecrementRef(RefCount *ptr);
+extern inline uint ExchangeRef(RefCount *ptr, uint newval);
+extern inline uint CompExchangeRef(RefCount *ptr, uint oldval, uint newval);
+
 extern inline int ExchangeInt(volatile int *ptr, int newval);
 extern inline void *ExchangePtr(XchgPtr *ptr, void *newval);
 extern inline int CompExchangeInt(volatile int *ptr, int oldval, int newval);
diff --git a/include/atomic.h b/include/atomic.h
index 174aebe9..a35d9be0 100644
--- a/include/atomic.h
+++ b/include/atomic.h
@@ -10,27 +10,10 @@ extern "C" {
 typedef void *volatile XchgPtr;
 
 typedef unsigned int uint;
-typedef union {
-    uint value;
-} RefCount;
-
-#define STATIC_REFCOUNT_INIT(V)  {(V)}
 
+/* Atomics using GCC intrinsics */
 #if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) && !defined(__QNXNTO__)
 
-inline void InitRef(volatile RefCount *ptr, uint value)
-{ ptr->value = value; }
-inline uint ReadRef(volatile RefCount *ptr)
-{ __sync_synchronize(); return ptr->value; }
-inline uint IncrementRef(volatile RefCount *ptr)
-{ return __sync_add_and_fetch(&ptr->value, 1); }
-inline uint DecrementRef(volatile RefCount *ptr)
-{ return __sync_sub_and_fetch(&ptr->value, 1); }
-inline uint ExchangeRef(volatile RefCount *ptr, uint newval)
-{ return __sync_lock_test_and_set(&ptr->value, newval); }
-inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval)
-{ return __sync_val_compare_and_swap(&ptr->value, oldval, newval); }
-
 inline int ExchangeInt(volatile int *ptr, int newval)
 { return __sync_lock_test_and_set(ptr, newval); }
 inline void *ExchangePtr(XchgPtr *ptr, void *newval)
@@ -56,6 +39,17 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
     __sync_synchronize();                  \
 } while(0)
 
+#define ATOMIC_ADD(T, _val, _incr)  __extension__({                           \
+    static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
+    T _r = __sync_fetch_and_add(&(_val).value, (_incr));                      \
+    _r;                                                                       \
+})
+#define ATOMIC_SUB(T, _val, _decr)  __extension__({                           \
+    static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
+    T _r = __sync_fetch_and_sub(&(_val).value, (_decr));                      \
+    _r;                                                                       \
+})
+
 #define ATOMIC_EXCHANGE(T, _val, _newval)  __extension__({                    \
     static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
     T _r = __sync_lock_test_and_set(&(_val).value, (_newval));                \
@@ -67,52 +61,29 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
     _r;                                                                       \
 })
 
+/* Atomics using x86/x86-64 GCC inline assembly */
 #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
 
-inline uint xaddl(volatile uint *dest, int incr)
-{
-    unsigned int ret;
-    __asm__ __volatile__("lock; xaddl %0,(%1)"
-                         : "=r" (ret)
-                         : "r" (dest), "0" (incr)
-                         : "memory");
-    return ret;
-}
-
-inline void InitRef(volatile RefCount *ptr, uint value)
-{ ptr->value = value; }
-inline uint ReadRef(volatile RefCount *ptr)
-{ __asm__ __volatile__("" ::: "memory"); return ptr->value; }
-inline uint IncrementRef(volatile RefCount *ptr)
-{ return xaddl(&ptr->value, 1)+1; }
-inline uint DecrementRef(volatile RefCount *ptr)
-{ return xaddl(&ptr->value, -1)-1; }
-inline uint ExchangeRef(volatile RefCount *ptr, uint newval)
-{
-    int ret;
-    __asm__ __volatile__("lock; xchgl %0,(%1)"
-                         : "=r" (ret)
-                         : "r" (&ptr->value), "0" (newval)
-                         : "memory");
-    return ret;
-}
-inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval)
-{
-    int ret;
-    __asm__ __volatile__("lock; cmpxchgl %2,(%1)"
-                         : "=a" (ret)
-                         : "r" (&ptr->value), "r" (newval), "0" (oldval)
-                         : "memory");
-    return ret;
-}
+#define WRAP_ADD(ret, dest, incr) __asm__ __volatile__(                       \
+    "lock; xaddl %0,(%1)"                                                     \
+    : "=r" (ret)                                                              \
+    : "r" (dest), "0" (incr)                                                  \
+    : "memory"                                                                \
+)
+#define WRAP_SUB(ret, dest, decr) __asm__ __volatile__(                       \
+    "lock; xaddl %0,(%1)"                                                     \
+    : "=r" (ret)                                                              \
+    : "r" (dest), "0" (-(decr))                                               \
+    : "memory"                                                                \
+)
 
-#define EXCHANGE(S, ret, dest, newval) __asm__ __volatile__(                  \
+#define WRAP_XCHG(S, ret, dest, newval) __asm__ __volatile__(                 \
     "lock; xchg"S" %0,(%1)"                                                   \
     : "=r" (ret)                                                              \
     : "r" (dest), "0" (newval)                                                \
     : "memory"                                                                \
 )
-#define COMP_EXCHANGE(S, ret, dest, oldval, newval) __asm__ __volatile__(     \
+#define WRAP_CMPXCHG(S, ret, dest, oldval, newval) __asm__ __volatile__(      \
     "lock; cmpxchg"S" %2,(%1)"                                                \
     : "=a" (ret)                                                              \
     : "r" (dest), "r" (newval), "0" (oldval)                                  \
@@ -123,13 +94,13 @@ inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval)
 inline int ExchangeInt(volatile int *dest, int newval)
 {
     int ret;
-    EXCHANGE("l", ret, dest, newval);
+    WRAP_XCHG("l", ret, dest, newval);
     return ret;
 }
 inline int CompExchangeInt(volatile int *dest, int oldval, int newval)
 {
     int ret;
-    COMP_EXCHANGE("l", ret, dest, oldval, newval);
+    WRAP_CMPXCHG("l", ret, dest, oldval, newval);
     return ret;
 }
 
@@ -137,26 +108,26 @@ inline int CompExchangeInt(volatile int *dest, int oldval, int newval)
 inline void *ExchangePtr(XchgPtr *dest, void *newval)
 {
     void *ret;
-    EXCHANGE("l", ret, dest, newval);
+    WRAP_XCHG("l", ret, dest, newval);
     return ret;
 }
 inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval)
 {
     void *ret;
-    COMP_EXCHANGE("l", ret, dest, oldval, newval);
+    WRAP_CMPXCHG("l", ret, dest, oldval, newval);
     return ret;
 }
 #else
 inline void *ExchangePtr(XchgPtr *dest, void *newval)
 {
     void *ret;
-    EXCHANGE("q", ret, dest, newval);
+    WRAP_XCHG("q", ret, dest, newval);
     return ret;
 }
 inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval)
 {
     void *ret;
-    COMP_EXCHANGE("q", ret, dest, oldval, newval);
+    WRAP_CMPXCHG("q", ret, dest, oldval, newval);
     return ret;
 }
 #endif
@@ -171,117 +142,73 @@ inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval)
     (_val).value = (_newval);                     \
 } while(0)
 
-#define ATOMIC_LOAD(_val)  (__asm__ __volatile__("" ::: "memory"),(_val).value)
+inline void _al_mem_barrier(void)
+{ __asm__ __volatile__("" ::: "memory"); }
+
+#define ATOMIC_LOAD(_val)  (_al_mem_barrier(),(_val).value)
 #define ATOMIC_STORE(_val, _newval)  do {  \
     (_val).value = (_newval);              \
-    __asm__ __volatile__("" ::: "memory"); \
+    _al_mem_barrier();                     \
 } while(0)
 
+#define ATOMIC_ADD(T, _val, _incr)  __extension__({                           \
+    T _r;                                                                     \
+    static_assert(sizeof(T)==4, "Type "#T" has incorrect size!");             \
+    static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
+    WRAP_ADD(_r, &(_val).value, (_incr));                                     \
+    _r;                                                                       \
+})
+#define ATOMIC_SUB(T, _val, _decr)  __extension__({                           \
+    T _r;                                                                     \
+    static_assert(sizeof(T)==4, "Type "#T" has incorrect size!");             \
+    static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
+    WRAP_SUB(_r, &(_val).value, (_decr));                                     \
+    _r;                                                                       \
+})
+
 #define ATOMIC_EXCHANGE(T, _val, _newval)  __extension__({                    \
     T _r;                                                                     \
     static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
-    if(sizeof(T) == 4) EXCHANGE("l", _r, &(_val).value, (_newval));           \
-    else if(sizeof(T) == 8) EXCHANGE("q", _r, &(_val).value, (_newval));      \
+    if(sizeof(T) == 4) WRAP_XCHG("l", _r, &(_val).value, (_newval));          \
+    else if(sizeof(T) == 8) WRAP_XCHG("q", _r, &(_val).value, (_newval));     \
     _r;                                                                       \
 })
 #define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) __extension__({    \
     T _r;                                                                     \
     static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
-    if(sizeof(T) == 4) COMP_EXCHANGE("l", _r, &(_val).value, (_oldval), (_newval)); \
-    else if(sizeof(T) == 8) COMP_EXCHANGE("q", _r, &(_val).value, (_oldval), (_newval)); \
+    if(sizeof(T) == 4) WRAP_CMPXCHG("l", _r, &(_val).value, (_oldval), (_newval)); \
+    else if(sizeof(T) == 8) WRAP_CMPXCHG("q", _r, &(_val).value, (_oldval), (_newval)); \
     _r;                                                                       \
 })
 
+/* Atomics using Windows methods */
 #elif defined(_WIN32)
 
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 
-static_assert(sizeof(LONG)==sizeof(uint), "sizeof LONG does not match sizeof uint");
-
-inline void InitRef(volatile RefCount *ptr, uint value)
-{ ptr->value = value; }
-inline uint ReadRef(volatile RefCount *ptr)
-{ _ReadBarrier(); return ptr->value; }
-inline uint IncrementRef(volatile RefCount *ptr)
-{
-    union {
-        volatile uint *u;
-        volatile LONG *l;
-    } u = { &ptr->value };
-    return InterlockedIncrement(u.l);
-}
-inline uint DecrementRef(volatile RefCount *ptr)
-{
-    union {
-        volatile uint *u;
-        volatile LONG *l;
-    } u = { &ptr->value };
-    return InterlockedDecrement(u.l);
-}
-inline uint ExchangeRef(volatile RefCount *ptr, uint newval)
-{
-    union {
-        volatile uint *i;
-        volatile LONG *l;
-    } u = { &ptr->value };
-    return InterlockedExchange(u.l, newval);
-}
-inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval)
-{
-    union {
-        volatile uint *i;
-        volatile LONG *l;
-    } u = { &ptr->value };
-    return InterlockedCompareExchange(u.l, newval, oldval);
-}
-
-inline int ExchangeInt32(volatile int *ptr, int newval)
-{
-    union {
-        volatile int *i;
-        volatile LONG *l;
-    } u = { ptr };
-    return InterlockedExchange(u.l, newval);
-}
-inline int CompExchangeInt32(volatile int *ptr, int oldval, int newval)
-{
-    union {
-        volatile int *i;
-        volatile LONG *l;
-    } u = { ptr };
-    return InterlockedCompareExchange(u.l, newval, oldval);
-}
-inline __int64 ExchangeInt64(volatile __int64 *ptr, __int64 newval)
-{
-    union {
-        volatile __int64 *i;
-        volatile LONGLONG *l;
-    } u = { ptr };
-    return InterlockedExchange64(u.l, newval);
-}
-inline __int64 CompExchangeInt64(volatile __int64 *ptr, __int64 oldval, __int64 newval)
-{
-    union {
-        volatile __int64 *i;
-        volatile LONGLONG *l;
-    } u = { ptr };
-    return InterlockedCompareExchange64(u.l, newval, oldval);
-}
+#define RAW_CAST(T1, T2, _val)  (((union{T2 from; T1 to;}){.from=(_val)}).to)
+#define WRAP_ADD(T1, T2, _func, _ptr, _incr)  RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_incr))))
+#define WRAP_SUB(T1, T2, _func, _ptr, _decr)  RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), -RAW_CAST(T1,T2,(_decr))))
+#define WRAP_XCHG(T1, T2, _func, _ptr, _newval)  RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_newval))))
+#define WRAP_CMPXCHG(T1, T2, _func, _ptr, _oldval, _newval)  RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_oldval)), RAW_CAST(T1,T2,(_newval))))
 
 inline int ExchangeInt(volatile int *ptr, int newval)
-{ return ExchangeInt32(ptr, newval); }
+{ return WRAP_XCHG(LONG,int,InterlockedExchange,ptr,newval); }
 inline int CompExchangeInt(volatile int *ptr, int oldval, int newval)
-{ return CompExchangeInt32(ptr, oldval, newval); }
+{ return WRAP_CMPXCHG(LONG,int,InterlockedCompareExchange,ptr,newval, oldval); }
 
+#ifdef _WIN64
 inline void *ExchangePtr(XchgPtr *ptr, void *newval)
-{
-    return InterlockedExchangePointer(ptr, newval);
-}
+{ return WRAP_XCHG(LONGLONG,void*,InterlockedExchange64,ptr,newval); }
 inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
-{
-    return InterlockedCompareExchangePointer(ptr, newval, oldval);
-}
+{ return WRAP_CMPXCHG(LONGLONG,void*,InterlockedCompareExchange64,ptr,newval,oldval); }
+#else
+inline void *ExchangePtr(XchgPtr *ptr, void *newval)
+{ return WRAP_XCHG(LONG,void*,InterlockedExchange,ptr,newval); }
+inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
+{ return WRAP_CMPXCHG(LONG,void*,InterlockedCompareExchange,ptr,newval,oldval); }
+#endif
 
 
 #define ATOMIC(T)  struct { T volatile value; }
@@ -293,7 +220,9 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
     (_val).value = (_newval);                     \
 } while(0)
 
-#define ATOMIC_LOAD(_val)  (_ReadBarrier(),(_val).value)
+inline void _al_mem_barrier(void) { _ReadBarrier(); }
+
+#define ATOMIC_LOAD(_val)  (_al_mem_barrier(),(_val).value)
 #define ATOMIC_STORE(_val, _newval)  do {  \
     (_val).value = (_newval);              \
     _WriteBarrier();                       \
@@ -301,15 +230,42 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
 
 int _al_invalid_atomic_size(); /* not defined */
 
-#define ATOMIC_FUNC_SELECT(T, C, F32, F64)  ((sizeof(T) == 4) ? (C)F32 : ((sizeof(T) == 8) ? (C)F64 : (C)_al_invalid_atomic_size))
-
-#define ATOMIC_EXCHANGE(T, _val, _newval)  (ATOMIC_FUNC_SELECT(T, T(*)(volatile T*,T), ExchangeInt32, ExchangeInt64)(&(_val).value, (_newval)))
-#define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) (ATOMIC_FUNC_SELECT(T, T(*)(volatile T*,T,T), CompExchangeInt32, CompExchangeInt64)(&(_val).value, (_oldval), (_newval)))
+#define ATOMIC_ADD(T, _val, _incr)                                            \
+    ((sizeof(T)==4) ? WRAP_ADD(LONG, T, InterlockedExchangeAdd, &(_val).value, (_incr)) : \
+     (T)_al_invalid_atomic_size())
+#define ATOMIC_SUB(T, _val, _decr)                                            \
+    ((sizeof(T)==4) ? WRAP_SUB(LONG, T, InterlockedExchangeAdd, &(_val).value, (_decr)) : \
+     (T)_al_invalid_atomic_size())
+
+#define ATOMIC_EXCHANGE(T, _val, _newval)                                     \
+    ((sizeof(T)==4) ? WRAP_XCHG(LONG, T, InterlockedExchange, &(_val).value, (_newval)) : \
+     (sizeof(T)==8) ? WRAP_XCHG(LONGLONG, T, InterlockedExchange64, &(_val).value, (_newval)) : \
+     (T)_al_invalid_atomic_size())
+#define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval)                    \
+    ((sizeof(T)==4) ? WRAP_CMPXCHG(LONG, T, InterlockedCompareExchange, &(_val).value, (_newval), (_oldval)) : \
+     (sizeof(T)==8) ? WRAP_CMPXCHG(LONGLONG, T, InterlockedCompareExchange64, &(_val).value, (_newval), (_oldval)) : \
+     (T)_al_invalid_atomic_size())
 
 #else
 #error "No atomic functions available on this platform!"
 #endif
 
+
+typedef ATOMIC(uint) RefCount;
+
+inline void InitRef(RefCount *ptr, uint value)
+{ ATOMIC_STORE_UNSAFE(*ptr, value); }
+inline uint ReadRef(RefCount *ptr)
+{ return ATOMIC_LOAD(*ptr); }
+inline uint IncrementRef(RefCount *ptr)
+{ return ATOMIC_ADD(uint, *ptr, 1)+1; }
+inline uint DecrementRef(RefCount *ptr)
+{ return ATOMIC_SUB(uint, *ptr, 1)-1; }
+inline uint ExchangeRef(RefCount *ptr, uint newval)
+{ return ATOMIC_EXCHANGE(uint, *ptr, newval); }
+inline uint CompExchangeRef(RefCount *ptr, uint oldval, uint newval)
+{ return ATOMIC_COMPARE_EXCHANGE(uint, *ptr, oldval, newval); }
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/rwlock.h b/include/rwlock.h
index 03482515..f2a8feab 100644
--- a/include/rwlock.h
+++ b/include/rwlock.h
@@ -15,7 +15,7 @@ typedef struct {
     volatile int read_entry_lock;
     volatile int write_lock;
 } RWLock;
-#define RWLOCK_STATIC_INITIALIZE { STATIC_REFCOUNT_INIT(0), STATIC_REFCOUNT_INIT(0), false, false, false }
+#define RWLOCK_STATIC_INITIALIZE { ATOMIC_INIT_STATIC(0), ATOMIC_INIT_STATIC(0), false, false, false }
 
 void RWLockInit(RWLock *lock);
 void ReadLock(RWLock *lock);