aboutsummaryrefslogtreecommitdiffstats
path: root/include/atomic.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/atomic.h')
-rw-r--r--include/atomic.h260
1 files changed, 108 insertions, 152 deletions
diff --git a/include/atomic.h b/include/atomic.h
index 174aebe9..a35d9be0 100644
--- a/include/atomic.h
+++ b/include/atomic.h
@@ -10,27 +10,10 @@ extern "C" {
typedef void *volatile XchgPtr;
typedef unsigned int uint;
-typedef union {
- uint value;
-} RefCount;
-
-#define STATIC_REFCOUNT_INIT(V) {(V)}
+/* Atomics using GCC intrinsics */
#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) && !defined(__QNXNTO__)
-inline void InitRef(volatile RefCount *ptr, uint value)
-{ ptr->value = value; }
-inline uint ReadRef(volatile RefCount *ptr)
-{ __sync_synchronize(); return ptr->value; }
-inline uint IncrementRef(volatile RefCount *ptr)
-{ return __sync_add_and_fetch(&ptr->value, 1); }
-inline uint DecrementRef(volatile RefCount *ptr)
-{ return __sync_sub_and_fetch(&ptr->value, 1); }
-inline uint ExchangeRef(volatile RefCount *ptr, uint newval)
-{ return __sync_lock_test_and_set(&ptr->value, newval); }
-inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval)
-{ return __sync_val_compare_and_swap(&ptr->value, oldval, newval); }
-
inline int ExchangeInt(volatile int *ptr, int newval)
{ return __sync_lock_test_and_set(ptr, newval); }
inline void *ExchangePtr(XchgPtr *ptr, void *newval)
@@ -56,6 +39,17 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
__sync_synchronize(); \
} while(0)
+#define ATOMIC_ADD(T, _val, _incr) __extension__({ \
+ static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
+ T _r = __sync_fetch_and_add(&(_val).value, (_incr)); \
+ _r; \
+})
+#define ATOMIC_SUB(T, _val, _decr) __extension__({ \
+ static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
+ T _r = __sync_fetch_and_sub(&(_val).value, (_decr)); \
+ _r; \
+})
+
#define ATOMIC_EXCHANGE(T, _val, _newval) __extension__({ \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
T _r = __sync_lock_test_and_set(&(_val).value, (_newval)); \
@@ -67,52 +61,29 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
_r; \
})
+/* Atomics using x86/x86-64 GCC inline assembly */
#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
-inline uint xaddl(volatile uint *dest, int incr)
-{
- unsigned int ret;
- __asm__ __volatile__("lock; xaddl %0,(%1)"
- : "=r" (ret)
- : "r" (dest), "0" (incr)
- : "memory");
- return ret;
-}
-
-inline void InitRef(volatile RefCount *ptr, uint value)
-{ ptr->value = value; }
-inline uint ReadRef(volatile RefCount *ptr)
-{ __asm__ __volatile__("" ::: "memory"); return ptr->value; }
-inline uint IncrementRef(volatile RefCount *ptr)
-{ return xaddl(&ptr->value, 1)+1; }
-inline uint DecrementRef(volatile RefCount *ptr)
-{ return xaddl(&ptr->value, -1)-1; }
-inline uint ExchangeRef(volatile RefCount *ptr, uint newval)
-{
- int ret;
- __asm__ __volatile__("lock; xchgl %0,(%1)"
- : "=r" (ret)
- : "r" (&ptr->value), "0" (newval)
- : "memory");
- return ret;
-}
-inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval)
-{
- int ret;
- __asm__ __volatile__("lock; cmpxchgl %2,(%1)"
- : "=a" (ret)
- : "r" (&ptr->value), "r" (newval), "0" (oldval)
- : "memory");
- return ret;
-}
+#define WRAP_ADD(ret, dest, incr) __asm__ __volatile__( \
+ "lock; xaddl %0,(%1)" \
+ : "=r" (ret) \
+ : "r" (dest), "0" (incr) \
+ : "memory" \
+)
+#define WRAP_SUB(ret, dest, decr) __asm__ __volatile__( \
+ "lock; xaddl %0,(%1)" \
+ : "=r" (ret) \
+ : "r" (dest), "0" (-(decr)) \
+ : "memory" \
+)
-#define EXCHANGE(S, ret, dest, newval) __asm__ __volatile__( \
+#define WRAP_XCHG(S, ret, dest, newval) __asm__ __volatile__( \
"lock; xchg"S" %0,(%1)" \
: "=r" (ret) \
: "r" (dest), "0" (newval) \
: "memory" \
)
-#define COMP_EXCHANGE(S, ret, dest, oldval, newval) __asm__ __volatile__( \
+#define WRAP_CMPXCHG(S, ret, dest, oldval, newval) __asm__ __volatile__( \
"lock; cmpxchg"S" %2,(%1)" \
: "=a" (ret) \
: "r" (dest), "r" (newval), "0" (oldval) \
@@ -123,13 +94,13 @@ inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval)
inline int ExchangeInt(volatile int *dest, int newval)
{
int ret;
- EXCHANGE("l", ret, dest, newval);
+ WRAP_XCHG("l", ret, dest, newval);
return ret;
}
inline int CompExchangeInt(volatile int *dest, int oldval, int newval)
{
int ret;
- COMP_EXCHANGE("l", ret, dest, oldval, newval);
+ WRAP_CMPXCHG("l", ret, dest, oldval, newval);
return ret;
}
@@ -137,26 +108,26 @@ inline int CompExchangeInt(volatile int *dest, int oldval, int newval)
inline void *ExchangePtr(XchgPtr *dest, void *newval)
{
void *ret;
- EXCHANGE("l", ret, dest, newval);
+ WRAP_XCHG("l", ret, dest, newval);
return ret;
}
inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval)
{
void *ret;
- COMP_EXCHANGE("l", ret, dest, oldval, newval);
+ WRAP_CMPXCHG("l", ret, dest, oldval, newval);
return ret;
}
#else
inline void *ExchangePtr(XchgPtr *dest, void *newval)
{
void *ret;
- EXCHANGE("q", ret, dest, newval);
+ WRAP_XCHG("q", ret, dest, newval);
return ret;
}
inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval)
{
void *ret;
- COMP_EXCHANGE("q", ret, dest, oldval, newval);
+ WRAP_CMPXCHG("q", ret, dest, oldval, newval);
return ret;
}
#endif
@@ -171,117 +142,73 @@ inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval)
(_val).value = (_newval); \
} while(0)
-#define ATOMIC_LOAD(_val) (__asm__ __volatile__("" ::: "memory"),(_val).value)
+inline void _al_mem_barrier(void)
+{ __asm__ __volatile__("" ::: "memory"); }
+
+#define ATOMIC_LOAD(_val) (_al_mem_barrier(),(_val).value)
#define ATOMIC_STORE(_val, _newval) do { \
(_val).value = (_newval); \
- __asm__ __volatile__("" ::: "memory"); \
+ _al_mem_barrier(); \
} while(0)
+#define ATOMIC_ADD(T, _val, _incr) __extension__({ \
+ T _r; \
+ static_assert(sizeof(T)==4, "Type "#T" has incorrect size!"); \
+ static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
+ WRAP_ADD(_r, &(_val).value, (_incr)); \
+ _r; \
+})
+#define ATOMIC_SUB(T, _val, _decr) __extension__({ \
+ T _r; \
+ static_assert(sizeof(T)==4, "Type "#T" has incorrect size!"); \
+ static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
+ WRAP_SUB(_r, &(_val).value, (_decr)); \
+ _r; \
+})
+
#define ATOMIC_EXCHANGE(T, _val, _newval) __extension__({ \
T _r; \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
- if(sizeof(T) == 4) EXCHANGE("l", _r, &(_val).value, (_newval)); \
- else if(sizeof(T) == 8) EXCHANGE("q", _r, &(_val).value, (_newval)); \
+ if(sizeof(T) == 4) WRAP_XCHG("l", _r, &(_val).value, (_newval)); \
+ else if(sizeof(T) == 8) WRAP_XCHG("q", _r, &(_val).value, (_newval)); \
_r; \
})
#define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) __extension__({ \
T _r; \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
- if(sizeof(T) == 4) COMP_EXCHANGE("l", _r, &(_val).value, (_oldval), (_newval)); \
- else if(sizeof(T) == 8) COMP_EXCHANGE("q", _r, &(_val).value, (_oldval), (_newval)); \
+ if(sizeof(T) == 4) WRAP_CMPXCHG("l", _r, &(_val).value, (_oldval), (_newval)); \
+ else if(sizeof(T) == 8) WRAP_CMPXCHG("q", _r, &(_val).value, (_oldval), (_newval)); \
_r; \
})
+/* Atomics using Windows methods */
#elif defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
-static_assert(sizeof(LONG)==sizeof(uint), "sizeof LONG does not match sizeof uint");
-
-inline void InitRef(volatile RefCount *ptr, uint value)
-{ ptr->value = value; }
-inline uint ReadRef(volatile RefCount *ptr)
-{ _ReadBarrier(); return ptr->value; }
-inline uint IncrementRef(volatile RefCount *ptr)
-{
- union {
- volatile uint *u;
- volatile LONG *l;
- } u = { &ptr->value };
- return InterlockedIncrement(u.l);
-}
-inline uint DecrementRef(volatile RefCount *ptr)
-{
- union {
- volatile uint *u;
- volatile LONG *l;
- } u = { &ptr->value };
- return InterlockedDecrement(u.l);
-}
-inline uint ExchangeRef(volatile RefCount *ptr, uint newval)
-{
- union {
- volatile uint *i;
- volatile LONG *l;
- } u = { &ptr->value };
- return InterlockedExchange(u.l, newval);
-}
-inline uint CompExchangeRef(volatile RefCount *ptr, uint oldval, uint newval)
-{
- union {
- volatile uint *i;
- volatile LONG *l;
- } u = { &ptr->value };
- return InterlockedCompareExchange(u.l, newval, oldval);
-}
-
-inline int ExchangeInt32(volatile int *ptr, int newval)
-{
- union {
- volatile int *i;
- volatile LONG *l;
- } u = { ptr };
- return InterlockedExchange(u.l, newval);
-}
-inline int CompExchangeInt32(volatile int *ptr, int oldval, int newval)
-{
- union {
- volatile int *i;
- volatile LONG *l;
- } u = { ptr };
- return InterlockedCompareExchange(u.l, newval, oldval);
-}
-inline __int64 ExchangeInt64(volatile __int64 *ptr, __int64 newval)
-{
- union {
- volatile __int64 *i;
- volatile LONGLONG *l;
- } u = { ptr };
- return InterlockedExchange64(u.l, newval);
-}
-inline __int64 CompExchangeInt64(volatile __int64 *ptr, __int64 oldval, __int64 newval)
-{
- union {
- volatile __int64 *i;
- volatile LONGLONG *l;
- } u = { ptr };
- return InterlockedCompareExchange64(u.l, newval, oldval);
-}
+#define RAW_CAST(T1, T2, _val) (((union{T2 from; T1 to;}){.from=(_val)}).to)
+#define WRAP_ADD(T1, T2, _func, _ptr, _incr) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_incr))))
+#define WRAP_SUB(T1, T2, _func, _ptr, _decr) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), -RAW_CAST(T1,T2,(_decr))))
+#define WRAP_XCHG(T1, T2, _func, _ptr, _newval) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_newval))))
+#define WRAP_CMPXCHG(T1, T2, _func, _ptr, _oldval, _newval) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_oldval)), RAW_CAST(T1,T2,(_newval))))
inline int ExchangeInt(volatile int *ptr, int newval)
-{ return ExchangeInt32(ptr, newval); }
+{ return WRAP_XCHG(LONG,int,InterlockedExchange,ptr,newval); }
inline int CompExchangeInt(volatile int *ptr, int oldval, int newval)
-{ return CompExchangeInt32(ptr, oldval, newval); }
+{ return WRAP_CMPXCHG(LONG,int,InterlockedCompareExchange,ptr,newval, oldval); }
+#ifdef _WIN64
inline void *ExchangePtr(XchgPtr *ptr, void *newval)
-{
- return InterlockedExchangePointer(ptr, newval);
-}
+{ return WRAP_XCHG(LONGLONG,void*,InterlockedExchange64,ptr,newval); }
inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
-{
- return InterlockedCompareExchangePointer(ptr, newval, oldval);
-}
+{ return WRAP_CMPXCHG(LONGLONG,void*,InterlockedCompareExchange64,ptr,newval,oldval); }
+#else
+inline void *ExchangePtr(XchgPtr *ptr, void *newval)
+{ return WRAP_XCHG(LONG,void*,InterlockedExchange,ptr,newval); }
+inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
+{ return WRAP_CMPXCHG(LONG,void*,InterlockedCompareExchange,ptr,newval,oldval); }
+#endif
#define ATOMIC(T) struct { T volatile value; }
@@ -293,7 +220,9 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
(_val).value = (_newval); \
} while(0)
-#define ATOMIC_LOAD(_val) (_ReadBarrier(),(_val).value)
+inline void _al_mem_barrier(void) { _ReadBarrier(); }
+
+#define ATOMIC_LOAD(_val) (_al_mem_barrier(),(_val).value)
#define ATOMIC_STORE(_val, _newval) do { \
(_val).value = (_newval); \
_WriteBarrier(); \
@@ -301,15 +230,42 @@ inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
int _al_invalid_atomic_size(); /* not defined */
-#define ATOMIC_FUNC_SELECT(T, C, F32, F64) ((sizeof(T) == 4) ? (C)F32 : ((sizeof(T) == 8) ? (C)F64 : (C)_al_invalid_atomic_size))
-
-#define ATOMIC_EXCHANGE(T, _val, _newval) (ATOMIC_FUNC_SELECT(T, T(*)(volatile T*,T), ExchangeInt32, ExchangeInt64)(&(_val).value, (_newval)))
-#define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) (ATOMIC_FUNC_SELECT(T, T(*)(volatile T*,T,T), CompExchangeInt32, CompExchangeInt64)(&(_val).value, (_oldval), (_newval)))
+#define ATOMIC_ADD(T, _val, _incr) \
+ ((sizeof(T)==4) ? WRAP_ADD(LONG, T, InterlockedExchangeAdd, &(_val).value, (_incr)) : \
+ (T)_al_invalid_atomic_size())
+#define ATOMIC_SUB(T, _val, _decr) \
+ ((sizeof(T)==4) ? WRAP_SUB(LONG, T, InterlockedExchangeAdd, &(_val).value, (_decr)) : \
+ (T)_al_invalid_atomic_size())
+
+#define ATOMIC_EXCHANGE(T, _val, _newval) \
+ ((sizeof(T)==4) ? WRAP_XCHG(LONG, T, InterlockedExchange, &(_val).value, (_newval)) : \
+ (sizeof(T)==8) ? WRAP_XCHG(LONGLONG, T, InterlockedExchange64, &(_val).value, (_newval)) : \
+ (T)_al_invalid_atomic_size())
+#define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) \
+ ((sizeof(T)==4) ? WRAP_CMPXCHG(LONG, T, InterlockedCompareExchange, &(_val).value, (_newval), (_oldval)) : \
+ (sizeof(T)==8) ? WRAP_CMPXCHG(LONGLONG, T, InterlockedCompareExchange64, &(_val).value, (_newval), (_oldval)) : \
+ (T)_al_invalid_atomic_size())
#else
#error "No atomic functions available on this platform!"
#endif
+
+typedef ATOMIC(uint) RefCount;
+
+inline void InitRef(RefCount *ptr, uint value)
+{ ATOMIC_STORE_UNSAFE(*ptr, value); }
+inline uint ReadRef(RefCount *ptr)
+{ return ATOMIC_LOAD(*ptr); }
+inline uint IncrementRef(RefCount *ptr)
+{ return ATOMIC_ADD(uint, *ptr, 1)+1; }
+inline uint DecrementRef(RefCount *ptr)
+{ return ATOMIC_SUB(uint, *ptr, 1)-1; }
+inline uint ExchangeRef(RefCount *ptr, uint newval)
+{ return ATOMIC_EXCHANGE(uint, *ptr, newval); }
+inline uint CompExchangeRef(RefCount *ptr, uint oldval, uint newval)
+{ return ATOMIC_COMPARE_EXCHANGE(uint, *ptr, oldval, newval); }
+
#ifdef __cplusplus
}
#endif
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-14 23:41:25 +0000