1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
|
#ifndef AL_ATOMIC_H
#define AL_ATOMIC_H
#include "static_assert.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef void *volatile XchgPtr;
typedef unsigned int uint;
/* Atomics using GCC intrinsics */
#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) && !defined(__QNXNTO__)
inline int ExchangeInt(volatile int *ptr, int newval)
{ return __sync_lock_test_and_set(ptr, newval); }
inline void *ExchangePtr(XchgPtr *ptr, void *newval)
{ return __sync_lock_test_and_set(ptr, newval); }
inline int CompExchangeInt(volatile int *ptr, int oldval, int newval)
{ return __sync_val_compare_and_swap(ptr, oldval, newval); }
inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
{ return __sync_val_compare_and_swap(ptr, oldval, newval); }
#define ATOMIC(T) struct { T volatile value; }
#define ATOMIC_INIT_STATIC(_newval) {(_newval)}
#define ATOMIC_LOAD_UNSAFE(_val) ((_val).value)
#define ATOMIC_STORE_UNSAFE(_val, _newval) do { \
(_val).value = (_newval); \
} while(0)
#define ATOMIC_LOAD(_val) (__sync_synchronize(),(_val).value)
#define ATOMIC_STORE(_val, _newval) do { \
(_val).value = (_newval); \
__sync_synchronize(); \
} while(0)
#define ATOMIC_ADD(T, _val, _incr) __extension__({ \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
T _r = __sync_fetch_and_add(&(_val).value, (_incr)); \
_r; \
})
#define ATOMIC_SUB(T, _val, _decr) __extension__({ \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
T _r = __sync_fetch_and_sub(&(_val).value, (_decr)); \
_r; \
})
#define ATOMIC_EXCHANGE(T, _val, _newval) __extension__({ \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
T _r = __sync_lock_test_and_set(&(_val).value, (_newval)); \
_r; \
})
#define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) __extension__({ \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
T _r = __sync_val_compare_and_swap(&(_val).value, (_oldval), (_newval)); \
_r; \
})
/* Atomics using x86/x86-64 GCC inline assembly */
#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
#define WRAP_ADD(ret, dest, incr) __asm__ __volatile__( \
"lock; xaddl %0,(%1)" \
: "=r" (ret) \
: "r" (dest), "0" (incr) \
: "memory" \
)
#define WRAP_SUB(ret, dest, decr) __asm__ __volatile__( \
"lock; xaddl %0,(%1)" \
: "=r" (ret) \
: "r" (dest), "0" (-(decr)) \
: "memory" \
)
#define WRAP_XCHG(S, ret, dest, newval) __asm__ __volatile__( \
"lock; xchg"S" %0,(%1)" \
: "=r" (ret) \
: "r" (dest), "0" (newval) \
: "memory" \
)
#define WRAP_CMPXCHG(S, ret, dest, oldval, newval) __asm__ __volatile__( \
"lock; cmpxchg"S" %2,(%1)" \
: "=a" (ret) \
: "r" (dest), "r" (newval), "0" (oldval) \
: "memory" \
)
inline int ExchangeInt(volatile int *dest, int newval)
{
int ret;
WRAP_XCHG("l", ret, dest, newval);
return ret;
}
inline int CompExchangeInt(volatile int *dest, int oldval, int newval)
{
int ret;
WRAP_CMPXCHG("l", ret, dest, oldval, newval);
return ret;
}
#ifdef __i386__
inline void *ExchangePtr(XchgPtr *dest, void *newval)
{
void *ret;
WRAP_XCHG("l", ret, dest, newval);
return ret;
}
inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval)
{
void *ret;
WRAP_CMPXCHG("l", ret, dest, oldval, newval);
return ret;
}
#else
inline void *ExchangePtr(XchgPtr *dest, void *newval)
{
void *ret;
WRAP_XCHG("q", ret, dest, newval);
return ret;
}
inline void *CompExchangePtr(XchgPtr *dest, void *oldval, void *newval)
{
void *ret;
WRAP_CMPXCHG("q", ret, dest, oldval, newval);
return ret;
}
#endif
#define ATOMIC(T) struct { T volatile value; }
#define ATOMIC_INIT_STATIC(_newval) {(_newval)}
#define ATOMIC_LOAD_UNSAFE(_val) ((_val).value)
#define ATOMIC_STORE_UNSAFE(_val, _newval) do { \
(_val).value = (_newval); \
} while(0)
inline void _al_mem_barrier(void)
{ __asm__ __volatile__("" ::: "memory"); }
#define ATOMIC_LOAD(_val) (_al_mem_barrier(),(_val).value)
#define ATOMIC_STORE(_val, _newval) do { \
(_val).value = (_newval); \
_al_mem_barrier(); \
} while(0)
#define ATOMIC_ADD(T, _val, _incr) __extension__({ \
T _r; \
static_assert(sizeof(T)==4, "Type "#T" has incorrect size!"); \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
WRAP_ADD(_r, &(_val).value, (_incr)); \
_r; \
})
#define ATOMIC_SUB(T, _val, _decr) __extension__({ \
T _r; \
static_assert(sizeof(T)==4, "Type "#T" has incorrect size!"); \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
WRAP_SUB(_r, &(_val).value, (_decr)); \
_r; \
})
#define ATOMIC_EXCHANGE(T, _val, _newval) __extension__({ \
T _r; \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
if(sizeof(T) == 4) WRAP_XCHG("l", _r, &(_val).value, (_newval)); \
else if(sizeof(T) == 8) WRAP_XCHG("q", _r, &(_val).value, (_newval)); \
_r; \
})
#define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) __extension__({ \
T _r; \
static_assert(sizeof(T)==sizeof((_val).value), "Type "#T" has incorrect size!"); \
if(sizeof(T) == 4) WRAP_CMPXCHG("l", _r, &(_val).value, (_oldval), (_newval)); \
else if(sizeof(T) == 8) WRAP_CMPXCHG("q", _r, &(_val).value, (_oldval), (_newval)); \
_r; \
})
/* Atomics using Windows methods */
#elif defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#define RAW_CAST(T1, T2, _val) (((union{T2 from; T1 to;}){.from=(_val)}).to)
#define WRAP_ADD(T1, T2, _func, _ptr, _incr) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_incr))))
#define WRAP_SUB(T1, T2, _func, _ptr, _decr) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), -RAW_CAST(T1,T2,(_decr))))
#define WRAP_XCHG(T1, T2, _func, _ptr, _newval) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_newval))))
#define WRAP_CMPXCHG(T1, T2, _func, _ptr, _oldval, _newval) RAW_CAST(T2,T1,_func(RAW_CAST(T1 volatile*,T2 volatile*,(_ptr)), RAW_CAST(T1,T2,(_oldval)), RAW_CAST(T1,T2,(_newval))))
inline int ExchangeInt(volatile int *ptr, int newval)
{ return WRAP_XCHG(LONG,int,InterlockedExchange,ptr,newval); }
inline int CompExchangeInt(volatile int *ptr, int oldval, int newval)
{ return WRAP_CMPXCHG(LONG,int,InterlockedCompareExchange,ptr,newval, oldval); }
#ifdef _WIN64
inline void *ExchangePtr(XchgPtr *ptr, void *newval)
{ return WRAP_XCHG(LONGLONG,void*,InterlockedExchange64,ptr,newval); }
inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
{ return WRAP_CMPXCHG(LONGLONG,void*,InterlockedCompareExchange64,ptr,newval,oldval); }
#else
inline void *ExchangePtr(XchgPtr *ptr, void *newval)
{ return WRAP_XCHG(LONG,void*,InterlockedExchange,ptr,newval); }
inline void *CompExchangePtr(XchgPtr *ptr, void *oldval, void *newval)
{ return WRAP_CMPXCHG(LONG,void*,InterlockedCompareExchange,ptr,newval,oldval); }
#endif
#define ATOMIC(T) struct { T volatile value; }
#define ATOMIC_INIT_STATIC(_newval) {(_newval)}
#define ATOMIC_LOAD_UNSAFE(_val) ((_val).value)
#define ATOMIC_STORE_UNSAFE(_val, _newval) do { \
(_val).value = (_newval); \
} while(0)
inline void _al_mem_barrier(void) { _ReadBarrier(); }
#define ATOMIC_LOAD(_val) (_al_mem_barrier(),(_val).value)
#define ATOMIC_STORE(_val, _newval) do { \
(_val).value = (_newval); \
_WriteBarrier(); \
} while(0)
int _al_invalid_atomic_size(); /* not defined */
#define ATOMIC_ADD(T, _val, _incr) \
((sizeof(T)==4) ? WRAP_ADD(LONG, T, InterlockedExchangeAdd, &(_val).value, (_incr)) : \
(T)_al_invalid_atomic_size())
#define ATOMIC_SUB(T, _val, _decr) \
((sizeof(T)==4) ? WRAP_SUB(LONG, T, InterlockedExchangeAdd, &(_val).value, (_decr)) : \
(T)_al_invalid_atomic_size())
#define ATOMIC_EXCHANGE(T, _val, _newval) \
((sizeof(T)==4) ? WRAP_XCHG(LONG, T, InterlockedExchange, &(_val).value, (_newval)) : \
(sizeof(T)==8) ? WRAP_XCHG(LONGLONG, T, InterlockedExchange64, &(_val).value, (_newval)) : \
(T)_al_invalid_atomic_size())
#define ATOMIC_COMPARE_EXCHANGE(T, _val, _oldval, _newval) \
((sizeof(T)==4) ? WRAP_CMPXCHG(LONG, T, InterlockedCompareExchange, &(_val).value, (_newval), (_oldval)) : \
(sizeof(T)==8) ? WRAP_CMPXCHG(LONGLONG, T, InterlockedCompareExchange64, &(_val).value, (_newval), (_oldval)) : \
(T)_al_invalid_atomic_size())
#else
#error "No atomic functions available on this platform!"
#endif
typedef ATOMIC(uint) RefCount;
inline void InitRef(RefCount *ptr, uint value)
{ ATOMIC_STORE_UNSAFE(*ptr, value); }
inline uint ReadRef(RefCount *ptr)
{ return ATOMIC_LOAD(*ptr); }
inline uint IncrementRef(RefCount *ptr)
{ return ATOMIC_ADD(uint, *ptr, 1)+1; }
inline uint DecrementRef(RefCount *ptr)
{ return ATOMIC_SUB(uint, *ptr, 1)-1; }
inline uint ExchangeRef(RefCount *ptr, uint newval)
{ return ATOMIC_EXCHANGE(uint, *ptr, newval); }
inline uint CompExchangeRef(RefCount *ptr, uint oldval, uint newval)
{ return ATOMIC_COMPARE_EXCHANGE(uint, *ptr, oldval, newval); }
#ifdef __cplusplus
}
#endif
#endif /* AL_ATOMIC_H */
|
