ReactOS 0.4.17-dev-683-g0dafdc5
atomic.h
Go to the documentation of this file.
1#pragma once
2
3/*
4 * Most of this header is a giant workaround for MSVC to make atomics into a
5 * somewhat unified interface with how GCC and Clang handle them.
6 *
7 * We don't use the absolutely disgusting C11 stdatomic.h header because it is
8 * unable to operate on non _Atomic types, which enforce implicit sequential
9 * consistency and alter the behavior of the standard C binary/unary operators.
10 *
11 * The strictness of the atomic helpers defined here is assumed to be at least
12 * acquire for loads and release for stores. Cmpxchg uses the standard acq/rel
13 * for success, acq for failure, and is assumed to be strong.
14 */
15
16#ifdef UACPI_OVERRIDE_ATOMIC
17#include "uacpi_atomic.h"
18#else
19
20#include <uacpi/platform/compiler.h>
21
22#if defined(_MSC_VER) && !defined(__clang__)
23
24#include <intrin.h>
25
26// mimic __atomic_compare_exchange_n that doesn't exist on MSVC
27#define UACPI_MAKE_MSVC_CMPXCHG(width, type, suffix) \
28 static inline int uacpi_do_atomic_cmpxchg##width( \
29 type volatile *ptr, type volatile *expected, type desired \
30 ) \
31 { \
32 type current; \
33 \
34 current = _InterlockedCompareExchange##suffix(ptr, *expected, desired); \
35 if (current != *expected) { \
36 *expected = current; \
37 return 0; \
38 } \
39 return 1; \
40 }
41
42#define UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, width, type) \
43 uacpi_do_atomic_cmpxchg##width( \
44 (type volatile*)ptr, (type volatile*)expected, desired \
45 )
46
47#define UACPI_MSVC_ATOMIC_STORE(ptr, value, type, width) \
48 _InterlockedExchange##width((type volatile*)(ptr), (type)(value))
49
50#define UACPI_MSVC_ATOMIC_LOAD(ptr, type, width) \
51 _InterlockedOr##width((type volatile*)(ptr), 0)
52
53#define UACPI_MSVC_ATOMIC_INC(ptr, type, width) \
54 _InterlockedIncrement##width((type volatile*)(ptr))
55
56#define UACPI_MSVC_ATOMIC_DEC(ptr, type, width) \
57 _InterlockedDecrement##width((type volatile*)(ptr))
58
59UACPI_MAKE_MSVC_CMPXCHG(64, __int64, 64)
60UACPI_MAKE_MSVC_CMPXCHG(32, long,)
61UACPI_MAKE_MSVC_CMPXCHG(16, short, 16)
62
63#define uacpi_atomic_cmpxchg16(ptr, expected, desired) \
64 UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, 16, short)
65
66#define uacpi_atomic_cmpxchg32(ptr, expected, desired) \
67 UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, 32, long)
68
69#define uacpi_atomic_cmpxchg64(ptr, expected, desired) \
70 UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, 64, __int64)
71
72#define uacpi_atomic_load8(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, char, 8)
73#define uacpi_atomic_load16(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, short, 16)
74#define uacpi_atomic_load32(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, long,)
75#define uacpi_atomic_load64(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, __int64, 64)
76
77#define uacpi_atomic_store8(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, char, 8)
78#define uacpi_atomic_store16(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, short, 16)
79#define uacpi_atomic_store32(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, long,)
80#define uacpi_atomic_store64(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, __int64, 64)
81
82#define uacpi_atomic_inc16(ptr) UACPI_MSVC_ATOMIC_INC(ptr, short, 16)
83#define uacpi_atomic_inc32(ptr) UACPI_MSVC_ATOMIC_INC(ptr, long,)
84#define uacpi_atomic_inc64(ptr) UACPI_MSVC_ATOMIC_INC(ptr, __int64, 64)
85
86#define uacpi_atomic_dec16(ptr) UACPI_MSVC_ATOMIC_DEC(ptr, short, 16)
87#define uacpi_atomic_dec32(ptr) UACPI_MSVC_ATOMIC_DEC(ptr, long,)
88#define uacpi_atomic_dec64(ptr) UACPI_MSVC_ATOMIC_DEC(ptr, __int64, 64)
89#elif defined(__WATCOMC__)
90
91#include <stdint.h>
92
93static int uacpi_do_atomic_cmpxchg16(volatile uint16_t *ptr, volatile uint16_t *expected, uint16_t desired);
94#pragma aux uacpi_do_atomic_cmpxchg16 = \
95 ".486" \
96 "mov ax, [esi]" \
97 "lock cmpxchg [edi], bx" \
98 "mov [esi], ax" \
99 "setz al" \
100 "movzx eax, al" \
101 parm [ edi ] [ esi ] [ ebx ] \
102 value [ eax ]
103
104static int uacpi_do_atomic_cmpxchg32(volatile uint32_t *ptr, volatile uint32_t *expected, uint32_t desired);
105#pragma aux uacpi_do_atomic_cmpxchg32 = \
106 ".486" \
107 "mov eax, [esi]" \
108 "lock cmpxchg [edi], ebx" \
109 "mov [esi], eax" \
110 "setz al" \
111 "movzx eax, al" \
112 parm [ edi ] [ esi ] [ ebx ] \
113 value [ eax ]
114
115static int uacpi_do_atomic_cmpxchg64_asm(volatile uint64_t *ptr, volatile uint64_t *expected, uint32_t low, uint32_t high);
116#pragma aux uacpi_do_atomic_cmpxchg64_asm = \
117 ".586" \
118 "mov eax, [esi]" \
119 "mov edx, [esi + 4]" \
120 "lock cmpxchg8b [edi]" \
121 "mov [esi], eax" \
122 "mov [esi + 4], edx" \
123 "setz al" \
124 "movzx eax, al" \
125 modify [ edx ] \
126 parm [ edi ] [ esi ] [ ebx ] [ ecx ] \
127 value [ eax ]
128
129static inline int uacpi_do_atomic_cmpxchg64(volatile uint64_t *ptr, volatile uint64_t *expected, uint64_t desired) {
130 return uacpi_do_atomic_cmpxchg64_asm(ptr, expected, desired, desired >> 32);
131}
132
133#define uacpi_atomic_cmpxchg16(ptr, expected, desired) \
134 uacpi_do_atomic_cmpxchg16((volatile uint16_t*)ptr, (volatile uint16_t*)expected, (uint16_t)desired)
135#define uacpi_atomic_cmpxchg32(ptr, expected, desired) \
136 uacpi_do_atomic_cmpxchg32((volatile uint32_t*)ptr, (volatile uint32_t*)expected, (uint32_t)desired)
137#define uacpi_atomic_cmpxchg64(ptr, expected, desired) \
138 uacpi_do_atomic_cmpxchg64((volatile uint64_t*)ptr, (volatile uint64_t*)expected, (uint64_t)desired)
139
140static uint8_t uacpi_do_atomic_load8(volatile uint8_t *ptr);
141#pragma aux uacpi_do_atomic_load8 = \
142 "mov al, [esi]" \
143 parm [ esi ] \
144 value [ al ]
145
146static uint16_t uacpi_do_atomic_load16(volatile uint16_t *ptr);
147#pragma aux uacpi_do_atomic_load16 = \
148 "mov ax, [esi]" \
149 parm [ esi ] \
150 value [ ax ]
151
152static uint32_t uacpi_do_atomic_load32(volatile uint32_t *ptr);
153#pragma aux uacpi_do_atomic_load32 = \
154 "mov eax, [esi]" \
155 parm [ esi ] \
156 value [ eax ]
157
158static void uacpi_do_atomic_load64_asm(volatile uint64_t *ptr, uint64_t *out);
159#pragma aux uacpi_do_atomic_load64_asm = \
160 ".586" \
161 "xor eax, eax" \
162 "xor ebx, ebx" \
163 "xor ecx, ecx" \
164 "xor edx, edx" \
165 "lock cmpxchg8b [esi]" \
166 "mov [edi], eax" \
167 "mov [edi + 4], edx" \
168 modify [ eax ebx ecx edx ] \
169 parm [ esi ] [ edi ]
170
171static inline uint64_t uacpi_do_atomic_load64(volatile uint64_t *ptr) {
172 uint64_t value;
173 uacpi_do_atomic_load64_asm(ptr, &value);
174 return value;
175}
176
177#define uacpi_atomic_load8(ptr) uacpi_do_atomic_load8((volatile uint8_t*)ptr)
178#define uacpi_atomic_load16(ptr) uacpi_do_atomic_load16((volatile uint16_t*)ptr)
179#define uacpi_atomic_load32(ptr) uacpi_do_atomic_load32((volatile uint32_t*)ptr)
180#define uacpi_atomic_load64(ptr) uacpi_do_atomic_load64((volatile uint64_t*)ptr)
181
182static void uacpi_do_atomic_store8(volatile uint8_t *ptr, uint8_t value);
183#pragma aux uacpi_do_atomic_store8 = \
184 "mov [edi], al" \
185 parm [ edi ] [ eax ]
186
187static void uacpi_do_atomic_store16(volatile uint16_t *ptr, uint16_t value);
188#pragma aux uacpi_do_atomic_store16 = \
189 "mov [edi], ax" \
190 parm [ edi ] [ eax ]
191
192static void uacpi_do_atomic_store32(volatile uint32_t *ptr, uint32_t value);
193#pragma aux uacpi_do_atomic_store32 = \
194 "mov [edi], eax" \
195 parm [ edi ] [ eax ]
196
197static void uacpi_do_atomic_store64_asm(volatile uint64_t *ptr, uint32_t low, uint32_t high);
198#pragma aux uacpi_do_atomic_store64_asm = \
199 ".586" \
200 "xor eax, eax" \
201 "xor edx, edx" \
202 "retry: lock cmpxchg8b [edi]" \
203 "jnz retry" \
204 modify [ eax edx ] \
205 parm [ edi ] [ ebx ] [ ecx ]
206
207static inline void uacpi_do_atomic_store64(volatile uint64_t *ptr, uint64_t value) {
208 uacpi_do_atomic_store64_asm(ptr, value, value >> 32);
209}
210
211#define uacpi_atomic_store8(ptr, value) uacpi_do_atomic_store8((volatile uint8_t*)ptr, (uint8_t)value)
212#define uacpi_atomic_store16(ptr, value) uacpi_do_atomic_store16((volatile uint16_t*)ptr, (uint16_t)value)
213#define uacpi_atomic_store32(ptr, value) uacpi_do_atomic_store32((volatile uint32_t*)ptr, (uint32_t)value)
214#define uacpi_atomic_store64(ptr, value) uacpi_do_atomic_store64((volatile uint64_t*)ptr, (uint64_t)value)
215
216static uint16_t uacpi_do_atomic_inc16(volatile uint16_t *ptr);
217#pragma aux uacpi_do_atomic_inc16 = \
218 ".486" \
219 "mov ax, 1" \
220 "lock xadd [edi], ax" \
221 "add ax, 1" \
222 parm [ edi ] \
223 value [ ax ]
224
225static uint32_t uacpi_do_atomic_inc32(volatile uint32_t *ptr);
226#pragma aux uacpi_do_atomic_inc32 = \
227 ".486" \
228 "mov eax, 1" \
229 "lock xadd [edi], eax" \
230 "add eax, 1" \
231 parm [ edi ] \
232 value [ eax ]
233
234static void uacpi_do_atomic_inc64_asm(volatile uint64_t *ptr, uint64_t *out);
235#pragma aux uacpi_do_atomic_inc64_asm = \
236 ".586" \
237 "xor eax, eax" \
238 "xor edx, edx" \
239 "mov ebx, 1" \
240 "mov ecx, 1" \
241 "retry: lock cmpxchg8b [esi]" \
242 "mov ebx, eax" \
243 "mov ecx, edx" \
244 "add ebx, 1" \
245 "adc ecx, 0" \
246 "jnz retry" \
247 "mov [edi], ebx" \
248 "mov [edi + 4], ecx" \
249 modify [ eax ebx ecx edx ] \
250 parm [ esi ] [ edi ]
251
252static inline uint64_t uacpi_do_atomic_inc64(volatile uint64_t *ptr) {
253 uint64_t value;
254 uacpi_do_atomic_inc64_asm(ptr, &value);
255 return value;
256}
257
258#define uacpi_atomic_inc16(ptr) uacpi_do_atomic_inc16((volatile uint16_t*)ptr)
259#define uacpi_atomic_inc32(ptr) uacpi_do_atomic_inc32((volatile uint32_t*)ptr)
260#define uacpi_atomic_inc64(ptr) uacpi_do_atomic_inc64((volatile uint64_t*)ptr)
261
262static uint16_t uacpi_do_atomic_dec16(volatile uint16_t *ptr);
263#pragma aux uacpi_do_atomic_dec16 = \
264 ".486" \
265 "mov ax, -1" \
266 "lock xadd [edi], ax" \
267 "add ax, -1" \
268 parm [ edi ] \
269 value [ ax ]
270
271static uint32_t uacpi_do_atomic_dec32(volatile uint32_t *ptr);
272#pragma aux uacpi_do_atomic_dec32 = \
273 ".486" \
274 "mov eax, -1" \
275 "lock xadd [edi], eax" \
276 "add eax, -1" \
277 parm [ edi ] \
278 value [ eax ]
279
280static void uacpi_do_atomic_dec64_asm(volatile uint64_t *ptr, uint64_t *out);
281#pragma aux uacpi_do_atomic_dec64_asm = \
282 ".586" \
283 "xor eax, eax" \
284 "xor edx, edx" \
285 "mov ebx, -1" \
286 "mov ecx, -1" \
287 "retry: lock cmpxchg8b [esi]" \
288 "mov ebx, eax" \
289 "mov ecx, edx" \
290 "sub ebx, 1" \
291 "sbb ecx, 0" \
292 "jnz retry" \
293 "mov [edi], ebx" \
294 "mov [edi + 4], ecx" \
295 modify [ eax ebx ecx edx ] \
296 parm [ esi ] [ edi ]
297
298static inline uint64_t uacpi_do_atomic_dec64(volatile uint64_t *ptr) {
299 uint64_t value;
300 uacpi_do_atomic_dec64_asm(ptr, &value);
301 return value;
302}
303
304#define uacpi_atomic_dec16(ptr) uacpi_do_atomic_dec16((volatile uint16_t*)ptr)
305#define uacpi_atomic_dec32(ptr) uacpi_do_atomic_dec32((volatile uint32_t*)ptr)
306#define uacpi_atomic_dec64(ptr) uacpi_do_atomic_dec64((volatile uint64_t*)ptr)
307#else
308
309#define UACPI_DO_CMPXCHG(ptr, expected, desired) \
310 __atomic_compare_exchange_n(ptr, expected, desired, 0, \
311 __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)
312
313#define uacpi_atomic_cmpxchg16(ptr, expected, desired) \
314 UACPI_DO_CMPXCHG(ptr, expected, desired)
315#define uacpi_atomic_cmpxchg32(ptr, expected, desired) \
316 UACPI_DO_CMPXCHG(ptr, expected, desired)
317#define uacpi_atomic_cmpxchg64(ptr, expected, desired) \
318 UACPI_DO_CMPXCHG(ptr, expected, desired)
319
320#define uacpi_atomic_load8(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE)
321#define uacpi_atomic_load16(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE)
322#define uacpi_atomic_load32(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE)
323#define uacpi_atomic_load64(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE)
324
325#define uacpi_atomic_store8(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE)
326#define uacpi_atomic_store16(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE)
327#define uacpi_atomic_store32(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE)
328#define uacpi_atomic_store64(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE)
329
330#define uacpi_atomic_inc16(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_ACQ_REL)
331#define uacpi_atomic_inc32(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_ACQ_REL)
332#define uacpi_atomic_inc64(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_ACQ_REL)
333
334#define uacpi_atomic_dec16(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_ACQ_REL)
335#define uacpi_atomic_dec32(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_ACQ_REL)
336#define uacpi_atomic_dec64(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_ACQ_REL)
337#endif
338
339#if UACPI_POINTER_SIZE == 4
340#define uacpi_atomic_load_ptr(ptr_to_ptr) uacpi_atomic_load32(ptr_to_ptr)
341#define uacpi_atomic_store_ptr(ptr_to_ptr, value) uacpi_atomic_store32(ptr_to_ptr, value)
342#else
343#define uacpi_atomic_load_ptr(ptr_to_ptr) uacpi_atomic_load64(ptr_to_ptr)
344#define uacpi_atomic_store_ptr(ptr_to_ptr, value) uacpi_atomic_store64(ptr_to_ptr, value)
345#endif
346
347#endif
uint32_t
UINT32 uint32_t
Definition: types.h:75
uint64_t
UINT64 uint64_t
Definition: types.h:77
__int64
#define __int64
Definition: corecrt.h:72
uint16_t
unsigned short uint16_t
Definition: stdint.h:35
uint8_t
unsigned char uint8_t
Definition: stdint.h:33
compiler.h
desired
int desired
Definition: jpeglib.h:1121
ptr
static PVOID ptr
Definition: dispmode.c:27
expected
BOOL expected
Definition: store.c:2000
value
Definition: pdh_main.c:96
out
wchar_t tm const _CrtWcstime_Writes_and_advances_ptr_ count wchar_t ** out
Definition: wcsftime.cpp:383