rcutorture: make goflag volatile
[urcu.git] / urcu / uatomic_arch_x86.h
1 #ifndef _URCU_ARCH_UATOMIC_X86_H
2 #define _URCU_ARCH_UATOMIC_X86_H
3
4 /*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 *
10 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
11 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
12 *
13 * Permission is hereby granted to use or copy this program
14 * for any purpose, provided the above notices are retained on all copies.
15 * Permission to modify the code and to distribute modified code is granted,
16 * provided the above notices are retained, and a notice that the code was
17 * modified is included with the above copyright notice.
18 *
19 * Code inspired from libuatomic_ops-1.2, inherited in part from the
20 * Boehm-Demers-Weiser conservative garbage collector.
21 */
22
23 #include <urcu/compiler.h>
24 #include <urcu/system.h>
25
26 #define UATOMIC_HAS_ATOMIC_BYTE
27 #define UATOMIC_HAS_ATOMIC_SHORT
28
29 #ifdef __cplusplus
30 extern "C" {
31 #endif
32
33 /*
34 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
35 */
36
37 struct __uatomic_dummy {
38 unsigned long v[10];
39 };
40 #define __hp(x) ((struct __uatomic_dummy *)(x))
41
42 #define _uatomic_set(addr, v) CMM_STORE_SHARED(*(addr), (v))
43
44 /* cmpxchg */
45
46 static inline __attribute__((always_inline))
47 unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
48 unsigned long _new, int len)
49 {
50 switch (len) {
51 case 1:
52 {
53 unsigned char result = old;
54
55 __asm__ __volatile__(
56 "lock; cmpxchgb %2, %1"
57 : "+a"(result), "+m"(*__hp(addr))
58 : "q"((unsigned char)_new)
59 : "memory");
60 return result;
61 }
62 case 2:
63 {
64 unsigned short result = old;
65
66 __asm__ __volatile__(
67 "lock; cmpxchgw %2, %1"
68 : "+a"(result), "+m"(*__hp(addr))
69 : "r"((unsigned short)_new)
70 : "memory");
71 return result;
72 }
73 case 4:
74 {
75 unsigned int result = old;
76
77 __asm__ __volatile__(
78 "lock; cmpxchgl %2, %1"
79 : "+a"(result), "+m"(*__hp(addr))
80 : "r"((unsigned int)_new)
81 : "memory");
82 return result;
83 }
84 #if (CAA_BITS_PER_LONG == 64)
85 case 8:
86 {
87 unsigned long result = old;
88
89 __asm__ __volatile__(
90 "lock; cmpxchgq %2, %1"
91 : "+a"(result), "+m"(*__hp(addr))
92 : "r"((unsigned long)_new)
93 : "memory");
94 return result;
95 }
96 #endif
97 }
98 /* generate an illegal instruction. Cannot catch this with linker tricks
99 * when optimizations are disabled. */
100 __asm__ __volatile__("ud2");
101 return 0;
102 }
103
104 #define _uatomic_cmpxchg(addr, old, _new) \
105 ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), (unsigned long)(old),\
106 (unsigned long)(_new), \
107 sizeof(*(addr))))
108
109 /* xchg */
110
111 static inline __attribute__((always_inline))
112 unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
113 {
114 /* Note: the "xchg" instruction does not need a "lock" prefix. */
115 switch (len) {
116 case 1:
117 {
118 unsigned char result;
119 __asm__ __volatile__(
120 "xchgb %0, %1"
121 : "=q"(result), "+m"(*__hp(addr))
122 : "0" ((unsigned char)val)
123 : "memory");
124 return result;
125 }
126 case 2:
127 {
128 unsigned short result;
129 __asm__ __volatile__(
130 "xchgw %0, %1"
131 : "=r"(result), "+m"(*__hp(addr))
132 : "0" ((unsigned short)val)
133 : "memory");
134 return result;
135 }
136 case 4:
137 {
138 unsigned int result;
139 __asm__ __volatile__(
140 "xchgl %0, %1"
141 : "=r"(result), "+m"(*__hp(addr))
142 : "0" ((unsigned int)val)
143 : "memory");
144 return result;
145 }
146 #if (CAA_BITS_PER_LONG == 64)
147 case 8:
148 {
149 unsigned long result;
150 __asm__ __volatile__(
151 "xchgq %0, %1"
152 : "=r"(result), "+m"(*__hp(addr))
153 : "0" ((unsigned long)val)
154 : "memory");
155 return result;
156 }
157 #endif
158 }
159 /* generate an illegal instruction. Cannot catch this with linker tricks
160 * when optimizations are disabled. */
161 __asm__ __volatile__("ud2");
162 return 0;
163 }
164
165 #define _uatomic_xchg(addr, v) \
166 ((__typeof__(*(addr))) __uatomic_exchange((addr), (unsigned long)(v), \
167 sizeof(*(addr))))
168
169 /* uatomic_add_return */
170
171 static inline __attribute__((always_inline))
172 unsigned long __uatomic_add_return(void *addr, unsigned long val,
173 int len)
174 {
175 switch (len) {
176 case 1:
177 {
178 unsigned char result = val;
179
180 __asm__ __volatile__(
181 "lock; xaddb %1, %0"
182 : "+m"(*__hp(addr)), "+q" (result)
183 :
184 : "memory");
185 return result + (unsigned char)val;
186 }
187 case 2:
188 {
189 unsigned short result = val;
190
191 __asm__ __volatile__(
192 "lock; xaddw %1, %0"
193 : "+m"(*__hp(addr)), "+r" (result)
194 :
195 : "memory");
196 return result + (unsigned short)val;
197 }
198 case 4:
199 {
200 unsigned int result = val;
201
202 __asm__ __volatile__(
203 "lock; xaddl %1, %0"
204 : "+m"(*__hp(addr)), "+r" (result)
205 :
206 : "memory");
207 return result + (unsigned int)val;
208 }
209 #if (CAA_BITS_PER_LONG == 64)
210 case 8:
211 {
212 unsigned long result = val;
213
214 __asm__ __volatile__(
215 "lock; xaddq %1, %0"
216 : "+m"(*__hp(addr)), "+r" (result)
217 :
218 : "memory");
219 return result + (unsigned long)val;
220 }
221 #endif
222 }
223 /* generate an illegal instruction. Cannot catch this with linker tricks
224 * when optimizations are disabled. */
225 __asm__ __volatile__("ud2");
226 return 0;
227 }
228
229 #define _uatomic_add_return(addr, v) \
230 ((__typeof__(*(addr))) __uatomic_add_return((addr), \
231 (unsigned long)(v), \
232 sizeof(*(addr))))
233
234 /* uatomic_add */
235
236 static inline __attribute__((always_inline))
237 void __uatomic_add(void *addr, unsigned long val, int len)
238 {
239 switch (len) {
240 case 1:
241 {
242 __asm__ __volatile__(
243 "lock; addb %1, %0"
244 : "=m"(*__hp(addr))
245 : "iq" ((unsigned char)val)
246 : "memory");
247 return;
248 }
249 case 2:
250 {
251 __asm__ __volatile__(
252 "lock; addw %1, %0"
253 : "=m"(*__hp(addr))
254 : "ir" ((unsigned short)val)
255 : "memory");
256 return;
257 }
258 case 4:
259 {
260 __asm__ __volatile__(
261 "lock; addl %1, %0"
262 : "=m"(*__hp(addr))
263 : "ir" ((unsigned int)val)
264 : "memory");
265 return;
266 }
267 #if (CAA_BITS_PER_LONG == 64)
268 case 8:
269 {
270 __asm__ __volatile__(
271 "lock; addq %1, %0"
272 : "=m"(*__hp(addr))
273 : "er" ((unsigned long)val)
274 : "memory");
275 return;
276 }
277 #endif
278 }
279 /* generate an illegal instruction. Cannot catch this with linker tricks
280 * when optimizations are disabled. */
281 __asm__ __volatile__("ud2");
282 return;
283 }
284
285 #define _uatomic_add(addr, v) \
286 (__uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
287
288
289 /* uatomic_inc */
290
291 static inline __attribute__((always_inline))
292 void __uatomic_inc(void *addr, int len)
293 {
294 switch (len) {
295 case 1:
296 {
297 __asm__ __volatile__(
298 "lock; incb %0"
299 : "=m"(*__hp(addr))
300 :
301 : "memory");
302 return;
303 }
304 case 2:
305 {
306 __asm__ __volatile__(
307 "lock; incw %0"
308 : "=m"(*__hp(addr))
309 :
310 : "memory");
311 return;
312 }
313 case 4:
314 {
315 __asm__ __volatile__(
316 "lock; incl %0"
317 : "=m"(*__hp(addr))
318 :
319 : "memory");
320 return;
321 }
322 #if (CAA_BITS_PER_LONG == 64)
323 case 8:
324 {
325 __asm__ __volatile__(
326 "lock; incq %0"
327 : "=m"(*__hp(addr))
328 :
329 : "memory");
330 return;
331 }
332 #endif
333 }
334 /* generate an illegal instruction. Cannot catch this with linker tricks
335 * when optimizations are disabled. */
336 __asm__ __volatile__("ud2");
337 return;
338 }
339
340 #define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
341
342 /* uatomic_dec */
343
344 static inline __attribute__((always_inline))
345 void __uatomic_dec(void *addr, int len)
346 {
347 switch (len) {
348 case 1:
349 {
350 __asm__ __volatile__(
351 "lock; decb %0"
352 : "=m"(*__hp(addr))
353 :
354 : "memory");
355 return;
356 }
357 case 2:
358 {
359 __asm__ __volatile__(
360 "lock; decw %0"
361 : "=m"(*__hp(addr))
362 :
363 : "memory");
364 return;
365 }
366 case 4:
367 {
368 __asm__ __volatile__(
369 "lock; decl %0"
370 : "=m"(*__hp(addr))
371 :
372 : "memory");
373 return;
374 }
375 #if (CAA_BITS_PER_LONG == 64)
376 case 8:
377 {
378 __asm__ __volatile__(
379 "lock; decq %0"
380 : "=m"(*__hp(addr))
381 :
382 : "memory");
383 return;
384 }
385 #endif
386 }
387 /* generate an illegal instruction. Cannot catch this with linker tricks
388 * when optimizations are disabled. */
389 __asm__ __volatile__("ud2");
390 return;
391 }
392
393 #define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
394
395 #if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
396 extern int __rcu_cas_avail;
397 extern int __rcu_cas_init(void);
398
399 #define UATOMIC_COMPAT(insn) \
400 ((likely(__rcu_cas_avail > 0)) \
401 ? (_uatomic_##insn) \
402 : ((unlikely(__rcu_cas_avail < 0) \
403 ? ((__rcu_cas_init() > 0) \
404 ? (_uatomic_##insn) \
405 : (compat_uatomic_##insn)) \
406 : (compat_uatomic_##insn))))
407
408 extern unsigned long _compat_uatomic_set(void *addr,
409 unsigned long _new, int len);
410 #define compat_uatomic_set(addr, _new) \
411 ((__typeof__(*(addr))) _compat_uatomic_set((addr), \
412 (unsigned long)(_new), \
413 sizeof(*(addr))))
414
415
416 extern unsigned long _compat_uatomic_xchg(void *addr,
417 unsigned long _new, int len);
418 #define compat_uatomic_xchg(addr, _new) \
419 ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
420 (unsigned long)(_new), \
421 sizeof(*(addr))))
422
423 extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
424 unsigned long _new, int len);
425 #define compat_uatomic_cmpxchg(addr, old, _new) \
426 ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
427 (unsigned long)(old), \
428 (unsigned long)(_new), \
429 sizeof(*(addr))))
430
431 extern unsigned long _compat_uatomic_xchg(void *addr,
432 unsigned long _new, int len);
433 #define compat_uatomic_add_return(addr, v) \
434 ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
435 (unsigned long)(v), \
436 sizeof(*(addr))))
437
438 #define compat_uatomic_add(addr, v) \
439 ((void)compat_uatomic_add_return((addr), (v)))
440 #define compat_uatomic_inc(addr) \
441 (compat_uatomic_add((addr), 1))
442 #define compat_uatomic_dec(addr) \
443 (compat_uatomic_add((addr), -1))
444
445 #else
446 #define UATOMIC_COMPAT(insn) (_uatomic_##insn)
447 #endif
448
449 /* Read is atomic even in compat mode */
450 #define uatomic_set(addr, v) \
451 UATOMIC_COMPAT(set(addr, v))
452
453 #define uatomic_cmpxchg(addr, old, _new) \
454 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
455 #define uatomic_xchg(addr, v) \
456 UATOMIC_COMPAT(xchg(addr, v))
457 #define uatomic_add_return(addr, v) \
458 UATOMIC_COMPAT(add_return(addr, v))
459
460 #define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
461 #define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
462 #define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
463
464 #ifdef __cplusplus
465 }
466 #endif
467
468 #include <urcu/uatomic_generic.h>
469
470 #endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.038625 seconds and 5 git commands to generate.