Fix: x86 and s390: uatomic __hp() macro clang support
[urcu.git] / include / urcu / uatomic / x86.h
CommitLineData
ec4e58a3
MD
1#ifndef _URCU_ARCH_UATOMIC_X86_H
2#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f 3
67ecffc0 4/*
0114ba7f
MD
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 *
10 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
11 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
12 *
13 * Permission is hereby granted to use or copy this program
14 * for any purpose, provided the above notices are retained on all copies.
15 * Permission to modify the code and to distribute modified code is granted,
16 * provided the above notices are retained, and a notice that the code was
17 * modified is included with the above copyright notice.
18 *
ec4e58a3 19 * Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f
MD
20 * Boehm-Demers-Weiser conservative garbage collector.
21 */
22
0b1e236d 23#include <urcu/arch.h>
375db287 24#include <urcu/config.h>
ec4e58a3 25#include <urcu/compiler.h>
bf9de1b7 26#include <urcu/system.h>
0fad128b 27
f469d839
PB
28#define UATOMIC_HAS_ATOMIC_BYTE
29#define UATOMIC_HAS_ATOMIC_SHORT
30
36bc70a8
MD
31#ifdef __cplusplus
32extern "C" {
67ecffc0 33#endif
36bc70a8 34
0114ba7f 35/*
0114ba7f
MD
36 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
37 */
38
835b9ab3 39/*
71323499 40 * The __hp() macro casts the void pointer @x to a pointer to a structure
835b9ab3
MD
41 * containing an array of char of the specified size. This allows passing the
42 * @addr arguments of the following inline functions as "m" and "+m" operands
71323499
MD
43 * to the assembly. The @size parameter should be a constant to support
44 * compilers such as clang which do not support VLA.
835b9ab3
MD
45 */
46
47#define __hp(size, x) ((struct { char v[size]; } *)(x))
cc1be41b 48
424d4ed5 49#define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
0fad128b 50
cc1be41b
MD
51/* cmpxchg */
52
5dba80f9 53static inline __attribute__((always_inline))
bf9de1b7 54unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b 55 unsigned long _new, int len)
0114ba7f 56{
cc1be41b
MD
57 switch (len) {
58 case 1:
59 {
60 unsigned char result = old;
0fad128b 61
cc1be41b
MD
62 __asm__ __volatile__(
63 "lock; cmpxchgb %2, %1"
71323499 64 : "+a"(result), "+m"(*__hp(1, addr))
cc1be41b 65 : "q"((unsigned char)_new)
0114ba7f 66 : "memory");
cc1be41b
MD
67 return result;
68 }
69 case 2:
70 {
71 unsigned short result = old;
0fad128b 72
cc1be41b
MD
73 __asm__ __volatile__(
74 "lock; cmpxchgw %2, %1"
71323499 75 : "+a"(result), "+m"(*__hp(2, addr))
cc1be41b
MD
76 : "r"((unsigned short)_new)
77 : "memory");
78 return result;
79 }
80 case 4:
81 {
82 unsigned int result = old;
0fad128b 83
cc1be41b
MD
84 __asm__ __volatile__(
85 "lock; cmpxchgl %2, %1"
71323499 86 : "+a"(result), "+m"(*__hp(4, addr))
cc1be41b
MD
87 : "r"((unsigned int)_new)
88 : "memory");
89 return result;
90 }
e040d717 91#if (CAA_BITS_PER_LONG == 64)
cc1be41b
MD
92 case 8:
93 {
6edb297e 94 unsigned long result = old;
0fad128b 95
cc1be41b 96 __asm__ __volatile__(
2c5e5fb3 97 "lock; cmpxchgq %2, %1"
71323499 98 : "+a"(result), "+m"(*__hp(8, addr))
cc1be41b
MD
99 : "r"((unsigned long)_new)
100 : "memory");
101 return result;
102 }
103#endif
104 }
d0bbd9c2
MD
105 /*
106 * generate an illegal instruction. Cannot catch this with
107 * linker tricks when optimizations are disabled.
108 */
cc1be41b
MD
109 __asm__ __volatile__("ud2");
110 return 0;
0114ba7f
MD
111}
112
bf9de1b7 113#define _uatomic_cmpxchg(addr, old, _new) \
e56d99bf
MD
114 ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
115 caa_cast_long_keep_sign(old), \
116 caa_cast_long_keep_sign(_new),\
cc1be41b
MD
117 sizeof(*(addr))))
118
119/* xchg */
0114ba7f 120
5dba80f9 121static inline __attribute__((always_inline))
bf9de1b7 122unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f 123{
cc1be41b
MD
124 /* Note: the "xchg" instruction does not need a "lock" prefix. */
125 switch (len) {
126 case 1:
127 {
128 unsigned char result;
129 __asm__ __volatile__(
130 "xchgb %0, %1"
71323499 131 : "=q"(result), "+m"(*__hp(1, addr))
cc1be41b
MD
132 : "0" ((unsigned char)val)
133 : "memory");
134 return result;
135 }
136 case 2:
137 {
138 unsigned short result;
139 __asm__ __volatile__(
140 "xchgw %0, %1"
71323499 141 : "=r"(result), "+m"(*__hp(2, addr))
cc1be41b
MD
142 : "0" ((unsigned short)val)
143 : "memory");
144 return result;
145 }
146 case 4:
147 {
148 unsigned int result;
149 __asm__ __volatile__(
150 "xchgl %0, %1"
71323499 151 : "=r"(result), "+m"(*__hp(4, addr))
cc1be41b
MD
152 : "0" ((unsigned int)val)
153 : "memory");
154 return result;
155 }
e040d717 156#if (CAA_BITS_PER_LONG == 64)
cc1be41b
MD
157 case 8:
158 {
159 unsigned long result;
160 __asm__ __volatile__(
0114ba7f 161 "xchgq %0, %1"
71323499 162 : "=r"(result), "+m"(*__hp(8, addr))
cc1be41b 163 : "0" ((unsigned long)val)
0114ba7f 164 : "memory");
cc1be41b
MD
165 return result;
166 }
167#endif
168 }
d0bbd9c2
MD
169 /*
170 * generate an illegal instruction. Cannot catch this with
171 * linker tricks when optimizations are disabled.
172 */
cc1be41b
MD
173 __asm__ __volatile__("ud2");
174 return 0;
0114ba7f
MD
175}
176
bf9de1b7 177#define _uatomic_xchg(addr, v) \
e56d99bf
MD
178 ((__typeof__(*(addr))) __uatomic_exchange((addr), \
179 caa_cast_long_keep_sign(v), \
cc1be41b
MD
180 sizeof(*(addr))))
181
8760d94e 182/* uatomic_add_return */
0fad128b
MD
183
184static inline __attribute__((always_inline))
bf9de1b7 185unsigned long __uatomic_add_return(void *addr, unsigned long val,
0fad128b
MD
186 int len)
187{
188 switch (len) {
189 case 1:
190 {
191 unsigned char result = val;
192
193 __asm__ __volatile__(
194 "lock; xaddb %1, %0"
71323499 195 : "+m"(*__hp(1, addr)), "+q" (result)
0fad128b
MD
196 :
197 : "memory");
198 return result + (unsigned char)val;
199 }
200 case 2:
201 {
202 unsigned short result = val;
203
204 __asm__ __volatile__(
205 "lock; xaddw %1, %0"
71323499 206 : "+m"(*__hp(2, addr)), "+r" (result)
0fad128b
MD
207 :
208 : "memory");
209 return result + (unsigned short)val;
210 }
211 case 4:
212 {
213 unsigned int result = val;
214
215 __asm__ __volatile__(
216 "lock; xaddl %1, %0"
71323499 217 : "+m"(*__hp(4, addr)), "+r" (result)
0fad128b
MD
218 :
219 : "memory");
220 return result + (unsigned int)val;
221 }
e040d717 222#if (CAA_BITS_PER_LONG == 64)
0fad128b
MD
223 case 8:
224 {
225 unsigned long result = val;
226
227 __asm__ __volatile__(
228 "lock; xaddq %1, %0"
71323499 229 : "+m"(*__hp(8, addr)), "+r" (result)
0fad128b
MD
230 :
231 : "memory");
232 return result + (unsigned long)val;
233 }
234#endif
235 }
d0bbd9c2
MD
236 /*
237 * generate an illegal instruction. Cannot catch this with
238 * linker tricks when optimizations are disabled.
239 */
0fad128b
MD
240 __asm__ __volatile__("ud2");
241 return 0;
242}
243
e56d99bf
MD
244#define _uatomic_add_return(addr, v) \
245 ((__typeof__(*(addr))) __uatomic_add_return((addr), \
246 caa_cast_long_keep_sign(v), \
247 sizeof(*(addr))))
0fad128b 248
bf33aaea
PB
249/* uatomic_and */
250
251static inline __attribute__((always_inline))
252void __uatomic_and(void *addr, unsigned long val, int len)
253{
254 switch (len) {
255 case 1:
256 {
257 __asm__ __volatile__(
258 "lock; andb %1, %0"
71323499 259 : "=m"(*__hp(1, addr))
bf33aaea
PB
260 : "iq" ((unsigned char)val)
261 : "memory");
262 return;
263 }
264 case 2:
265 {
266 __asm__ __volatile__(
267 "lock; andw %1, %0"
71323499 268 : "=m"(*__hp(2, addr))
bf33aaea
PB
269 : "ir" ((unsigned short)val)
270 : "memory");
271 return;
272 }
273 case 4:
274 {
275 __asm__ __volatile__(
276 "lock; andl %1, %0"
71323499 277 : "=m"(*__hp(4, addr))
bf33aaea
PB
278 : "ir" ((unsigned int)val)
279 : "memory");
280 return;
281 }
282#if (CAA_BITS_PER_LONG == 64)
283 case 8:
284 {
285 __asm__ __volatile__(
286 "lock; andq %1, %0"
71323499 287 : "=m"(*__hp(8, addr))
bf33aaea
PB
288 : "er" ((unsigned long)val)
289 : "memory");
290 return;
291 }
292#endif
293 }
d0bbd9c2
MD
294 /*
295 * generate an illegal instruction. Cannot catch this with
296 * linker tricks when optimizations are disabled.
297 */
bf33aaea
PB
298 __asm__ __volatile__("ud2");
299 return;
300}
301
302#define _uatomic_and(addr, v) \
e56d99bf 303 (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
bf33aaea 304
985b35b1
PB
305/* uatomic_or */
306
307static inline __attribute__((always_inline))
308void __uatomic_or(void *addr, unsigned long val, int len)
309{
310 switch (len) {
311 case 1:
312 {
313 __asm__ __volatile__(
314 "lock; orb %1, %0"
71323499 315 : "=m"(*__hp(1, addr))
985b35b1
PB
316 : "iq" ((unsigned char)val)
317 : "memory");
318 return;
319 }
320 case 2:
321 {
322 __asm__ __volatile__(
323 "lock; orw %1, %0"
71323499 324 : "=m"(*__hp(2, addr))
985b35b1
PB
325 : "ir" ((unsigned short)val)
326 : "memory");
327 return;
328 }
329 case 4:
330 {
331 __asm__ __volatile__(
332 "lock; orl %1, %0"
71323499 333 : "=m"(*__hp(4, addr))
985b35b1
PB
334 : "ir" ((unsigned int)val)
335 : "memory");
336 return;
337 }
338#if (CAA_BITS_PER_LONG == 64)
339 case 8:
340 {
341 __asm__ __volatile__(
342 "lock; orq %1, %0"
71323499 343 : "=m"(*__hp(8, addr))
985b35b1
PB
344 : "er" ((unsigned long)val)
345 : "memory");
346 return;
347 }
348#endif
349 }
d0bbd9c2
MD
350 /*
351 * generate an illegal instruction. Cannot catch this with
352 * linker tricks when optimizations are disabled.
353 */
985b35b1
PB
354 __asm__ __volatile__("ud2");
355 return;
356}
357
358#define _uatomic_or(addr, v) \
e56d99bf 359 (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
985b35b1 360
8760d94e 361/* uatomic_add */
0114ba7f 362
5dba80f9 363static inline __attribute__((always_inline))
bf9de1b7 364void __uatomic_add(void *addr, unsigned long val, int len)
0114ba7f
MD
365{
366 switch (len) {
cc1be41b
MD
367 case 1:
368 {
369 __asm__ __volatile__(
370 "lock; addb %1, %0"
71323499 371 : "=m"(*__hp(1, addr))
87322fe8
MD
372 : "iq" ((unsigned char)val)
373 : "memory");
cc1be41b
MD
374 return;
375 }
376 case 2:
377 {
378 __asm__ __volatile__(
379 "lock; addw %1, %0"
71323499 380 : "=m"(*__hp(2, addr))
87322fe8
MD
381 : "ir" ((unsigned short)val)
382 : "memory");
cc1be41b
MD
383 return;
384 }
385 case 4:
386 {
387 __asm__ __volatile__(
388 "lock; addl %1, %0"
71323499 389 : "=m"(*__hp(4, addr))
87322fe8
MD
390 : "ir" ((unsigned int)val)
391 : "memory");
cc1be41b
MD
392 return;
393 }
e040d717 394#if (CAA_BITS_PER_LONG == 64)
cc1be41b
MD
395 case 8:
396 {
397 __asm__ __volatile__(
398 "lock; addq %1, %0"
71323499 399 : "=m"(*__hp(8, addr))
87322fe8
MD
400 : "er" ((unsigned long)val)
401 : "memory");
cc1be41b
MD
402 return;
403 }
0114ba7f
MD
404#endif
405 }
d0bbd9c2
MD
406 /*
407 * generate an illegal instruction. Cannot catch this with
408 * linker tricks when optimizations are disabled.
409 */
0114ba7f 410 __asm__ __volatile__("ud2");
a81b8e5e 411 return;
0114ba7f
MD
412}
413
bf9de1b7 414#define _uatomic_add(addr, v) \
e56d99bf 415 (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
0114ba7f 416
2c5e5fb3 417
ec4e58a3 418/* uatomic_inc */
2c5e5fb3
MD
419
420static inline __attribute__((always_inline))
bf9de1b7 421void __uatomic_inc(void *addr, int len)
2c5e5fb3
MD
422{
423 switch (len) {
424 case 1:
425 {
426 __asm__ __volatile__(
427 "lock; incb %0"
71323499 428 : "=m"(*__hp(1, addr))
2c5e5fb3
MD
429 :
430 : "memory");
431 return;
432 }
433 case 2:
434 {
435 __asm__ __volatile__(
436 "lock; incw %0"
71323499 437 : "=m"(*__hp(2, addr))
2c5e5fb3
MD
438 :
439 : "memory");
440 return;
441 }
442 case 4:
443 {
444 __asm__ __volatile__(
445 "lock; incl %0"
71323499 446 : "=m"(*__hp(4, addr))
2c5e5fb3
MD
447 :
448 : "memory");
449 return;
450 }
e040d717 451#if (CAA_BITS_PER_LONG == 64)
2c5e5fb3
MD
452 case 8:
453 {
454 __asm__ __volatile__(
455 "lock; incq %0"
71323499 456 : "=m"(*__hp(8, addr))
2c5e5fb3
MD
457 :
458 : "memory");
459 return;
460 }
461#endif
462 }
463 /* generate an illegal instruction. Cannot catch this with linker tricks
464 * when optimizations are disabled. */
465 __asm__ __volatile__("ud2");
466 return;
467}
468
bf9de1b7 469#define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3 470
ec4e58a3 471/* uatomic_dec */
2c5e5fb3
MD
472
473static inline __attribute__((always_inline))
bf9de1b7 474void __uatomic_dec(void *addr, int len)
2c5e5fb3
MD
475{
476 switch (len) {
477 case 1:
478 {
479 __asm__ __volatile__(
480 "lock; decb %0"
71323499 481 : "=m"(*__hp(1, addr))
2c5e5fb3
MD
482 :
483 : "memory");
484 return;
485 }
486 case 2:
487 {
488 __asm__ __volatile__(
489 "lock; decw %0"
71323499 490 : "=m"(*__hp(2, addr))
2c5e5fb3
MD
491 :
492 : "memory");
493 return;
494 }
495 case 4:
496 {
497 __asm__ __volatile__(
498 "lock; decl %0"
71323499 499 : "=m"(*__hp(4, addr))
2c5e5fb3
MD
500 :
501 : "memory");
502 return;
503 }
e040d717 504#if (CAA_BITS_PER_LONG == 64)
2c5e5fb3
MD
505 case 8:
506 {
507 __asm__ __volatile__(
508 "lock; decq %0"
71323499 509 : "=m"(*__hp(8, addr))
2c5e5fb3
MD
510 :
511 : "memory");
512 return;
513 }
514#endif
515 }
d0bbd9c2
MD
516 /*
517 * generate an illegal instruction. Cannot catch this with
518 * linker tricks when optimizations are disabled.
519 */
2c5e5fb3
MD
520 __asm__ __volatile__("ud2");
521 return;
522}
523
bf9de1b7 524#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
0114ba7f 525
0b1e236d
MJ
526#if ((CAA_BITS_PER_LONG != 64) && defined(URCU_ARCH_I386))
527
528/* For backwards compat */
529#define CONFIG_RCU_COMPAT_ARCH 1
530
02be5561
MD
531extern int __rcu_cas_avail;
532extern int __rcu_cas_init(void);
bf9de1b7
MD
533
534#define UATOMIC_COMPAT(insn) \
a0b7f7ea 535 ((caa_likely(__rcu_cas_avail > 0)) \
bf9de1b7 536 ? (_uatomic_##insn) \
a0b7f7ea 537 : ((caa_unlikely(__rcu_cas_avail < 0) \
02be5561 538 ? ((__rcu_cas_init() > 0) \
bf9de1b7
MD
539 ? (_uatomic_##insn) \
540 : (compat_uatomic_##insn)) \
541 : (compat_uatomic_##insn))))
542
424d4ed5
MD
543/*
544 * We leave the return value so we don't break the ABI, but remove the
545 * return value from the API.
546 */
bf9de1b7
MD
547extern unsigned long _compat_uatomic_set(void *addr,
548 unsigned long _new, int len);
549#define compat_uatomic_set(addr, _new) \
424d4ed5
MD
550 ((void) _compat_uatomic_set((addr), \
551 caa_cast_long_keep_sign(_new), \
552 sizeof(*(addr))))
bf9de1b7
MD
553
554
555extern unsigned long _compat_uatomic_xchg(void *addr,
556 unsigned long _new, int len);
557#define compat_uatomic_xchg(addr, _new) \
558 ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
e56d99bf 559 caa_cast_long_keep_sign(_new), \
bf9de1b7 560 sizeof(*(addr))))
7d413817
MD
561
562extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
bf9de1b7
MD
563 unsigned long _new, int len);
564#define compat_uatomic_cmpxchg(addr, old, _new) \
565 ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
e56d99bf
MD
566 caa_cast_long_keep_sign(old), \
567 caa_cast_long_keep_sign(_new), \
bf9de1b7 568 sizeof(*(addr))))
7d413817 569
8c43fe72 570extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
bf33aaea 571#define compat_uatomic_and(addr, v) \
8c43fe72 572 (_compat_uatomic_and((addr), \
e56d99bf 573 caa_cast_long_keep_sign(v), \
8c43fe72 574 sizeof(*(addr))))
bf33aaea 575
8c43fe72 576extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
985b35b1 577#define compat_uatomic_or(addr, v) \
8c43fe72 578 (_compat_uatomic_or((addr), \
e56d99bf 579 caa_cast_long_keep_sign(v), \
8c43fe72 580 sizeof(*(addr))))
985b35b1 581
28ca843d
PB
582extern unsigned long _compat_uatomic_add_return(void *addr,
583 unsigned long _new, int len);
e56d99bf
MD
584#define compat_uatomic_add_return(addr, v) \
585 ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
586 caa_cast_long_keep_sign(v), \
587 sizeof(*(addr))))
bf9de1b7 588
bf9de1b7
MD
589#define compat_uatomic_add(addr, v) \
590 ((void)compat_uatomic_add_return((addr), (v)))
bf9de1b7
MD
591#define compat_uatomic_inc(addr) \
592 (compat_uatomic_add((addr), 1))
593#define compat_uatomic_dec(addr) \
8760d94e 594 (compat_uatomic_add((addr), -1))
bf9de1b7
MD
595
596#else
597#define UATOMIC_COMPAT(insn) (_uatomic_##insn)
7d413817
MD
598#endif
599
bf9de1b7 600/* Read is atomic even in compat mode */
bf9de1b7
MD
601#define uatomic_set(addr, v) \
602 UATOMIC_COMPAT(set(addr, v))
8760d94e 603
bf9de1b7
MD
604#define uatomic_cmpxchg(addr, old, _new) \
605 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
606#define uatomic_xchg(addr, v) \
607 UATOMIC_COMPAT(xchg(addr, v))
2812a2d2 608
bf33aaea
PB
609#define uatomic_and(addr, v) \
610 UATOMIC_COMPAT(and(addr, v))
42e83919
MD
611#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
612#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
2812a2d2 613
985b35b1
PB
614#define uatomic_or(addr, v) \
615 UATOMIC_COMPAT(or(addr, v))
42e83919
MD
616#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
617#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
2812a2d2 618
bf9de1b7
MD
619#define uatomic_add_return(addr, v) \
620 UATOMIC_COMPAT(add_return(addr, v))
8760d94e 621
bf9de1b7 622#define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
42e83919
MD
623#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
624#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
2812a2d2 625
bf9de1b7 626#define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
42e83919
MD
627#define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
628#define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
2812a2d2 629
bf9de1b7 630#define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
42e83919
MD
631#define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
632#define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
bf9de1b7 633
67ecffc0 634#ifdef __cplusplus
36bc70a8
MD
635}
636#endif
637
a2e7bf9c 638#include <urcu/uatomic/generic.h>
8760d94e 639
ec4e58a3 640#endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.120522 seconds and 4 git commands to generate.