1 // SPDX-FileCopyrightText: 1991-1994 by Xerox Corporation. All rights reserved.
2 // SPDX-FileCopyrightText: 1996-1999 by Silicon Graphics. All rights reserved.
3 // SPDX-FileCopyrightText: 1999-2004 Hewlett-Packard Development Company, L.P.
4 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 // SPDX-License-Identifier: LicenseRef-Boehm-GC
8 #ifndef _URCU_ARCH_UATOMIC_X86_H
9 #define _URCU_ARCH_UATOMIC_X86_H
12 * Code inspired from libuatomic_ops-1.2, inherited in part from the
13 * Boehm-Demers-Weiser conservative garbage collector.
16 #include <urcu/arch.h>
17 #include <urcu/config.h>
18 #include <urcu/compiler.h>
19 #include <urcu/system.h>
21 #define UATOMIC_HAS_ATOMIC_BYTE
22 #define UATOMIC_HAS_ATOMIC_SHORT
29 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
33 * The __hp() macro casts the void pointer @x to a pointer to a structure
34 * containing an array of char of the specified size. This allows passing the
35 * @addr arguments of the following inline functions as "m" and "+m" operands
36 * to the assembly. The @size parameter should be a constant to support
37 * compilers such as clang which do not support VLA. Create typedefs because
38 * C++ does not allow types be defined in casts.
41 typedef struct { char v
[1]; } __hp_1
;
42 typedef struct { char v
[2]; } __hp_2
;
43 typedef struct { char v
[4]; } __hp_4
;
44 typedef struct { char v
[8]; } __hp_8
;
46 #define __hp(size, x) ((__hp_##size *)(x))
48 #define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
52 static inline __attribute__((always_inline
))
53 unsigned long __uatomic_cmpxchg(void *addr
, unsigned long old
,
54 unsigned long _new
, int len
)
59 unsigned char result
= old
;
62 "lock; cmpxchgb %2, %1"
63 : "+a"(result
), "+m"(*__hp(1, addr
))
64 : "q"((unsigned char)_new
)
70 unsigned short result
= old
;
73 "lock; cmpxchgw %2, %1"
74 : "+a"(result
), "+m"(*__hp(2, addr
))
75 : "r"((unsigned short)_new
)
81 unsigned int result
= old
;
84 "lock; cmpxchgl %2, %1"
85 : "+a"(result
), "+m"(*__hp(4, addr
))
86 : "r"((unsigned int)_new
)
90 #if (CAA_BITS_PER_LONG == 64)
93 unsigned long result
= old
;
96 "lock; cmpxchgq %2, %1"
97 : "+a"(result
), "+m"(*__hp(8, addr
))
98 : "r"((unsigned long)_new
)
105 * generate an illegal instruction. Cannot catch this with
106 * linker tricks when optimizations are disabled.
108 __asm__
__volatile__("ud2");
112 #define _uatomic_cmpxchg(addr, old, _new) \
113 ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
114 caa_cast_long_keep_sign(old), \
115 caa_cast_long_keep_sign(_new),\
120 static inline __attribute__((always_inline
))
121 unsigned long __uatomic_exchange(void *addr
, unsigned long val
, int len
)
123 /* Note: the "xchg" instruction does not need a "lock" prefix. */
127 unsigned char result
;
128 __asm__
__volatile__(
130 : "=q"(result
), "+m"(*__hp(1, addr
))
131 : "0" ((unsigned char)val
)
137 unsigned short result
;
138 __asm__
__volatile__(
140 : "=r"(result
), "+m"(*__hp(2, addr
))
141 : "0" ((unsigned short)val
)
148 __asm__
__volatile__(
150 : "=r"(result
), "+m"(*__hp(4, addr
))
151 : "0" ((unsigned int)val
)
155 #if (CAA_BITS_PER_LONG == 64)
158 unsigned long result
;
159 __asm__
__volatile__(
161 : "=r"(result
), "+m"(*__hp(8, addr
))
162 : "0" ((unsigned long)val
)
169 * generate an illegal instruction. Cannot catch this with
170 * linker tricks when optimizations are disabled.
172 __asm__
__volatile__("ud2");
176 #define _uatomic_xchg(addr, v) \
177 ((__typeof__(*(addr))) __uatomic_exchange((addr), \
178 caa_cast_long_keep_sign(v), \
181 /* uatomic_add_return */
183 static inline __attribute__((always_inline
))
184 unsigned long __uatomic_add_return(void *addr
, unsigned long val
,
190 unsigned char result
= val
;
192 __asm__
__volatile__(
194 : "+m"(*__hp(1, addr
)), "+q" (result
)
197 return result
+ (unsigned char)val
;
201 unsigned short result
= val
;
203 __asm__
__volatile__(
205 : "+m"(*__hp(2, addr
)), "+r" (result
)
208 return result
+ (unsigned short)val
;
212 unsigned int result
= val
;
214 __asm__
__volatile__(
216 : "+m"(*__hp(4, addr
)), "+r" (result
)
219 return result
+ (unsigned int)val
;
221 #if (CAA_BITS_PER_LONG == 64)
224 unsigned long result
= val
;
226 __asm__
__volatile__(
228 : "+m"(*__hp(8, addr
)), "+r" (result
)
231 return result
+ (unsigned long)val
;
236 * generate an illegal instruction. Cannot catch this with
237 * linker tricks when optimizations are disabled.
239 __asm__
__volatile__("ud2");
243 #define _uatomic_add_return(addr, v) \
244 ((__typeof__(*(addr))) __uatomic_add_return((addr), \
245 caa_cast_long_keep_sign(v), \
250 static inline __attribute__((always_inline
))
251 void __uatomic_and(void *addr
, unsigned long val
, int len
)
256 __asm__
__volatile__(
258 : "=m"(*__hp(1, addr
))
259 : "iq" ((unsigned char)val
)
265 __asm__
__volatile__(
267 : "=m"(*__hp(2, addr
))
268 : "ir" ((unsigned short)val
)
274 __asm__
__volatile__(
276 : "=m"(*__hp(4, addr
))
277 : "ir" ((unsigned int)val
)
281 #if (CAA_BITS_PER_LONG == 64)
284 __asm__
__volatile__(
286 : "=m"(*__hp(8, addr
))
287 : "er" ((unsigned long)val
)
294 * generate an illegal instruction. Cannot catch this with
295 * linker tricks when optimizations are disabled.
297 __asm__
__volatile__("ud2");
301 #define _uatomic_and(addr, v) \
302 (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
306 static inline __attribute__((always_inline
))
307 void __uatomic_or(void *addr
, unsigned long val
, int len
)
312 __asm__
__volatile__(
314 : "=m"(*__hp(1, addr
))
315 : "iq" ((unsigned char)val
)
321 __asm__
__volatile__(
323 : "=m"(*__hp(2, addr
))
324 : "ir" ((unsigned short)val
)
330 __asm__
__volatile__(
332 : "=m"(*__hp(4, addr
))
333 : "ir" ((unsigned int)val
)
337 #if (CAA_BITS_PER_LONG == 64)
340 __asm__
__volatile__(
342 : "=m"(*__hp(8, addr
))
343 : "er" ((unsigned long)val
)
350 * generate an illegal instruction. Cannot catch this with
351 * linker tricks when optimizations are disabled.
353 __asm__
__volatile__("ud2");
357 #define _uatomic_or(addr, v) \
358 (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
362 static inline __attribute__((always_inline
))
363 void __uatomic_add(void *addr
, unsigned long val
, int len
)
368 __asm__
__volatile__(
370 : "=m"(*__hp(1, addr
))
371 : "iq" ((unsigned char)val
)
377 __asm__
__volatile__(
379 : "=m"(*__hp(2, addr
))
380 : "ir" ((unsigned short)val
)
386 __asm__
__volatile__(
388 : "=m"(*__hp(4, addr
))
389 : "ir" ((unsigned int)val
)
393 #if (CAA_BITS_PER_LONG == 64)
396 __asm__
__volatile__(
398 : "=m"(*__hp(8, addr
))
399 : "er" ((unsigned long)val
)
406 * generate an illegal instruction. Cannot catch this with
407 * linker tricks when optimizations are disabled.
409 __asm__
__volatile__("ud2");
413 #define _uatomic_add(addr, v) \
414 (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
419 static inline __attribute__((always_inline
))
420 void __uatomic_inc(void *addr
, int len
)
425 __asm__
__volatile__(
427 : "=m"(*__hp(1, addr
))
434 __asm__
__volatile__(
436 : "=m"(*__hp(2, addr
))
443 __asm__
__volatile__(
445 : "=m"(*__hp(4, addr
))
450 #if (CAA_BITS_PER_LONG == 64)
453 __asm__
__volatile__(
455 : "=m"(*__hp(8, addr
))
462 /* generate an illegal instruction. Cannot catch this with linker tricks
463 * when optimizations are disabled. */
464 __asm__
__volatile__("ud2");
468 #define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
472 static inline __attribute__((always_inline
))
473 void __uatomic_dec(void *addr
, int len
)
478 __asm__
__volatile__(
480 : "=m"(*__hp(1, addr
))
487 __asm__
__volatile__(
489 : "=m"(*__hp(2, addr
))
496 __asm__
__volatile__(
498 : "=m"(*__hp(4, addr
))
503 #if (CAA_BITS_PER_LONG == 64)
506 __asm__
__volatile__(
508 : "=m"(*__hp(8, addr
))
516 * generate an illegal instruction. Cannot catch this with
517 * linker tricks when optimizations are disabled.
519 __asm__
__volatile__("ud2");
523 #define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
525 #ifdef URCU_ARCH_X86_NO_CAS
527 /* For backwards compat */
528 #define CONFIG_RCU_COMPAT_ARCH 1
530 extern int __rcu_cas_avail
;
531 extern int __rcu_cas_init(void);
533 #define UATOMIC_COMPAT(insn) \
534 ((caa_likely(__rcu_cas_avail > 0)) \
535 ? (_uatomic_##insn) \
536 : ((caa_unlikely(__rcu_cas_avail < 0) \
537 ? ((__rcu_cas_init() > 0) \
538 ? (_uatomic_##insn) \
539 : (compat_uatomic_##insn)) \
540 : (compat_uatomic_##insn))))
543 * We leave the return value so we don't break the ABI, but remove the
544 * return value from the API.
546 extern unsigned long _compat_uatomic_set(void *addr
,
547 unsigned long _new
, int len
);
548 #define compat_uatomic_set(addr, _new) \
549 ((void) _compat_uatomic_set((addr), \
550 caa_cast_long_keep_sign(_new), \
554 extern unsigned long _compat_uatomic_xchg(void *addr
,
555 unsigned long _new
, int len
);
556 #define compat_uatomic_xchg(addr, _new) \
557 ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
558 caa_cast_long_keep_sign(_new), \
561 extern unsigned long _compat_uatomic_cmpxchg(void *addr
, unsigned long old
,
562 unsigned long _new
, int len
);
563 #define compat_uatomic_cmpxchg(addr, old, _new) \
564 ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
565 caa_cast_long_keep_sign(old), \
566 caa_cast_long_keep_sign(_new), \
569 extern void _compat_uatomic_and(void *addr
, unsigned long _new
, int len
);
570 #define compat_uatomic_and(addr, v) \
571 (_compat_uatomic_and((addr), \
572 caa_cast_long_keep_sign(v), \
575 extern void _compat_uatomic_or(void *addr
, unsigned long _new
, int len
);
576 #define compat_uatomic_or(addr, v) \
577 (_compat_uatomic_or((addr), \
578 caa_cast_long_keep_sign(v), \
581 extern unsigned long _compat_uatomic_add_return(void *addr
,
582 unsigned long _new
, int len
);
583 #define compat_uatomic_add_return(addr, v) \
584 ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
585 caa_cast_long_keep_sign(v), \
588 #define compat_uatomic_add(addr, v) \
589 ((void)compat_uatomic_add_return((addr), (v)))
590 #define compat_uatomic_inc(addr) \
591 (compat_uatomic_add((addr), 1))
592 #define compat_uatomic_dec(addr) \
593 (compat_uatomic_add((addr), -1))
596 #define UATOMIC_COMPAT(insn) (_uatomic_##insn)
599 /* Read is atomic even in compat mode */
600 #define uatomic_set(addr, v) \
601 UATOMIC_COMPAT(set(addr, v))
603 #define uatomic_cmpxchg(addr, old, _new) \
604 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
605 #define uatomic_xchg(addr, v) \
606 UATOMIC_COMPAT(xchg(addr, v))
608 #define uatomic_and(addr, v) \
609 UATOMIC_COMPAT(and(addr, v))
610 #define cmm_smp_mb__before_uatomic_and() cmm_barrier()
611 #define cmm_smp_mb__after_uatomic_and() cmm_barrier()
613 #define uatomic_or(addr, v) \
614 UATOMIC_COMPAT(or(addr, v))
615 #define cmm_smp_mb__before_uatomic_or() cmm_barrier()
616 #define cmm_smp_mb__after_uatomic_or() cmm_barrier()
618 #define uatomic_add_return(addr, v) \
619 UATOMIC_COMPAT(add_return(addr, v))
621 #define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
622 #define cmm_smp_mb__before_uatomic_add() cmm_barrier()
623 #define cmm_smp_mb__after_uatomic_add() cmm_barrier()
625 #define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
626 #define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
627 #define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
629 #define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
630 #define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
631 #define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
637 #include <urcu/uatomic/generic.h>
639 #endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.04282 seconds and 4 git commands to generate.