fix: explicitly include urcu/config.h in files using CONFIG_RCU_ defines
[urcu.git] / include / urcu / uatomic / x86.h
... / ...
CommitLineData
1#ifndef _URCU_ARCH_UATOMIC_X86_H
2#define _URCU_ARCH_UATOMIC_X86_H
3
4/*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 *
10 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
11 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
12 *
13 * Permission is hereby granted to use or copy this program
14 * for any purpose, provided the above notices are retained on all copies.
15 * Permission to modify the code and to distribute modified code is granted,
16 * provided the above notices are retained, and a notice that the code was
17 * modified is included with the above copyright notice.
18 *
19 * Code inspired from libuatomic_ops-1.2, inherited in part from the
20 * Boehm-Demers-Weiser conservative garbage collector.
21 */
22
23#include <urcu/config.h>
24#include <urcu/compiler.h>
25#include <urcu/system.h>
26
27#define UATOMIC_HAS_ATOMIC_BYTE
28#define UATOMIC_HAS_ATOMIC_SHORT
29
30#ifdef __cplusplus
31extern "C" {
32#endif
33
34/*
35 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
36 */
37
38struct __uatomic_dummy {
39 unsigned long v[10];
40};
41#define __hp(x) ((struct __uatomic_dummy *)(x))
42
43#define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
44
45/* cmpxchg */
46
47static inline __attribute__((always_inline))
48unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
49 unsigned long _new, int len)
50{
51 switch (len) {
52 case 1:
53 {
54 unsigned char result = old;
55
56 __asm__ __volatile__(
57 "lock; cmpxchgb %2, %1"
58 : "+a"(result), "+m"(*__hp(addr))
59 : "q"((unsigned char)_new)
60 : "memory");
61 return result;
62 }
63 case 2:
64 {
65 unsigned short result = old;
66
67 __asm__ __volatile__(
68 "lock; cmpxchgw %2, %1"
69 : "+a"(result), "+m"(*__hp(addr))
70 : "r"((unsigned short)_new)
71 : "memory");
72 return result;
73 }
74 case 4:
75 {
76 unsigned int result = old;
77
78 __asm__ __volatile__(
79 "lock; cmpxchgl %2, %1"
80 : "+a"(result), "+m"(*__hp(addr))
81 : "r"((unsigned int)_new)
82 : "memory");
83 return result;
84 }
85#if (CAA_BITS_PER_LONG == 64)
86 case 8:
87 {
88 unsigned long result = old;
89
90 __asm__ __volatile__(
91 "lock; cmpxchgq %2, %1"
92 : "+a"(result), "+m"(*__hp(addr))
93 : "r"((unsigned long)_new)
94 : "memory");
95 return result;
96 }
97#endif
98 }
99 /*
100 * generate an illegal instruction. Cannot catch this with
101 * linker tricks when optimizations are disabled.
102 */
103 __asm__ __volatile__("ud2");
104 return 0;
105}
106
107#define _uatomic_cmpxchg(addr, old, _new) \
108 ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
109 caa_cast_long_keep_sign(old), \
110 caa_cast_long_keep_sign(_new),\
111 sizeof(*(addr))))
112
113/* xchg */
114
115static inline __attribute__((always_inline))
116unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
117{
118 /* Note: the "xchg" instruction does not need a "lock" prefix. */
119 switch (len) {
120 case 1:
121 {
122 unsigned char result;
123 __asm__ __volatile__(
124 "xchgb %0, %1"
125 : "=q"(result), "+m"(*__hp(addr))
126 : "0" ((unsigned char)val)
127 : "memory");
128 return result;
129 }
130 case 2:
131 {
132 unsigned short result;
133 __asm__ __volatile__(
134 "xchgw %0, %1"
135 : "=r"(result), "+m"(*__hp(addr))
136 : "0" ((unsigned short)val)
137 : "memory");
138 return result;
139 }
140 case 4:
141 {
142 unsigned int result;
143 __asm__ __volatile__(
144 "xchgl %0, %1"
145 : "=r"(result), "+m"(*__hp(addr))
146 : "0" ((unsigned int)val)
147 : "memory");
148 return result;
149 }
150#if (CAA_BITS_PER_LONG == 64)
151 case 8:
152 {
153 unsigned long result;
154 __asm__ __volatile__(
155 "xchgq %0, %1"
156 : "=r"(result), "+m"(*__hp(addr))
157 : "0" ((unsigned long)val)
158 : "memory");
159 return result;
160 }
161#endif
162 }
163 /*
164 * generate an illegal instruction. Cannot catch this with
165 * linker tricks when optimizations are disabled.
166 */
167 __asm__ __volatile__("ud2");
168 return 0;
169}
170
171#define _uatomic_xchg(addr, v) \
172 ((__typeof__(*(addr))) __uatomic_exchange((addr), \
173 caa_cast_long_keep_sign(v), \
174 sizeof(*(addr))))
175
176/* uatomic_add_return */
177
178static inline __attribute__((always_inline))
179unsigned long __uatomic_add_return(void *addr, unsigned long val,
180 int len)
181{
182 switch (len) {
183 case 1:
184 {
185 unsigned char result = val;
186
187 __asm__ __volatile__(
188 "lock; xaddb %1, %0"
189 : "+m"(*__hp(addr)), "+q" (result)
190 :
191 : "memory");
192 return result + (unsigned char)val;
193 }
194 case 2:
195 {
196 unsigned short result = val;
197
198 __asm__ __volatile__(
199 "lock; xaddw %1, %0"
200 : "+m"(*__hp(addr)), "+r" (result)
201 :
202 : "memory");
203 return result + (unsigned short)val;
204 }
205 case 4:
206 {
207 unsigned int result = val;
208
209 __asm__ __volatile__(
210 "lock; xaddl %1, %0"
211 : "+m"(*__hp(addr)), "+r" (result)
212 :
213 : "memory");
214 return result + (unsigned int)val;
215 }
216#if (CAA_BITS_PER_LONG == 64)
217 case 8:
218 {
219 unsigned long result = val;
220
221 __asm__ __volatile__(
222 "lock; xaddq %1, %0"
223 : "+m"(*__hp(addr)), "+r" (result)
224 :
225 : "memory");
226 return result + (unsigned long)val;
227 }
228#endif
229 }
230 /*
231 * generate an illegal instruction. Cannot catch this with
232 * linker tricks when optimizations are disabled.
233 */
234 __asm__ __volatile__("ud2");
235 return 0;
236}
237
238#define _uatomic_add_return(addr, v) \
239 ((__typeof__(*(addr))) __uatomic_add_return((addr), \
240 caa_cast_long_keep_sign(v), \
241 sizeof(*(addr))))
242
243/* uatomic_and */
244
245static inline __attribute__((always_inline))
246void __uatomic_and(void *addr, unsigned long val, int len)
247{
248 switch (len) {
249 case 1:
250 {
251 __asm__ __volatile__(
252 "lock; andb %1, %0"
253 : "=m"(*__hp(addr))
254 : "iq" ((unsigned char)val)
255 : "memory");
256 return;
257 }
258 case 2:
259 {
260 __asm__ __volatile__(
261 "lock; andw %1, %0"
262 : "=m"(*__hp(addr))
263 : "ir" ((unsigned short)val)
264 : "memory");
265 return;
266 }
267 case 4:
268 {
269 __asm__ __volatile__(
270 "lock; andl %1, %0"
271 : "=m"(*__hp(addr))
272 : "ir" ((unsigned int)val)
273 : "memory");
274 return;
275 }
276#if (CAA_BITS_PER_LONG == 64)
277 case 8:
278 {
279 __asm__ __volatile__(
280 "lock; andq %1, %0"
281 : "=m"(*__hp(addr))
282 : "er" ((unsigned long)val)
283 : "memory");
284 return;
285 }
286#endif
287 }
288 /*
289 * generate an illegal instruction. Cannot catch this with
290 * linker tricks when optimizations are disabled.
291 */
292 __asm__ __volatile__("ud2");
293 return;
294}
295
296#define _uatomic_and(addr, v) \
297 (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
298
299/* uatomic_or */
300
301static inline __attribute__((always_inline))
302void __uatomic_or(void *addr, unsigned long val, int len)
303{
304 switch (len) {
305 case 1:
306 {
307 __asm__ __volatile__(
308 "lock; orb %1, %0"
309 : "=m"(*__hp(addr))
310 : "iq" ((unsigned char)val)
311 : "memory");
312 return;
313 }
314 case 2:
315 {
316 __asm__ __volatile__(
317 "lock; orw %1, %0"
318 : "=m"(*__hp(addr))
319 : "ir" ((unsigned short)val)
320 : "memory");
321 return;
322 }
323 case 4:
324 {
325 __asm__ __volatile__(
326 "lock; orl %1, %0"
327 : "=m"(*__hp(addr))
328 : "ir" ((unsigned int)val)
329 : "memory");
330 return;
331 }
332#if (CAA_BITS_PER_LONG == 64)
333 case 8:
334 {
335 __asm__ __volatile__(
336 "lock; orq %1, %0"
337 : "=m"(*__hp(addr))
338 : "er" ((unsigned long)val)
339 : "memory");
340 return;
341 }
342#endif
343 }
344 /*
345 * generate an illegal instruction. Cannot catch this with
346 * linker tricks when optimizations are disabled.
347 */
348 __asm__ __volatile__("ud2");
349 return;
350}
351
352#define _uatomic_or(addr, v) \
353 (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
354
355/* uatomic_add */
356
357static inline __attribute__((always_inline))
358void __uatomic_add(void *addr, unsigned long val, int len)
359{
360 switch (len) {
361 case 1:
362 {
363 __asm__ __volatile__(
364 "lock; addb %1, %0"
365 : "=m"(*__hp(addr))
366 : "iq" ((unsigned char)val)
367 : "memory");
368 return;
369 }
370 case 2:
371 {
372 __asm__ __volatile__(
373 "lock; addw %1, %0"
374 : "=m"(*__hp(addr))
375 : "ir" ((unsigned short)val)
376 : "memory");
377 return;
378 }
379 case 4:
380 {
381 __asm__ __volatile__(
382 "lock; addl %1, %0"
383 : "=m"(*__hp(addr))
384 : "ir" ((unsigned int)val)
385 : "memory");
386 return;
387 }
388#if (CAA_BITS_PER_LONG == 64)
389 case 8:
390 {
391 __asm__ __volatile__(
392 "lock; addq %1, %0"
393 : "=m"(*__hp(addr))
394 : "er" ((unsigned long)val)
395 : "memory");
396 return;
397 }
398#endif
399 }
400 /*
401 * generate an illegal instruction. Cannot catch this with
402 * linker tricks when optimizations are disabled.
403 */
404 __asm__ __volatile__("ud2");
405 return;
406}
407
408#define _uatomic_add(addr, v) \
409 (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
410
411
412/* uatomic_inc */
413
414static inline __attribute__((always_inline))
415void __uatomic_inc(void *addr, int len)
416{
417 switch (len) {
418 case 1:
419 {
420 __asm__ __volatile__(
421 "lock; incb %0"
422 : "=m"(*__hp(addr))
423 :
424 : "memory");
425 return;
426 }
427 case 2:
428 {
429 __asm__ __volatile__(
430 "lock; incw %0"
431 : "=m"(*__hp(addr))
432 :
433 : "memory");
434 return;
435 }
436 case 4:
437 {
438 __asm__ __volatile__(
439 "lock; incl %0"
440 : "=m"(*__hp(addr))
441 :
442 : "memory");
443 return;
444 }
445#if (CAA_BITS_PER_LONG == 64)
446 case 8:
447 {
448 __asm__ __volatile__(
449 "lock; incq %0"
450 : "=m"(*__hp(addr))
451 :
452 : "memory");
453 return;
454 }
455#endif
456 }
457 /* generate an illegal instruction. Cannot catch this with linker tricks
458 * when optimizations are disabled. */
459 __asm__ __volatile__("ud2");
460 return;
461}
462
463#define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
464
465/* uatomic_dec */
466
467static inline __attribute__((always_inline))
468void __uatomic_dec(void *addr, int len)
469{
470 switch (len) {
471 case 1:
472 {
473 __asm__ __volatile__(
474 "lock; decb %0"
475 : "=m"(*__hp(addr))
476 :
477 : "memory");
478 return;
479 }
480 case 2:
481 {
482 __asm__ __volatile__(
483 "lock; decw %0"
484 : "=m"(*__hp(addr))
485 :
486 : "memory");
487 return;
488 }
489 case 4:
490 {
491 __asm__ __volatile__(
492 "lock; decl %0"
493 : "=m"(*__hp(addr))
494 :
495 : "memory");
496 return;
497 }
498#if (CAA_BITS_PER_LONG == 64)
499 case 8:
500 {
501 __asm__ __volatile__(
502 "lock; decq %0"
503 : "=m"(*__hp(addr))
504 :
505 : "memory");
506 return;
507 }
508#endif
509 }
510 /*
511 * generate an illegal instruction. Cannot catch this with
512 * linker tricks when optimizations are disabled.
513 */
514 __asm__ __volatile__("ud2");
515 return;
516}
517
518#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
519
520#if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
521extern int __rcu_cas_avail;
522extern int __rcu_cas_init(void);
523
524#define UATOMIC_COMPAT(insn) \
525 ((caa_likely(__rcu_cas_avail > 0)) \
526 ? (_uatomic_##insn) \
527 : ((caa_unlikely(__rcu_cas_avail < 0) \
528 ? ((__rcu_cas_init() > 0) \
529 ? (_uatomic_##insn) \
530 : (compat_uatomic_##insn)) \
531 : (compat_uatomic_##insn))))
532
533/*
534 * We leave the return value so we don't break the ABI, but remove the
535 * return value from the API.
536 */
537extern unsigned long _compat_uatomic_set(void *addr,
538 unsigned long _new, int len);
539#define compat_uatomic_set(addr, _new) \
540 ((void) _compat_uatomic_set((addr), \
541 caa_cast_long_keep_sign(_new), \
542 sizeof(*(addr))))
543
544
545extern unsigned long _compat_uatomic_xchg(void *addr,
546 unsigned long _new, int len);
547#define compat_uatomic_xchg(addr, _new) \
548 ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
549 caa_cast_long_keep_sign(_new), \
550 sizeof(*(addr))))
551
552extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
553 unsigned long _new, int len);
554#define compat_uatomic_cmpxchg(addr, old, _new) \
555 ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
556 caa_cast_long_keep_sign(old), \
557 caa_cast_long_keep_sign(_new), \
558 sizeof(*(addr))))
559
560extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
561#define compat_uatomic_and(addr, v) \
562 (_compat_uatomic_and((addr), \
563 caa_cast_long_keep_sign(v), \
564 sizeof(*(addr))))
565
566extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
567#define compat_uatomic_or(addr, v) \
568 (_compat_uatomic_or((addr), \
569 caa_cast_long_keep_sign(v), \
570 sizeof(*(addr))))
571
572extern unsigned long _compat_uatomic_add_return(void *addr,
573 unsigned long _new, int len);
574#define compat_uatomic_add_return(addr, v) \
575 ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
576 caa_cast_long_keep_sign(v), \
577 sizeof(*(addr))))
578
579#define compat_uatomic_add(addr, v) \
580 ((void)compat_uatomic_add_return((addr), (v)))
581#define compat_uatomic_inc(addr) \
582 (compat_uatomic_add((addr), 1))
583#define compat_uatomic_dec(addr) \
584 (compat_uatomic_add((addr), -1))
585
586#else
587#define UATOMIC_COMPAT(insn) (_uatomic_##insn)
588#endif
589
590/* Read is atomic even in compat mode */
591#define uatomic_set(addr, v) \
592 UATOMIC_COMPAT(set(addr, v))
593
594#define uatomic_cmpxchg(addr, old, _new) \
595 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
596#define uatomic_xchg(addr, v) \
597 UATOMIC_COMPAT(xchg(addr, v))
598
599#define uatomic_and(addr, v) \
600 UATOMIC_COMPAT(and(addr, v))
601#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
602#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
603
604#define uatomic_or(addr, v) \
605 UATOMIC_COMPAT(or(addr, v))
606#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
607#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
608
609#define uatomic_add_return(addr, v) \
610 UATOMIC_COMPAT(add_return(addr, v))
611
612#define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
613#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
614#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
615
616#define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
617#define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
618#define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
619
620#define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
621#define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
622#define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
623
624#ifdef __cplusplus
625}
626#endif
627
628#include <urcu/uatomic/generic.h>
629
630#endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.040604 seconds and 4 git commands to generate.