ce1ce5e9b48b75f193d9c80f81977c2005888492
[urcu.git] / urcu / uatomic / x86.h
1 #ifndef _URCU_ARCH_UATOMIC_X86_H
2 #define _URCU_ARCH_UATOMIC_X86_H
3
4 /*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 *
10 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
11 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
12 *
13 * Permission is hereby granted to use or copy this program
14 * for any purpose, provided the above notices are retained on all copies.
15 * Permission to modify the code and to distribute modified code is granted,
16 * provided the above notices are retained, and a notice that the code was
17 * modified is included with the above copyright notice.
18 *
19 * Code inspired from libuatomic_ops-1.2, inherited in part from the
20 * Boehm-Demers-Weiser conservative garbage collector.
21 */
22
23 #include <urcu/compiler.h>
24 #include <urcu/system.h>
25
26 #define UATOMIC_HAS_ATOMIC_BYTE
27 #define UATOMIC_HAS_ATOMIC_SHORT
28
29 #ifdef __cplusplus
30 extern "C" {
31 #endif
32
33 /*
34 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
35 */
36
37 struct __uatomic_dummy {
38 unsigned long v[10];
39 };
40 #define __hp(x) ((struct __uatomic_dummy *)(x))
41
42 #define _uatomic_set(addr, v) CMM_STORE_SHARED(*(addr), (v))
43
44 /* cmpxchg */
45
46 static inline __attribute__((always_inline))
47 unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
48 unsigned long _new, int len)
49 {
50 switch (len) {
51 case 1:
52 {
53 unsigned char result = old;
54
55 __asm__ __volatile__(
56 "lock; cmpxchgb %2, %1"
57 : "+a"(result), "+m"(*__hp(addr))
58 : "q"((unsigned char)_new)
59 : "memory");
60 return result;
61 }
62 case 2:
63 {
64 unsigned short result = old;
65
66 __asm__ __volatile__(
67 "lock; cmpxchgw %2, %1"
68 : "+a"(result), "+m"(*__hp(addr))
69 : "r"((unsigned short)_new)
70 : "memory");
71 return result;
72 }
73 case 4:
74 {
75 unsigned int result = old;
76
77 __asm__ __volatile__(
78 "lock; cmpxchgl %2, %1"
79 : "+a"(result), "+m"(*__hp(addr))
80 : "r"((unsigned int)_new)
81 : "memory");
82 return result;
83 }
84 #if (CAA_BITS_PER_LONG == 64)
85 case 8:
86 {
87 unsigned long result = old;
88
89 __asm__ __volatile__(
90 "lock; cmpxchgq %2, %1"
91 : "+a"(result), "+m"(*__hp(addr))
92 : "r"((unsigned long)_new)
93 : "memory");
94 return result;
95 }
96 #endif
97 }
98 /*
99 * generate an illegal instruction. Cannot catch this with
100 * linker tricks when optimizations are disabled.
101 */
102 __asm__ __volatile__("ud2");
103 return 0;
104 }
105
106 #define _uatomic_cmpxchg(addr, old, _new) \
107 ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
108 caa_cast_long_keep_sign(old), \
109 caa_cast_long_keep_sign(_new),\
110 sizeof(*(addr))))
111
112 /* xchg */
113
114 static inline __attribute__((always_inline))
115 unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
116 {
117 /* Note: the "xchg" instruction does not need a "lock" prefix. */
118 switch (len) {
119 case 1:
120 {
121 unsigned char result;
122 __asm__ __volatile__(
123 "xchgb %0, %1"
124 : "=q"(result), "+m"(*__hp(addr))
125 : "0" ((unsigned char)val)
126 : "memory");
127 return result;
128 }
129 case 2:
130 {
131 unsigned short result;
132 __asm__ __volatile__(
133 "xchgw %0, %1"
134 : "=r"(result), "+m"(*__hp(addr))
135 : "0" ((unsigned short)val)
136 : "memory");
137 return result;
138 }
139 case 4:
140 {
141 unsigned int result;
142 __asm__ __volatile__(
143 "xchgl %0, %1"
144 : "=r"(result), "+m"(*__hp(addr))
145 : "0" ((unsigned int)val)
146 : "memory");
147 return result;
148 }
149 #if (CAA_BITS_PER_LONG == 64)
150 case 8:
151 {
152 unsigned long result;
153 __asm__ __volatile__(
154 "xchgq %0, %1"
155 : "=r"(result), "+m"(*__hp(addr))
156 : "0" ((unsigned long)val)
157 : "memory");
158 return result;
159 }
160 #endif
161 }
162 /*
163 * generate an illegal instruction. Cannot catch this with
164 * linker tricks when optimizations are disabled.
165 */
166 __asm__ __volatile__("ud2");
167 return 0;
168 }
169
170 #define _uatomic_xchg(addr, v) \
171 ((__typeof__(*(addr))) __uatomic_exchange((addr), \
172 caa_cast_long_keep_sign(v), \
173 sizeof(*(addr))))
174
175 /* uatomic_add_return */
176
177 static inline __attribute__((always_inline))
178 unsigned long __uatomic_add_return(void *addr, unsigned long val,
179 int len)
180 {
181 switch (len) {
182 case 1:
183 {
184 unsigned char result = val;
185
186 __asm__ __volatile__(
187 "lock; xaddb %1, %0"
188 : "+m"(*__hp(addr)), "+q" (result)
189 :
190 : "memory");
191 return result + (unsigned char)val;
192 }
193 case 2:
194 {
195 unsigned short result = val;
196
197 __asm__ __volatile__(
198 "lock; xaddw %1, %0"
199 : "+m"(*__hp(addr)), "+r" (result)
200 :
201 : "memory");
202 return result + (unsigned short)val;
203 }
204 case 4:
205 {
206 unsigned int result = val;
207
208 __asm__ __volatile__(
209 "lock; xaddl %1, %0"
210 : "+m"(*__hp(addr)), "+r" (result)
211 :
212 : "memory");
213 return result + (unsigned int)val;
214 }
215 #if (CAA_BITS_PER_LONG == 64)
216 case 8:
217 {
218 unsigned long result = val;
219
220 __asm__ __volatile__(
221 "lock; xaddq %1, %0"
222 : "+m"(*__hp(addr)), "+r" (result)
223 :
224 : "memory");
225 return result + (unsigned long)val;
226 }
227 #endif
228 }
229 /*
230 * generate an illegal instruction. Cannot catch this with
231 * linker tricks when optimizations are disabled.
232 */
233 __asm__ __volatile__("ud2");
234 return 0;
235 }
236
237 #define _uatomic_add_return(addr, v) \
238 ((__typeof__(*(addr))) __uatomic_add_return((addr), \
239 caa_cast_long_keep_sign(v), \
240 sizeof(*(addr))))
241
242 /* uatomic_and */
243
244 static inline __attribute__((always_inline))
245 void __uatomic_and(void *addr, unsigned long val, int len)
246 {
247 switch (len) {
248 case 1:
249 {
250 __asm__ __volatile__(
251 "lock; andb %1, %0"
252 : "=m"(*__hp(addr))
253 : "iq" ((unsigned char)val)
254 : "memory");
255 return;
256 }
257 case 2:
258 {
259 __asm__ __volatile__(
260 "lock; andw %1, %0"
261 : "=m"(*__hp(addr))
262 : "ir" ((unsigned short)val)
263 : "memory");
264 return;
265 }
266 case 4:
267 {
268 __asm__ __volatile__(
269 "lock; andl %1, %0"
270 : "=m"(*__hp(addr))
271 : "ir" ((unsigned int)val)
272 : "memory");
273 return;
274 }
275 #if (CAA_BITS_PER_LONG == 64)
276 case 8:
277 {
278 __asm__ __volatile__(
279 "lock; andq %1, %0"
280 : "=m"(*__hp(addr))
281 : "er" ((unsigned long)val)
282 : "memory");
283 return;
284 }
285 #endif
286 }
287 /*
288 * generate an illegal instruction. Cannot catch this with
289 * linker tricks when optimizations are disabled.
290 */
291 __asm__ __volatile__("ud2");
292 return;
293 }
294
295 #define _uatomic_and(addr, v) \
296 (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
297
298 /* uatomic_or */
299
300 static inline __attribute__((always_inline))
301 void __uatomic_or(void *addr, unsigned long val, int len)
302 {
303 switch (len) {
304 case 1:
305 {
306 __asm__ __volatile__(
307 "lock; orb %1, %0"
308 : "=m"(*__hp(addr))
309 : "iq" ((unsigned char)val)
310 : "memory");
311 return;
312 }
313 case 2:
314 {
315 __asm__ __volatile__(
316 "lock; orw %1, %0"
317 : "=m"(*__hp(addr))
318 : "ir" ((unsigned short)val)
319 : "memory");
320 return;
321 }
322 case 4:
323 {
324 __asm__ __volatile__(
325 "lock; orl %1, %0"
326 : "=m"(*__hp(addr))
327 : "ir" ((unsigned int)val)
328 : "memory");
329 return;
330 }
331 #if (CAA_BITS_PER_LONG == 64)
332 case 8:
333 {
334 __asm__ __volatile__(
335 "lock; orq %1, %0"
336 : "=m"(*__hp(addr))
337 : "er" ((unsigned long)val)
338 : "memory");
339 return;
340 }
341 #endif
342 }
343 /*
344 * generate an illegal instruction. Cannot catch this with
345 * linker tricks when optimizations are disabled.
346 */
347 __asm__ __volatile__("ud2");
348 return;
349 }
350
351 #define _uatomic_or(addr, v) \
352 (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
353
354 /* uatomic_add */
355
356 static inline __attribute__((always_inline))
357 void __uatomic_add(void *addr, unsigned long val, int len)
358 {
359 switch (len) {
360 case 1:
361 {
362 __asm__ __volatile__(
363 "lock; addb %1, %0"
364 : "=m"(*__hp(addr))
365 : "iq" ((unsigned char)val)
366 : "memory");
367 return;
368 }
369 case 2:
370 {
371 __asm__ __volatile__(
372 "lock; addw %1, %0"
373 : "=m"(*__hp(addr))
374 : "ir" ((unsigned short)val)
375 : "memory");
376 return;
377 }
378 case 4:
379 {
380 __asm__ __volatile__(
381 "lock; addl %1, %0"
382 : "=m"(*__hp(addr))
383 : "ir" ((unsigned int)val)
384 : "memory");
385 return;
386 }
387 #if (CAA_BITS_PER_LONG == 64)
388 case 8:
389 {
390 __asm__ __volatile__(
391 "lock; addq %1, %0"
392 : "=m"(*__hp(addr))
393 : "er" ((unsigned long)val)
394 : "memory");
395 return;
396 }
397 #endif
398 }
399 /*
400 * generate an illegal instruction. Cannot catch this with
401 * linker tricks when optimizations are disabled.
402 */
403 __asm__ __volatile__("ud2");
404 return;
405 }
406
407 #define _uatomic_add(addr, v) \
408 (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
409
410
411 /* uatomic_inc */
412
413 static inline __attribute__((always_inline))
414 void __uatomic_inc(void *addr, int len)
415 {
416 switch (len) {
417 case 1:
418 {
419 __asm__ __volatile__(
420 "lock; incb %0"
421 : "=m"(*__hp(addr))
422 :
423 : "memory");
424 return;
425 }
426 case 2:
427 {
428 __asm__ __volatile__(
429 "lock; incw %0"
430 : "=m"(*__hp(addr))
431 :
432 : "memory");
433 return;
434 }
435 case 4:
436 {
437 __asm__ __volatile__(
438 "lock; incl %0"
439 : "=m"(*__hp(addr))
440 :
441 : "memory");
442 return;
443 }
444 #if (CAA_BITS_PER_LONG == 64)
445 case 8:
446 {
447 __asm__ __volatile__(
448 "lock; incq %0"
449 : "=m"(*__hp(addr))
450 :
451 : "memory");
452 return;
453 }
454 #endif
455 }
456 /* generate an illegal instruction. Cannot catch this with linker tricks
457 * when optimizations are disabled. */
458 __asm__ __volatile__("ud2");
459 return;
460 }
461
462 #define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
463
464 /* uatomic_dec */
465
466 static inline __attribute__((always_inline))
467 void __uatomic_dec(void *addr, int len)
468 {
469 switch (len) {
470 case 1:
471 {
472 __asm__ __volatile__(
473 "lock; decb %0"
474 : "=m"(*__hp(addr))
475 :
476 : "memory");
477 return;
478 }
479 case 2:
480 {
481 __asm__ __volatile__(
482 "lock; decw %0"
483 : "=m"(*__hp(addr))
484 :
485 : "memory");
486 return;
487 }
488 case 4:
489 {
490 __asm__ __volatile__(
491 "lock; decl %0"
492 : "=m"(*__hp(addr))
493 :
494 : "memory");
495 return;
496 }
497 #if (CAA_BITS_PER_LONG == 64)
498 case 8:
499 {
500 __asm__ __volatile__(
501 "lock; decq %0"
502 : "=m"(*__hp(addr))
503 :
504 : "memory");
505 return;
506 }
507 #endif
508 }
509 /*
510 * generate an illegal instruction. Cannot catch this with
511 * linker tricks when optimizations are disabled.
512 */
513 __asm__ __volatile__("ud2");
514 return;
515 }
516
517 #define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
518
519 #if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
520 extern int __rcu_cas_avail;
521 extern int __rcu_cas_init(void);
522
523 #define UATOMIC_COMPAT(insn) \
524 ((caa_likely(__rcu_cas_avail > 0)) \
525 ? (_uatomic_##insn) \
526 : ((caa_unlikely(__rcu_cas_avail < 0) \
527 ? ((__rcu_cas_init() > 0) \
528 ? (_uatomic_##insn) \
529 : (compat_uatomic_##insn)) \
530 : (compat_uatomic_##insn))))
531
532 extern unsigned long _compat_uatomic_set(void *addr,
533 unsigned long _new, int len);
534 #define compat_uatomic_set(addr, _new) \
535 ((__typeof__(*(addr))) _compat_uatomic_set((addr), \
536 caa_cast_long_keep_sign(_new), \
537 sizeof(*(addr))))
538
539
540 extern unsigned long _compat_uatomic_xchg(void *addr,
541 unsigned long _new, int len);
542 #define compat_uatomic_xchg(addr, _new) \
543 ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
544 caa_cast_long_keep_sign(_new), \
545 sizeof(*(addr))))
546
547 extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
548 unsigned long _new, int len);
549 #define compat_uatomic_cmpxchg(addr, old, _new) \
550 ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
551 caa_cast_long_keep_sign(old), \
552 caa_cast_long_keep_sign(_new), \
553 sizeof(*(addr))))
554
555 extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
556 #define compat_uatomic_and(addr, v) \
557 (_compat_uatomic_and((addr), \
558 caa_cast_long_keep_sign(v), \
559 sizeof(*(addr))))
560
561 extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
562 #define compat_uatomic_or(addr, v) \
563 (_compat_uatomic_or((addr), \
564 caa_cast_long_keep_sign(v), \
565 sizeof(*(addr))))
566
567 extern unsigned long _compat_uatomic_add_return(void *addr,
568 unsigned long _new, int len);
569 #define compat_uatomic_add_return(addr, v) \
570 ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
571 caa_cast_long_keep_sign(v), \
572 sizeof(*(addr))))
573
574 #define compat_uatomic_add(addr, v) \
575 ((void)compat_uatomic_add_return((addr), (v)))
576 #define compat_uatomic_inc(addr) \
577 (compat_uatomic_add((addr), 1))
578 #define compat_uatomic_dec(addr) \
579 (compat_uatomic_add((addr), -1))
580
581 #else
582 #define UATOMIC_COMPAT(insn) (_uatomic_##insn)
583 #endif
584
585 /* Read is atomic even in compat mode */
586 #define uatomic_set(addr, v) \
587 UATOMIC_COMPAT(set(addr, v))
588
589 #define uatomic_cmpxchg(addr, old, _new) \
590 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
591 #define uatomic_xchg(addr, v) \
592 UATOMIC_COMPAT(xchg(addr, v))
593
594 #define uatomic_and(addr, v) \
595 UATOMIC_COMPAT(and(addr, v))
596 #define cmm_smp_mb__before_uatomic_and() cmm_barrier()
597 #define cmm_smp_mb__after_uatomic_and() cmm_barrier()
598
599 #define uatomic_or(addr, v) \
600 UATOMIC_COMPAT(or(addr, v))
601 #define cmm_smp_mb__before_uatomic_or() cmm_barrier()
602 #define cmm_smp_mb__after_uatomic_or() cmm_barrier()
603
604 #define uatomic_add_return(addr, v) \
605 UATOMIC_COMPAT(add_return(addr, v))
606
607 #define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
608 #define cmm_smp_mb__before_uatomic_add() cmm_barrier()
609 #define cmm_smp_mb__after_uatomic_add() cmm_barrier()
610
611 #define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
612 #define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
613 #define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
614
615 #define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
616 #define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
617 #define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
618
619 #ifdef __cplusplus
620 }
621 #endif
622
623 #include <urcu/uatomic/generic.h>
624
625 #endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.061606 seconds and 3 git commands to generate.