37f59cc11429229ce0dc19aa17a746b03f458cb7
[urcu.git] / urcu / uatomic / generic.h
1 #ifndef _URCU_UATOMIC_GENERIC_H
2 #define _URCU_UATOMIC_GENERIC_H
3
4 /*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 * Copyright (c) 2010 Paolo Bonzini
10 *
11 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
12 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
13 *
14 * Permission is hereby granted to use or copy this program
15 * for any purpose, provided the above notices are retained on all copies.
16 * Permission to modify the code and to distribute modified code is granted,
17 * provided the above notices are retained, and a notice that the code was
18 * modified is included with the above copyright notice.
19 *
20 * Code inspired from libuatomic_ops-1.2, inherited in part from the
21 * Boehm-Demers-Weiser conservative garbage collector.
22 */
23
24 #include <urcu/compiler.h>
25 #include <urcu/system.h>
26
27 #ifdef __cplusplus
28 extern "C" {
29 #endif
30
31 #ifndef uatomic_set
32 #define uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
33 #endif
34
35 #ifndef uatomic_read
36 #define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
37 #endif
38
39 #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR
40 static inline __attribute__((always_inline))
41 void _uatomic_link_error(void)
42 {
43 #ifdef ILLEGAL_INSTR
44 /*
45 * generate an illegal instruction. Cannot catch this with
46 * linker tricks when optimizations are disabled.
47 */
48 __asm__ __volatile__(ILLEGAL_INSTR);
49 #else
50 __builtin_trap();
51 #endif
52 }
53
54 #else /* #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
55 extern void _uatomic_link_error(void);
56 #endif /* #else #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
57
58 /* cmpxchg */
59
60 #ifndef uatomic_cmpxchg
61 static inline __attribute__((always_inline))
62 unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
63 unsigned long _new, int len)
64 {
65 switch (len) {
66 #ifdef UATOMIC_HAS_ATOMIC_BYTE
67 case 1:
68 return __sync_val_compare_and_swap_1(addr, old, _new);
69 #endif
70 #ifdef UATOMIC_HAS_ATOMIC_SHORT
71 case 2:
72 return __sync_val_compare_and_swap_2(addr, old, _new);
73 #endif
74 case 4:
75 return __sync_val_compare_and_swap_4(addr, old, _new);
76 #if (CAA_BITS_PER_LONG == 64)
77 case 8:
78 return __sync_val_compare_and_swap_8(addr, old, _new);
79 #endif
80 }
81 _uatomic_link_error();
82 return 0;
83 }
84
85
86 #define uatomic_cmpxchg(addr, old, _new) \
87 ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
88 caa_cast_long_keep_sign(old), \
89 caa_cast_long_keep_sign(_new),\
90 sizeof(*(addr))))
91
92
93 /* uatomic_and */
94
95 #ifndef uatomic_and
96 static inline __attribute__((always_inline))
97 void _uatomic_and(void *addr, unsigned long val,
98 int len)
99 {
100 switch (len) {
101 #ifdef UATOMIC_HAS_ATOMIC_BYTE
102 case 1:
103 __sync_and_and_fetch_1(addr, val);
104 return;
105 #endif
106 #ifdef UATOMIC_HAS_ATOMIC_SHORT
107 case 2:
108 __sync_and_and_fetch_2(addr, val);
109 return;
110 #endif
111 case 4:
112 __sync_and_and_fetch_4(addr, val);
113 return;
114 #if (CAA_BITS_PER_LONG == 64)
115 case 8:
116 __sync_and_and_fetch_8(addr, val);
117 return;
118 #endif
119 }
120 _uatomic_link_error();
121 }
122
123 #define uatomic_and(addr, v) \
124 (_uatomic_and((addr), \
125 caa_cast_long_keep_sign(v), \
126 sizeof(*(addr))))
127 #define cmm_smp_mb__before_uatomic_and() cmm_barrier()
128 #define cmm_smp_mb__after_uatomic_and() cmm_barrier()
129
130 #endif
131
132 /* uatomic_or */
133
134 #ifndef uatomic_or
135 static inline __attribute__((always_inline))
136 void _uatomic_or(void *addr, unsigned long val,
137 int len)
138 {
139 switch (len) {
140 #ifdef UATOMIC_HAS_ATOMIC_BYTE
141 case 1:
142 __sync_or_and_fetch_1(addr, val);
143 return;
144 #endif
145 #ifdef UATOMIC_HAS_ATOMIC_SHORT
146 case 2:
147 __sync_or_and_fetch_2(addr, val);
148 return;
149 #endif
150 case 4:
151 __sync_or_and_fetch_4(addr, val);
152 return;
153 #if (CAA_BITS_PER_LONG == 64)
154 case 8:
155 __sync_or_and_fetch_8(addr, val);
156 return;
157 #endif
158 }
159 _uatomic_link_error();
160 return;
161 }
162
163 #define uatomic_or(addr, v) \
164 (_uatomic_or((addr), \
165 caa_cast_long_keep_sign(v), \
166 sizeof(*(addr))))
167 #define cmm_smp_mb__before_uatomic_or() cmm_barrier()
168 #define cmm_smp_mb__after_uatomic_or() cmm_barrier()
169
170 #endif
171
172
173 /* uatomic_add_return */
174
175 #ifndef uatomic_add_return
176 static inline __attribute__((always_inline))
177 unsigned long _uatomic_add_return(void *addr, unsigned long val,
178 int len)
179 {
180 switch (len) {
181 #ifdef UATOMIC_HAS_ATOMIC_BYTE
182 case 1:
183 return __sync_add_and_fetch_1(addr, val);
184 #endif
185 #ifdef UATOMIC_HAS_ATOMIC_SHORT
186 case 2:
187 return __sync_add_and_fetch_2(addr, val);
188 #endif
189 case 4:
190 return __sync_add_and_fetch_4(addr, val);
191 #if (CAA_BITS_PER_LONG == 64)
192 case 8:
193 return __sync_add_and_fetch_8(addr, val);
194 #endif
195 }
196 _uatomic_link_error();
197 return 0;
198 }
199
200
201 #define uatomic_add_return(addr, v) \
202 ((__typeof__(*(addr))) _uatomic_add_return((addr), \
203 caa_cast_long_keep_sign(v), \
204 sizeof(*(addr))))
205 #endif /* #ifndef uatomic_add_return */
206
207 #ifndef uatomic_xchg
208 /* xchg */
209
210 static inline __attribute__((always_inline))
211 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
212 {
213 switch (len) {
214 #ifdef UATOMIC_HAS_ATOMIC_BYTE
215 case 1:
216 {
217 unsigned char old;
218
219 do {
220 old = uatomic_read((unsigned char *)addr);
221 } while (!__sync_bool_compare_and_swap_1(addr, old, val));
222
223 return old;
224 }
225 #endif
226 #ifdef UATOMIC_HAS_ATOMIC_SHORT
227 case 2:
228 {
229 unsigned short old;
230
231 do {
232 old = uatomic_read((unsigned short *)addr);
233 } while (!__sync_bool_compare_and_swap_2(addr, old, val));
234
235 return old;
236 }
237 #endif
238 case 4:
239 {
240 unsigned int old;
241
242 do {
243 old = uatomic_read((unsigned int *)addr);
244 } while (!__sync_bool_compare_and_swap_4(addr, old, val));
245
246 return old;
247 }
248 #if (CAA_BITS_PER_LONG == 64)
249 case 8:
250 {
251 unsigned long old;
252
253 do {
254 old = uatomic_read((unsigned long *)addr);
255 } while (!__sync_bool_compare_and_swap_8(addr, old, val));
256
257 return old;
258 }
259 #endif
260 }
261 _uatomic_link_error();
262 return 0;
263 }
264
265 #define uatomic_xchg(addr, v) \
266 ((__typeof__(*(addr))) _uatomic_exchange((addr), \
267 caa_cast_long_keep_sign(v), \
268 sizeof(*(addr))))
269 #endif /* #ifndef uatomic_xchg */
270
271 #else /* #ifndef uatomic_cmpxchg */
272
273 #ifndef uatomic_and
274 /* uatomic_and */
275
276 static inline __attribute__((always_inline))
277 void _uatomic_and(void *addr, unsigned long val, int len)
278 {
279 switch (len) {
280 #ifdef UATOMIC_HAS_ATOMIC_BYTE
281 case 1:
282 {
283 unsigned char old, oldt;
284
285 oldt = uatomic_read((unsigned char *)addr);
286 do {
287 old = oldt;
288 oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
289 } while (oldt != old);
290
291 return;
292 }
293 #endif
294 #ifdef UATOMIC_HAS_ATOMIC_SHORT
295 case 2:
296 {
297 unsigned short old, oldt;
298
299 oldt = uatomic_read((unsigned short *)addr);
300 do {
301 old = oldt;
302 oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
303 } while (oldt != old);
304 }
305 #endif
306 case 4:
307 {
308 unsigned int old, oldt;
309
310 oldt = uatomic_read((unsigned int *)addr);
311 do {
312 old = oldt;
313 oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
314 } while (oldt != old);
315
316 return;
317 }
318 #if (CAA_BITS_PER_LONG == 64)
319 case 8:
320 {
321 unsigned long old, oldt;
322
323 oldt = uatomic_read((unsigned long *)addr);
324 do {
325 old = oldt;
326 oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
327 } while (oldt != old);
328
329 return;
330 }
331 #endif
332 }
333 _uatomic_link_error();
334 }
335
336 #define uatomic_and(addr, v) \
337 (_uatomic_and((addr), \
338 caa_cast_long_keep_sign(v), \
339 sizeof(*(addr))))
340 #define cmm_smp_mb__before_uatomic_and() cmm_barrier()
341 #define cmm_smp_mb__after_uatomic_and() cmm_barrier()
342
343 #endif /* #ifndef uatomic_and */
344
345 #ifndef uatomic_or
346 /* uatomic_or */
347
348 static inline __attribute__((always_inline))
349 void _uatomic_or(void *addr, unsigned long val, int len)
350 {
351 switch (len) {
352 #ifdef UATOMIC_HAS_ATOMIC_BYTE
353 case 1:
354 {
355 unsigned char old, oldt;
356
357 oldt = uatomic_read((unsigned char *)addr);
358 do {
359 old = oldt;
360 oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
361 } while (oldt != old);
362
363 return;
364 }
365 #endif
366 #ifdef UATOMIC_HAS_ATOMIC_SHORT
367 case 2:
368 {
369 unsigned short old, oldt;
370
371 oldt = uatomic_read((unsigned short *)addr);
372 do {
373 old = oldt;
374 oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
375 } while (oldt != old);
376
377 return;
378 }
379 #endif
380 case 4:
381 {
382 unsigned int old, oldt;
383
384 oldt = uatomic_read((unsigned int *)addr);
385 do {
386 old = oldt;
387 oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
388 } while (oldt != old);
389
390 return;
391 }
392 #if (CAA_BITS_PER_LONG == 64)
393 case 8:
394 {
395 unsigned long old, oldt;
396
397 oldt = uatomic_read((unsigned long *)addr);
398 do {
399 old = oldt;
400 oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
401 } while (oldt != old);
402
403 return;
404 }
405 #endif
406 }
407 _uatomic_link_error();
408 }
409
410 #define uatomic_or(addr, v) \
411 (_uatomic_or((addr), \
412 caa_cast_long_keep_sign(v), \
413 sizeof(*(addr))))
414 #define cmm_smp_mb__before_uatomic_or() cmm_barrier()
415 #define cmm_smp_mb__after_uatomic_or() cmm_barrier()
416
417 #endif /* #ifndef uatomic_or */
418
419 #ifndef uatomic_add_return
420 /* uatomic_add_return */
421
422 static inline __attribute__((always_inline))
423 unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
424 {
425 switch (len) {
426 #ifdef UATOMIC_HAS_ATOMIC_BYTE
427 case 1:
428 {
429 unsigned char old, oldt;
430
431 oldt = uatomic_read((unsigned char *)addr);
432 do {
433 old = oldt;
434 oldt = uatomic_cmpxchg((unsigned char *)addr,
435 old, old + val);
436 } while (oldt != old);
437
438 return old + val;
439 }
440 #endif
441 #ifdef UATOMIC_HAS_ATOMIC_SHORT
442 case 2:
443 {
444 unsigned short old, oldt;
445
446 oldt = uatomic_read((unsigned short *)addr);
447 do {
448 old = oldt;
449 oldt = uatomic_cmpxchg((unsigned short *)addr,
450 old, old + val);
451 } while (oldt != old);
452
453 return old + val;
454 }
455 #endif
456 case 4:
457 {
458 unsigned int old, oldt;
459
460 oldt = uatomic_read((unsigned int *)addr);
461 do {
462 old = oldt;
463 oldt = uatomic_cmpxchg((unsigned int *)addr,
464 old, old + val);
465 } while (oldt != old);
466
467 return old + val;
468 }
469 #if (CAA_BITS_PER_LONG == 64)
470 case 8:
471 {
472 unsigned long old, oldt;
473
474 oldt = uatomic_read((unsigned long *)addr);
475 do {
476 old = oldt;
477 oldt = uatomic_cmpxchg((unsigned long *)addr,
478 old, old + val);
479 } while (oldt != old);
480
481 return old + val;
482 }
483 #endif
484 }
485 _uatomic_link_error();
486 return 0;
487 }
488
489 #define uatomic_add_return(addr, v) \
490 ((__typeof__(*(addr))) _uatomic_add_return((addr), \
491 caa_cast_long_keep_sign(v), \
492 sizeof(*(addr))))
493 #endif /* #ifndef uatomic_add_return */
494
495 #ifndef uatomic_xchg
496 /* xchg */
497
498 static inline __attribute__((always_inline))
499 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
500 {
501 switch (len) {
502 #ifdef UATOMIC_HAS_ATOMIC_BYTE
503 case 1:
504 {
505 unsigned char old, oldt;
506
507 oldt = uatomic_read((unsigned char *)addr);
508 do {
509 old = oldt;
510 oldt = uatomic_cmpxchg((unsigned char *)addr,
511 old, val);
512 } while (oldt != old);
513
514 return old;
515 }
516 #endif
517 #ifdef UATOMIC_HAS_ATOMIC_SHORT
518 case 2:
519 {
520 unsigned short old, oldt;
521
522 oldt = uatomic_read((unsigned short *)addr);
523 do {
524 old = oldt;
525 oldt = uatomic_cmpxchg((unsigned short *)addr,
526 old, val);
527 } while (oldt != old);
528
529 return old;
530 }
531 #endif
532 case 4:
533 {
534 unsigned int old, oldt;
535
536 oldt = uatomic_read((unsigned int *)addr);
537 do {
538 old = oldt;
539 oldt = uatomic_cmpxchg((unsigned int *)addr,
540 old, val);
541 } while (oldt != old);
542
543 return old;
544 }
545 #if (CAA_BITS_PER_LONG == 64)
546 case 8:
547 {
548 unsigned long old, oldt;
549
550 oldt = uatomic_read((unsigned long *)addr);
551 do {
552 old = oldt;
553 oldt = uatomic_cmpxchg((unsigned long *)addr,
554 old, val);
555 } while (oldt != old);
556
557 return old;
558 }
559 #endif
560 }
561 _uatomic_link_error();
562 return 0;
563 }
564
565 #define uatomic_xchg(addr, v) \
566 ((__typeof__(*(addr))) _uatomic_exchange((addr), \
567 caa_cast_long_keep_sign(v), \
568 sizeof(*(addr))))
569 #endif /* #ifndef uatomic_xchg */
570
571 #endif /* #else #ifndef uatomic_cmpxchg */
572
573 /* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
574
575 #ifndef uatomic_add
576 #define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
577 #define cmm_smp_mb__before_uatomic_add() cmm_barrier()
578 #define cmm_smp_mb__after_uatomic_add() cmm_barrier()
579 #endif
580
581 #define uatomic_sub_return(addr, v) \
582 uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
583 #define uatomic_sub(addr, v) \
584 uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
585 #define cmm_smp_mb__before_uatomic_sub() cmm_smp_mb__before_uatomic_add()
586 #define cmm_smp_mb__after_uatomic_sub() cmm_smp_mb__after_uatomic_add()
587
588 #ifndef uatomic_inc
589 #define uatomic_inc(addr) uatomic_add((addr), 1)
590 #define cmm_smp_mb__before_uatomic_inc() cmm_smp_mb__before_uatomic_add()
591 #define cmm_smp_mb__after_uatomic_inc() cmm_smp_mb__after_uatomic_add()
592 #endif
593
594 #ifndef uatomic_dec
595 #define uatomic_dec(addr) uatomic_add((addr), -1)
596 #define cmm_smp_mb__before_uatomic_dec() cmm_smp_mb__before_uatomic_add()
597 #define cmm_smp_mb__after_uatomic_dec() cmm_smp_mb__after_uatomic_add()
598 #endif
599
600 #ifdef __cplusplus
601 }
602 #endif
603
604 #endif /* _URCU_UATOMIC_GENERIC_H */
This page took 0.067638 seconds and 3 git commands to generate.