uatomic: Specify complete types for atomic function calls
[urcu.git] / urcu / uatomic / generic.h
1 #ifndef _URCU_UATOMIC_GENERIC_H
2 #define _URCU_UATOMIC_GENERIC_H
3
4 /*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 * Copyright (c) 2010 Paolo Bonzini
10 *
11 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
12 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
13 *
14 * Permission is hereby granted to use or copy this program
15 * for any purpose, provided the above notices are retained on all copies.
16 * Permission to modify the code and to distribute modified code is granted,
17 * provided the above notices are retained, and a notice that the code was
18 * modified is included with the above copyright notice.
19 *
20 * Code inspired from libuatomic_ops-1.2, inherited in part from the
21 * Boehm-Demers-Weiser conservative garbage collector.
22 */
23
24 #include <stdint.h>
25 #include <urcu/compiler.h>
26 #include <urcu/system.h>
27
28 #ifdef __cplusplus
29 extern "C" {
30 #endif
31
32 #ifndef uatomic_set
33 #define uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
34 #endif
35
36 #ifndef uatomic_read
37 #define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
38 #endif
39
40 #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR
41 static inline __attribute__((always_inline))
42 void _uatomic_link_error(void)
43 {
44 #ifdef ILLEGAL_INSTR
45 /*
46 * generate an illegal instruction. Cannot catch this with
47 * linker tricks when optimizations are disabled.
48 */
49 __asm__ __volatile__(ILLEGAL_INSTR);
50 #else
51 __builtin_trap();
52 #endif
53 }
54
55 #else /* #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
56 extern void _uatomic_link_error(void);
57 #endif /* #else #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
58
59 /* cmpxchg */
60
61 #ifndef uatomic_cmpxchg
62 static inline __attribute__((always_inline))
63 unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
64 unsigned long _new, int len)
65 {
66 switch (len) {
67 #ifdef UATOMIC_HAS_ATOMIC_BYTE
68 case 1:
69 return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
70 _new);
71 #endif
72 #ifdef UATOMIC_HAS_ATOMIC_SHORT
73 case 2:
74 return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
75 _new);
76 #endif
77 case 4:
78 return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
79 _new);
80 #if (CAA_BITS_PER_LONG == 64)
81 case 8:
82 return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
83 _new);
84 #endif
85 }
86 _uatomic_link_error();
87 return 0;
88 }
89
90
91 #define uatomic_cmpxchg(addr, old, _new) \
92 ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
93 caa_cast_long_keep_sign(old), \
94 caa_cast_long_keep_sign(_new),\
95 sizeof(*(addr))))
96
97
98 /* uatomic_and */
99
100 #ifndef uatomic_and
101 static inline __attribute__((always_inline))
102 void _uatomic_and(void *addr, unsigned long val,
103 int len)
104 {
105 switch (len) {
106 #ifdef UATOMIC_HAS_ATOMIC_BYTE
107 case 1:
108 __sync_and_and_fetch_1((uint8_t *) addr, val);
109 return;
110 #endif
111 #ifdef UATOMIC_HAS_ATOMIC_SHORT
112 case 2:
113 __sync_and_and_fetch_2((uint16_t *) addr, val);
114 return;
115 #endif
116 case 4:
117 __sync_and_and_fetch_4((uint32_t *) addr, val);
118 return;
119 #if (CAA_BITS_PER_LONG == 64)
120 case 8:
121 __sync_and_and_fetch_8((uint64_t *) addr, val);
122 return;
123 #endif
124 }
125 _uatomic_link_error();
126 }
127
128 #define uatomic_and(addr, v) \
129 (_uatomic_and((addr), \
130 caa_cast_long_keep_sign(v), \
131 sizeof(*(addr))))
132 #define cmm_smp_mb__before_uatomic_and() cmm_barrier()
133 #define cmm_smp_mb__after_uatomic_and() cmm_barrier()
134
135 #endif
136
137 /* uatomic_or */
138
139 #ifndef uatomic_or
140 static inline __attribute__((always_inline))
141 void _uatomic_or(void *addr, unsigned long val,
142 int len)
143 {
144 switch (len) {
145 #ifdef UATOMIC_HAS_ATOMIC_BYTE
146 case 1:
147 __sync_or_and_fetch_1((uint8_t *) addr, val);
148 return;
149 #endif
150 #ifdef UATOMIC_HAS_ATOMIC_SHORT
151 case 2:
152 __sync_or_and_fetch_2((uint16_t *) addr, val);
153 return;
154 #endif
155 case 4:
156 __sync_or_and_fetch_4((uint32_t *) addr, val);
157 return;
158 #if (CAA_BITS_PER_LONG == 64)
159 case 8:
160 __sync_or_and_fetch_8((uint64_t *) addr, val);
161 return;
162 #endif
163 }
164 _uatomic_link_error();
165 return;
166 }
167
168 #define uatomic_or(addr, v) \
169 (_uatomic_or((addr), \
170 caa_cast_long_keep_sign(v), \
171 sizeof(*(addr))))
172 #define cmm_smp_mb__before_uatomic_or() cmm_barrier()
173 #define cmm_smp_mb__after_uatomic_or() cmm_barrier()
174
175 #endif
176
177
178 /* uatomic_add_return */
179
180 #ifndef uatomic_add_return
181 static inline __attribute__((always_inline))
182 unsigned long _uatomic_add_return(void *addr, unsigned long val,
183 int len)
184 {
185 switch (len) {
186 #ifdef UATOMIC_HAS_ATOMIC_BYTE
187 case 1:
188 return __sync_add_and_fetch_1((uint8_t *) addr, val);
189 #endif
190 #ifdef UATOMIC_HAS_ATOMIC_SHORT
191 case 2:
192 return __sync_add_and_fetch_2((uint16_t *) addr, val);
193 #endif
194 case 4:
195 return __sync_add_and_fetch_4((uint32_t *) addr, val);
196 #if (CAA_BITS_PER_LONG == 64)
197 case 8:
198 return __sync_add_and_fetch_8((uint64_t *) addr, val);
199 #endif
200 }
201 _uatomic_link_error();
202 return 0;
203 }
204
205
206 #define uatomic_add_return(addr, v) \
207 ((__typeof__(*(addr))) _uatomic_add_return((addr), \
208 caa_cast_long_keep_sign(v), \
209 sizeof(*(addr))))
210 #endif /* #ifndef uatomic_add_return */
211
212 #ifndef uatomic_xchg
213 /* xchg */
214
215 static inline __attribute__((always_inline))
216 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
217 {
218 switch (len) {
219 #ifdef UATOMIC_HAS_ATOMIC_BYTE
220 case 1:
221 {
222 uint8_t old;
223
224 do {
225 old = uatomic_read((uint8_t *) addr);
226 } while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
227 old, val));
228
229 return old;
230 }
231 #endif
232 #ifdef UATOMIC_HAS_ATOMIC_SHORT
233 case 2:
234 {
235 uint16_t old;
236
237 do {
238 old = uatomic_read((uint16_t *) addr);
239 } while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
240 old, val));
241
242 return old;
243 }
244 #endif
245 case 4:
246 {
247 uint32_t old;
248
249 do {
250 old = uatomic_read((uint32_t *) addr);
251 } while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
252 old, val));
253
254 return old;
255 }
256 #if (CAA_BITS_PER_LONG == 64)
257 case 8:
258 {
259 uint64_t old;
260
261 do {
262 old = uatomic_read((uint64_t *) addr);
263 } while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
264 old, val));
265
266 return old;
267 }
268 #endif
269 }
270 _uatomic_link_error();
271 return 0;
272 }
273
274 #define uatomic_xchg(addr, v) \
275 ((__typeof__(*(addr))) _uatomic_exchange((addr), \
276 caa_cast_long_keep_sign(v), \
277 sizeof(*(addr))))
278 #endif /* #ifndef uatomic_xchg */
279
280 #else /* #ifndef uatomic_cmpxchg */
281
282 #ifndef uatomic_and
283 /* uatomic_and */
284
285 static inline __attribute__((always_inline))
286 void _uatomic_and(void *addr, unsigned long val, int len)
287 {
288 switch (len) {
289 #ifdef UATOMIC_HAS_ATOMIC_BYTE
290 case 1:
291 {
292 uint8_t old, oldt;
293
294 oldt = uatomic_read((uint8_t *) addr);
295 do {
296 old = oldt;
297 oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
298 } while (oldt != old);
299
300 return;
301 }
302 #endif
303 #ifdef UATOMIC_HAS_ATOMIC_SHORT
304 case 2:
305 {
306 uint16_t old, oldt;
307
308 oldt = uatomic_read((uint16_t *) addr);
309 do {
310 old = oldt;
311 oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
312 } while (oldt != old);
313 }
314 #endif
315 case 4:
316 {
317 uint32_t old, oldt;
318
319 oldt = uatomic_read((uint32_t *) addr);
320 do {
321 old = oldt;
322 oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
323 } while (oldt != old);
324
325 return;
326 }
327 #if (CAA_BITS_PER_LONG == 64)
328 case 8:
329 {
330 uint64_t old, oldt;
331
332 oldt = uatomic_read((uint64_t *) addr);
333 do {
334 old = oldt;
335 oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
336 } while (oldt != old);
337
338 return;
339 }
340 #endif
341 }
342 _uatomic_link_error();
343 }
344
345 #define uatomic_and(addr, v) \
346 (_uatomic_and((addr), \
347 caa_cast_long_keep_sign(v), \
348 sizeof(*(addr))))
349 #define cmm_smp_mb__before_uatomic_and() cmm_barrier()
350 #define cmm_smp_mb__after_uatomic_and() cmm_barrier()
351
352 #endif /* #ifndef uatomic_and */
353
354 #ifndef uatomic_or
355 /* uatomic_or */
356
357 static inline __attribute__((always_inline))
358 void _uatomic_or(void *addr, unsigned long val, int len)
359 {
360 switch (len) {
361 #ifdef UATOMIC_HAS_ATOMIC_BYTE
362 case 1:
363 {
364 uint8_t old, oldt;
365
366 oldt = uatomic_read((uint8_t *) addr);
367 do {
368 old = oldt;
369 oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
370 } while (oldt != old);
371
372 return;
373 }
374 #endif
375 #ifdef UATOMIC_HAS_ATOMIC_SHORT
376 case 2:
377 {
378 uint16_t old, oldt;
379
380 oldt = uatomic_read((uint16_t *) addr);
381 do {
382 old = oldt;
383 oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
384 } while (oldt != old);
385
386 return;
387 }
388 #endif
389 case 4:
390 {
391 uint32_t old, oldt;
392
393 oldt = uatomic_read((uint32_t *) addr);
394 do {
395 old = oldt;
396 oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
397 } while (oldt != old);
398
399 return;
400 }
401 #if (CAA_BITS_PER_LONG == 64)
402 case 8:
403 {
404 uint64_t old, oldt;
405
406 oldt = uatomic_read((uint64_t *) addr);
407 do {
408 old = oldt;
409 oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
410 } while (oldt != old);
411
412 return;
413 }
414 #endif
415 }
416 _uatomic_link_error();
417 }
418
419 #define uatomic_or(addr, v) \
420 (_uatomic_or((addr), \
421 caa_cast_long_keep_sign(v), \
422 sizeof(*(addr))))
423 #define cmm_smp_mb__before_uatomic_or() cmm_barrier()
424 #define cmm_smp_mb__after_uatomic_or() cmm_barrier()
425
426 #endif /* #ifndef uatomic_or */
427
428 #ifndef uatomic_add_return
429 /* uatomic_add_return */
430
431 static inline __attribute__((always_inline))
432 unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
433 {
434 switch (len) {
435 #ifdef UATOMIC_HAS_ATOMIC_BYTE
436 case 1:
437 {
438 uint8_t old, oldt;
439
440 oldt = uatomic_read((uint8_t *) addr);
441 do {
442 old = oldt;
443 oldt = uatomic_cmpxchg((uint8_t *) addr,
444 old, old + val);
445 } while (oldt != old);
446
447 return old + val;
448 }
449 #endif
450 #ifdef UATOMIC_HAS_ATOMIC_SHORT
451 case 2:
452 {
453 uint16_t old, oldt;
454
455 oldt = uatomic_read((uint16_t *) addr);
456 do {
457 old = oldt;
458 oldt = uatomic_cmpxchg((uint16_t *) addr,
459 old, old + val);
460 } while (oldt != old);
461
462 return old + val;
463 }
464 #endif
465 case 4:
466 {
467 uint32_t old, oldt;
468
469 oldt = uatomic_read((uint32_t *) addr);
470 do {
471 old = oldt;
472 oldt = uatomic_cmpxchg((uint32_t *) addr,
473 old, old + val);
474 } while (oldt != old);
475
476 return old + val;
477 }
478 #if (CAA_BITS_PER_LONG == 64)
479 case 8:
480 {
481 uint64_t old, oldt;
482
483 oldt = uatomic_read((uint64_t *) addr);
484 do {
485 old = oldt;
486 oldt = uatomic_cmpxchg((uint64_t *) addr,
487 old, old + val);
488 } while (oldt != old);
489
490 return old + val;
491 }
492 #endif
493 }
494 _uatomic_link_error();
495 return 0;
496 }
497
498 #define uatomic_add_return(addr, v) \
499 ((__typeof__(*(addr))) _uatomic_add_return((addr), \
500 caa_cast_long_keep_sign(v), \
501 sizeof(*(addr))))
502 #endif /* #ifndef uatomic_add_return */
503
504 #ifndef uatomic_xchg
505 /* xchg */
506
507 static inline __attribute__((always_inline))
508 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
509 {
510 switch (len) {
511 #ifdef UATOMIC_HAS_ATOMIC_BYTE
512 case 1:
513 {
514 uint8_t old, oldt;
515
516 oldt = uatomic_read((uint8_t *) addr);
517 do {
518 old = oldt;
519 oldt = uatomic_cmpxchg((uint8_t *) addr,
520 old, val);
521 } while (oldt != old);
522
523 return old;
524 }
525 #endif
526 #ifdef UATOMIC_HAS_ATOMIC_SHORT
527 case 2:
528 {
529 uint16_t old, oldt;
530
531 oldt = uatomic_read((uint16_t *) addr);
532 do {
533 old = oldt;
534 oldt = uatomic_cmpxchg((uint16_t *) addr,
535 old, val);
536 } while (oldt != old);
537
538 return old;
539 }
540 #endif
541 case 4:
542 {
543 uint32_t old, oldt;
544
545 oldt = uatomic_read((uint32_t *) addr);
546 do {
547 old = oldt;
548 oldt = uatomic_cmpxchg((uint32_t *) addr,
549 old, val);
550 } while (oldt != old);
551
552 return old;
553 }
554 #if (CAA_BITS_PER_LONG == 64)
555 case 8:
556 {
557 uint64_t old, oldt;
558
559 oldt = uatomic_read((uint64_t *) addr);
560 do {
561 old = oldt;
562 oldt = uatomic_cmpxchg((uint64_t *) addr,
563 old, val);
564 } while (oldt != old);
565
566 return old;
567 }
568 #endif
569 }
570 _uatomic_link_error();
571 return 0;
572 }
573
574 #define uatomic_xchg(addr, v) \
575 ((__typeof__(*(addr))) _uatomic_exchange((addr), \
576 caa_cast_long_keep_sign(v), \
577 sizeof(*(addr))))
578 #endif /* #ifndef uatomic_xchg */
579
580 #endif /* #else #ifndef uatomic_cmpxchg */
581
582 /* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
583
584 #ifndef uatomic_add
585 #define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
586 #define cmm_smp_mb__before_uatomic_add() cmm_barrier()
587 #define cmm_smp_mb__after_uatomic_add() cmm_barrier()
588 #endif
589
590 #define uatomic_sub_return(addr, v) \
591 uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
592 #define uatomic_sub(addr, v) \
593 uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
594 #define cmm_smp_mb__before_uatomic_sub() cmm_smp_mb__before_uatomic_add()
595 #define cmm_smp_mb__after_uatomic_sub() cmm_smp_mb__after_uatomic_add()
596
597 #ifndef uatomic_inc
598 #define uatomic_inc(addr) uatomic_add((addr), 1)
599 #define cmm_smp_mb__before_uatomic_inc() cmm_smp_mb__before_uatomic_add()
600 #define cmm_smp_mb__after_uatomic_inc() cmm_smp_mb__after_uatomic_add()
601 #endif
602
603 #ifndef uatomic_dec
604 #define uatomic_dec(addr) uatomic_add((addr), -1)
605 #define cmm_smp_mb__before_uatomic_dec() cmm_smp_mb__before_uatomic_add()
606 #define cmm_smp_mb__after_uatomic_dec() cmm_smp_mb__after_uatomic_add()
607 #endif
608
609 #ifdef __cplusplus
610 }
611 #endif
612
613 #endif /* _URCU_UATOMIC_GENERIC_H */
This page took 0.068084 seconds and 4 git commands to generate.