Static arch and uatomic headers
[urcu.git] / include / urcu / uatomic / generic.h
... / ...
CommitLineData
1#ifndef _URCU_UATOMIC_GENERIC_H
2#define _URCU_UATOMIC_GENERIC_H
3
4/*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 * Copyright (c) 2010 Paolo Bonzini
10 *
11 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
12 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
13 *
14 * Permission is hereby granted to use or copy this program
15 * for any purpose, provided the above notices are retained on all copies.
16 * Permission to modify the code and to distribute modified code is granted,
17 * provided the above notices are retained, and a notice that the code was
18 * modified is included with the above copyright notice.
19 *
20 * Code inspired from libuatomic_ops-1.2, inherited in part from the
21 * Boehm-Demers-Weiser conservative garbage collector.
22 */
23
24#include <stdint.h>
25#include <urcu/compiler.h>
26#include <urcu/system.h>
27
28#ifdef __cplusplus
29extern "C" {
30#endif
31
32#ifndef uatomic_set
33#define uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
34#endif
35
36#ifndef uatomic_read
37#define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
38#endif
39
40#if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR
41static inline __attribute__((always_inline))
42void _uatomic_link_error(void)
43{
44#ifdef ILLEGAL_INSTR
45 /*
46 * generate an illegal instruction. Cannot catch this with
47 * linker tricks when optimizations are disabled.
48 */
49 __asm__ __volatile__(ILLEGAL_INSTR);
50#else
51 __builtin_trap();
52#endif
53}
54
55#else /* #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
56extern void _uatomic_link_error(void);
57#endif /* #else #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
58
59/* cmpxchg */
60
61#ifndef uatomic_cmpxchg
62static inline __attribute__((always_inline))
63unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
64 unsigned long _new, int len)
65{
66 switch (len) {
67#ifdef UATOMIC_HAS_ATOMIC_BYTE
68 case 1:
69 return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
70 _new);
71#endif
72#ifdef UATOMIC_HAS_ATOMIC_SHORT
73 case 2:
74 return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
75 _new);
76#endif
77 case 4:
78 return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
79 _new);
80#if (CAA_BITS_PER_LONG == 64)
81 case 8:
82 return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
83 _new);
84#endif
85 }
86 _uatomic_link_error();
87 return 0;
88}
89
90
91#define uatomic_cmpxchg(addr, old, _new) \
92 ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
93 caa_cast_long_keep_sign(old), \
94 caa_cast_long_keep_sign(_new),\
95 sizeof(*(addr))))
96
97
98/* uatomic_and */
99
100#ifndef uatomic_and
101static inline __attribute__((always_inline))
102void _uatomic_and(void *addr, unsigned long val,
103 int len)
104{
105 switch (len) {
106#ifdef UATOMIC_HAS_ATOMIC_BYTE
107 case 1:
108 __sync_and_and_fetch_1((uint8_t *) addr, val);
109 return;
110#endif
111#ifdef UATOMIC_HAS_ATOMIC_SHORT
112 case 2:
113 __sync_and_and_fetch_2((uint16_t *) addr, val);
114 return;
115#endif
116 case 4:
117 __sync_and_and_fetch_4((uint32_t *) addr, val);
118 return;
119#if (CAA_BITS_PER_LONG == 64)
120 case 8:
121 __sync_and_and_fetch_8((uint64_t *) addr, val);
122 return;
123#endif
124 }
125 _uatomic_link_error();
126}
127
128#define uatomic_and(addr, v) \
129 (_uatomic_and((addr), \
130 caa_cast_long_keep_sign(v), \
131 sizeof(*(addr))))
132#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
133#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
134
135#endif
136
137/* uatomic_or */
138
139#ifndef uatomic_or
140static inline __attribute__((always_inline))
141void _uatomic_or(void *addr, unsigned long val,
142 int len)
143{
144 switch (len) {
145#ifdef UATOMIC_HAS_ATOMIC_BYTE
146 case 1:
147 __sync_or_and_fetch_1((uint8_t *) addr, val);
148 return;
149#endif
150#ifdef UATOMIC_HAS_ATOMIC_SHORT
151 case 2:
152 __sync_or_and_fetch_2((uint16_t *) addr, val);
153 return;
154#endif
155 case 4:
156 __sync_or_and_fetch_4((uint32_t *) addr, val);
157 return;
158#if (CAA_BITS_PER_LONG == 64)
159 case 8:
160 __sync_or_and_fetch_8((uint64_t *) addr, val);
161 return;
162#endif
163 }
164 _uatomic_link_error();
165 return;
166}
167
168#define uatomic_or(addr, v) \
169 (_uatomic_or((addr), \
170 caa_cast_long_keep_sign(v), \
171 sizeof(*(addr))))
172#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
173#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
174
175#endif
176
177
178/* uatomic_add_return */
179
180#ifndef uatomic_add_return
181static inline __attribute__((always_inline))
182unsigned long _uatomic_add_return(void *addr, unsigned long val,
183 int len)
184{
185 switch (len) {
186#ifdef UATOMIC_HAS_ATOMIC_BYTE
187 case 1:
188 return __sync_add_and_fetch_1((uint8_t *) addr, val);
189#endif
190#ifdef UATOMIC_HAS_ATOMIC_SHORT
191 case 2:
192 return __sync_add_and_fetch_2((uint16_t *) addr, val);
193#endif
194 case 4:
195 return __sync_add_and_fetch_4((uint32_t *) addr, val);
196#if (CAA_BITS_PER_LONG == 64)
197 case 8:
198 return __sync_add_and_fetch_8((uint64_t *) addr, val);
199#endif
200 }
201 _uatomic_link_error();
202 return 0;
203}
204
205
206#define uatomic_add_return(addr, v) \
207 ((__typeof__(*(addr))) _uatomic_add_return((addr), \
208 caa_cast_long_keep_sign(v), \
209 sizeof(*(addr))))
210#endif /* #ifndef uatomic_add_return */
211
212#ifndef uatomic_xchg
213/* xchg */
214
215static inline __attribute__((always_inline))
216unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
217{
218 switch (len) {
219#ifdef UATOMIC_HAS_ATOMIC_BYTE
220 case 1:
221 {
222 uint8_t old;
223
224 do {
225 old = uatomic_read((uint8_t *) addr);
226 } while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
227 old, val));
228
229 return old;
230 }
231#endif
232#ifdef UATOMIC_HAS_ATOMIC_SHORT
233 case 2:
234 {
235 uint16_t old;
236
237 do {
238 old = uatomic_read((uint16_t *) addr);
239 } while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
240 old, val));
241
242 return old;
243 }
244#endif
245 case 4:
246 {
247 uint32_t old;
248
249 do {
250 old = uatomic_read((uint32_t *) addr);
251 } while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
252 old, val));
253
254 return old;
255 }
256#if (CAA_BITS_PER_LONG == 64)
257 case 8:
258 {
259 uint64_t old;
260
261 do {
262 old = uatomic_read((uint64_t *) addr);
263 } while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
264 old, val));
265
266 return old;
267 }
268#endif
269 }
270 _uatomic_link_error();
271 return 0;
272}
273
274#define uatomic_xchg(addr, v) \
275 ((__typeof__(*(addr))) _uatomic_exchange((addr), \
276 caa_cast_long_keep_sign(v), \
277 sizeof(*(addr))))
278#endif /* #ifndef uatomic_xchg */
279
280#else /* #ifndef uatomic_cmpxchg */
281
282#ifndef uatomic_and
283/* uatomic_and */
284
285static inline __attribute__((always_inline))
286void _uatomic_and(void *addr, unsigned long val, int len)
287{
288 switch (len) {
289#ifdef UATOMIC_HAS_ATOMIC_BYTE
290 case 1:
291 {
292 uint8_t old, oldt;
293
294 oldt = uatomic_read((uint8_t *) addr);
295 do {
296 old = oldt;
297 oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
298 } while (oldt != old);
299
300 return;
301 }
302#endif
303#ifdef UATOMIC_HAS_ATOMIC_SHORT
304 case 2:
305 {
306 uint16_t old, oldt;
307
308 oldt = uatomic_read((uint16_t *) addr);
309 do {
310 old = oldt;
311 oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
312 } while (oldt != old);
313 }
314#endif
315 case 4:
316 {
317 uint32_t old, oldt;
318
319 oldt = uatomic_read((uint32_t *) addr);
320 do {
321 old = oldt;
322 oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
323 } while (oldt != old);
324
325 return;
326 }
327#if (CAA_BITS_PER_LONG == 64)
328 case 8:
329 {
330 uint64_t old, oldt;
331
332 oldt = uatomic_read((uint64_t *) addr);
333 do {
334 old = oldt;
335 oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
336 } while (oldt != old);
337
338 return;
339 }
340#endif
341 }
342 _uatomic_link_error();
343}
344
345#define uatomic_and(addr, v) \
346 (_uatomic_and((addr), \
347 caa_cast_long_keep_sign(v), \
348 sizeof(*(addr))))
349#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
350#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
351
352#endif /* #ifndef uatomic_and */
353
354#ifndef uatomic_or
355/* uatomic_or */
356
357static inline __attribute__((always_inline))
358void _uatomic_or(void *addr, unsigned long val, int len)
359{
360 switch (len) {
361#ifdef UATOMIC_HAS_ATOMIC_BYTE
362 case 1:
363 {
364 uint8_t old, oldt;
365
366 oldt = uatomic_read((uint8_t *) addr);
367 do {
368 old = oldt;
369 oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
370 } while (oldt != old);
371
372 return;
373 }
374#endif
375#ifdef UATOMIC_HAS_ATOMIC_SHORT
376 case 2:
377 {
378 uint16_t old, oldt;
379
380 oldt = uatomic_read((uint16_t *) addr);
381 do {
382 old = oldt;
383 oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
384 } while (oldt != old);
385
386 return;
387 }
388#endif
389 case 4:
390 {
391 uint32_t old, oldt;
392
393 oldt = uatomic_read((uint32_t *) addr);
394 do {
395 old = oldt;
396 oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
397 } while (oldt != old);
398
399 return;
400 }
401#if (CAA_BITS_PER_LONG == 64)
402 case 8:
403 {
404 uint64_t old, oldt;
405
406 oldt = uatomic_read((uint64_t *) addr);
407 do {
408 old = oldt;
409 oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
410 } while (oldt != old);
411
412 return;
413 }
414#endif
415 }
416 _uatomic_link_error();
417}
418
419#define uatomic_or(addr, v) \
420 (_uatomic_or((addr), \
421 caa_cast_long_keep_sign(v), \
422 sizeof(*(addr))))
423#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
424#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
425
426#endif /* #ifndef uatomic_or */
427
428#ifndef uatomic_add_return
429/* uatomic_add_return */
430
431static inline __attribute__((always_inline))
432unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
433{
434 switch (len) {
435#ifdef UATOMIC_HAS_ATOMIC_BYTE
436 case 1:
437 {
438 uint8_t old, oldt;
439
440 oldt = uatomic_read((uint8_t *) addr);
441 do {
442 old = oldt;
443 oldt = uatomic_cmpxchg((uint8_t *) addr,
444 old, old + val);
445 } while (oldt != old);
446
447 return old + val;
448 }
449#endif
450#ifdef UATOMIC_HAS_ATOMIC_SHORT
451 case 2:
452 {
453 uint16_t old, oldt;
454
455 oldt = uatomic_read((uint16_t *) addr);
456 do {
457 old = oldt;
458 oldt = uatomic_cmpxchg((uint16_t *) addr,
459 old, old + val);
460 } while (oldt != old);
461
462 return old + val;
463 }
464#endif
465 case 4:
466 {
467 uint32_t old, oldt;
468
469 oldt = uatomic_read((uint32_t *) addr);
470 do {
471 old = oldt;
472 oldt = uatomic_cmpxchg((uint32_t *) addr,
473 old, old + val);
474 } while (oldt != old);
475
476 return old + val;
477 }
478#if (CAA_BITS_PER_LONG == 64)
479 case 8:
480 {
481 uint64_t old, oldt;
482
483 oldt = uatomic_read((uint64_t *) addr);
484 do {
485 old = oldt;
486 oldt = uatomic_cmpxchg((uint64_t *) addr,
487 old, old + val);
488 } while (oldt != old);
489
490 return old + val;
491 }
492#endif
493 }
494 _uatomic_link_error();
495 return 0;
496}
497
498#define uatomic_add_return(addr, v) \
499 ((__typeof__(*(addr))) _uatomic_add_return((addr), \
500 caa_cast_long_keep_sign(v), \
501 sizeof(*(addr))))
502#endif /* #ifndef uatomic_add_return */
503
504#ifndef uatomic_xchg
505/* xchg */
506
507static inline __attribute__((always_inline))
508unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
509{
510 switch (len) {
511#ifdef UATOMIC_HAS_ATOMIC_BYTE
512 case 1:
513 {
514 uint8_t old, oldt;
515
516 oldt = uatomic_read((uint8_t *) addr);
517 do {
518 old = oldt;
519 oldt = uatomic_cmpxchg((uint8_t *) addr,
520 old, val);
521 } while (oldt != old);
522
523 return old;
524 }
525#endif
526#ifdef UATOMIC_HAS_ATOMIC_SHORT
527 case 2:
528 {
529 uint16_t old, oldt;
530
531 oldt = uatomic_read((uint16_t *) addr);
532 do {
533 old = oldt;
534 oldt = uatomic_cmpxchg((uint16_t *) addr,
535 old, val);
536 } while (oldt != old);
537
538 return old;
539 }
540#endif
541 case 4:
542 {
543 uint32_t old, oldt;
544
545 oldt = uatomic_read((uint32_t *) addr);
546 do {
547 old = oldt;
548 oldt = uatomic_cmpxchg((uint32_t *) addr,
549 old, val);
550 } while (oldt != old);
551
552 return old;
553 }
554#if (CAA_BITS_PER_LONG == 64)
555 case 8:
556 {
557 uint64_t old, oldt;
558
559 oldt = uatomic_read((uint64_t *) addr);
560 do {
561 old = oldt;
562 oldt = uatomic_cmpxchg((uint64_t *) addr,
563 old, val);
564 } while (oldt != old);
565
566 return old;
567 }
568#endif
569 }
570 _uatomic_link_error();
571 return 0;
572}
573
574#define uatomic_xchg(addr, v) \
575 ((__typeof__(*(addr))) _uatomic_exchange((addr), \
576 caa_cast_long_keep_sign(v), \
577 sizeof(*(addr))))
578#endif /* #ifndef uatomic_xchg */
579
580#endif /* #else #ifndef uatomic_cmpxchg */
581
582/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
583
584#ifndef uatomic_add
585#define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
586#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
587#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
588#endif
589
590#define uatomic_sub_return(addr, v) \
591 uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
592#define uatomic_sub(addr, v) \
593 uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
594#define cmm_smp_mb__before_uatomic_sub() cmm_smp_mb__before_uatomic_add()
595#define cmm_smp_mb__after_uatomic_sub() cmm_smp_mb__after_uatomic_add()
596
597#ifndef uatomic_inc
598#define uatomic_inc(addr) uatomic_add((addr), 1)
599#define cmm_smp_mb__before_uatomic_inc() cmm_smp_mb__before_uatomic_add()
600#define cmm_smp_mb__after_uatomic_inc() cmm_smp_mb__after_uatomic_add()
601#endif
602
603#ifndef uatomic_dec
604#define uatomic_dec(addr) uatomic_add((addr), -1)
605#define cmm_smp_mb__before_uatomic_dec() cmm_smp_mb__before_uatomic_add()
606#define cmm_smp_mb__after_uatomic_dec() cmm_smp_mb__after_uatomic_add()
607#endif
608
609#ifdef __cplusplus
610}
611#endif
612
613#endif /* _URCU_UATOMIC_GENERIC_H */
This page took 0.024906 seconds and 4 git commands to generate.