rculfhash: stress requirement in documentation
[urcu.git] / urcu / uatomic / generic.h
CommitLineData
8760d94e
PB
1#ifndef _URCU_UATOMIC_GENERIC_H
2#define _URCU_UATOMIC_GENERIC_H
3
4/*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 * Copyright (c) 2010 Paolo Bonzini
10 *
11 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
12 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
13 *
14 * Permission is hereby granted to use or copy this program
15 * for any purpose, provided the above notices are retained on all copies.
16 * Permission to modify the code and to distribute modified code is granted,
17 * provided the above notices are retained, and a notice that the code was
18 * modified is included with the above copyright notice.
19 *
20 * Code inspired from libuatomic_ops-1.2, inherited in part from the
21 * Boehm-Demers-Weiser conservative garbage collector.
22 */
23
24#include <urcu/compiler.h>
25#include <urcu/system.h>
26
27#ifdef __cplusplus
28extern "C" {
29#endif
30
8760d94e 31#ifndef uatomic_set
6cf3827c 32#define uatomic_set(addr, v) CMM_STORE_SHARED(*(addr), (v))
8760d94e
PB
33#endif
34
35#ifndef uatomic_read
6cf3827c 36#define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
8760d94e
PB
37#endif
38
39#if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR
40static inline __attribute__((always_inline))
41void _uatomic_link_error()
42{
43#ifdef ILLEGAL_INSTR
44 /* generate an illegal instruction. Cannot catch this with linker tricks
45 * when optimizations are disabled. */
46 __asm__ __volatile__(ILLEGAL_INSTR);
47#else
48 __builtin_trap ();
49#endif
50}
51
52#else /* #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
53extern void _uatomic_link_error ();
54#endif /* #else #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
55
56/* cmpxchg */
57
58#ifndef uatomic_cmpxchg
59static inline __attribute__((always_inline))
60unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
61 unsigned long _new, int len)
62{
63 switch (len) {
f469d839
PB
64#ifdef UATOMIC_HAS_ATOMIC_BYTE
65 case 1:
66 return __sync_val_compare_and_swap_1(addr, old, _new);
67#endif
68#ifdef UATOMIC_HAS_ATOMIC_SHORT
69 case 2:
70 return __sync_val_compare_and_swap_2(addr, old, _new);
71#endif
8760d94e
PB
72 case 4:
73 return __sync_val_compare_and_swap_4(addr, old, _new);
b39e1761 74#if (CAA_BITS_PER_LONG == 64)
8760d94e
PB
75 case 8:
76 return __sync_val_compare_and_swap_8(addr, old, _new);
77#endif
78 }
79 _uatomic_link_error();
80 return 0;
81}
82
83
e56d99bf
MD
84#define uatomic_cmpxchg(addr, old, _new) \
85 ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
86 caa_cast_long_keep_sign(old), \
87 caa_cast_long_keep_sign(_new),\
8760d94e
PB
88 sizeof(*(addr))))
89
90
bf33aaea
PB
91/* uatomic_and */
92
93#ifndef uatomic_and
94static inline __attribute__((always_inline))
95void _uatomic_and(void *addr, unsigned long val,
96 int len)
97{
98 switch (len) {
99#ifdef UATOMIC_HAS_ATOMIC_BYTE
100 case 1:
101 __sync_and_and_fetch_1(addr, val);
c51d5c6e 102 return;
bf33aaea
PB
103#endif
104#ifdef UATOMIC_HAS_ATOMIC_SHORT
105 case 2:
106 __sync_and_and_fetch_2(addr, val);
c51d5c6e 107 return;
bf33aaea
PB
108#endif
109 case 4:
110 __sync_and_and_fetch_4(addr, val);
c51d5c6e 111 return;
bf33aaea
PB
112#if (CAA_BITS_PER_LONG == 64)
113 case 8:
114 __sync_and_and_fetch_8(addr, val);
c51d5c6e 115 return;
bf33aaea
PB
116#endif
117 }
118 _uatomic_link_error();
bf33aaea
PB
119}
120
121#define uatomic_and(addr, v) \
122 (_uatomic_and((addr), \
e56d99bf
MD
123 caa_cast_long_keep_sign(v), \
124 sizeof(*(addr))))
bf33aaea
PB
125#endif
126
985b35b1
PB
127/* uatomic_or */
128
129#ifndef uatomic_or
130static inline __attribute__((always_inline))
131void _uatomic_or(void *addr, unsigned long val,
132 int len)
133{
134 switch (len) {
135#ifdef UATOMIC_HAS_ATOMIC_BYTE
136 case 1:
137 __sync_or_and_fetch_1(addr, val);
c51d5c6e 138 return;
985b35b1
PB
139#endif
140#ifdef UATOMIC_HAS_ATOMIC_SHORT
141 case 2:
142 __sync_or_and_fetch_2(addr, val);
c51d5c6e 143 return;
985b35b1
PB
144#endif
145 case 4:
146 __sync_or_and_fetch_4(addr, val);
c51d5c6e 147 return;
985b35b1
PB
148#if (CAA_BITS_PER_LONG == 64)
149 case 8:
150 __sync_or_and_fetch_8(addr, val);
c51d5c6e 151 return;
985b35b1
PB
152#endif
153 }
154 _uatomic_link_error();
c51d5c6e 155 return;
985b35b1
PB
156}
157
158#define uatomic_or(addr, v) \
159 (_uatomic_or((addr), \
e56d99bf
MD
160 caa_cast_long_keep_sign(v), \
161 sizeof(*(addr))))
985b35b1
PB
162#endif
163
8760d94e
PB
164/* uatomic_add_return */
165
166#ifndef uatomic_add_return
167static inline __attribute__((always_inline))
168unsigned long _uatomic_add_return(void *addr, unsigned long val,
169 int len)
170{
171 switch (len) {
f469d839
PB
172#ifdef UATOMIC_HAS_ATOMIC_BYTE
173 case 1:
174 return __sync_add_and_fetch_1(addr, val);
175#endif
176#ifdef UATOMIC_HAS_ATOMIC_SHORT
177 case 2:
178 return __sync_add_and_fetch_2(addr, val);
179#endif
8760d94e
PB
180 case 4:
181 return __sync_add_and_fetch_4(addr, val);
b39e1761 182#if (CAA_BITS_PER_LONG == 64)
8760d94e
PB
183 case 8:
184 return __sync_add_and_fetch_8(addr, val);
185#endif
186 }
187 _uatomic_link_error();
188 return 0;
189}
190
191
e56d99bf
MD
192#define uatomic_add_return(addr, v) \
193 ((__typeof__(*(addr))) _uatomic_add_return((addr), \
194 caa_cast_long_keep_sign(v), \
195 sizeof(*(addr))))
8760d94e
PB
196#endif /* #ifndef uatomic_add_return */
197
198#ifndef uatomic_xchg
199/* xchg */
200
201static inline __attribute__((always_inline))
202unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
203{
204 switch (len) {
f469d839
PB
205#ifdef UATOMIC_HAS_ATOMIC_BYTE
206 case 1:
207 {
208 unsigned char old;
209
210 do {
211 old = uatomic_read((unsigned char *)addr);
212 } while (!__sync_bool_compare_and_swap_1(addr, old, val));
213
214 return old;
215 }
216#endif
217#ifdef UATOMIC_HAS_ATOMIC_SHORT
218 case 2:
219 {
220 unsigned short old;
221
222 do {
223 old = uatomic_read((unsigned short *)addr);
224 } while (!__sync_bool_compare_and_swap_2(addr, old, val));
225
226 return old;
227 }
228#endif
8760d94e
PB
229 case 4:
230 {
231 unsigned int old;
232
233 do {
234 old = uatomic_read((unsigned int *)addr);
2f2908d0 235 } while (!__sync_bool_compare_and_swap_4(addr, old, val));
8760d94e 236
2f2908d0 237 return old;
8760d94e 238 }
b39e1761 239#if (CAA_BITS_PER_LONG == 64)
8760d94e
PB
240 case 8:
241 {
242 unsigned long old;
243
244 do {
245 old = uatomic_read((unsigned long *)addr);
246 } while (!__sync_bool_compare_and_swap_8(addr, old, val));
247
248 return old;
249 }
250#endif
251 }
252 _uatomic_link_error();
253 return 0;
254}
255
256#define uatomic_xchg(addr, v) \
e56d99bf
MD
257 ((__typeof__(*(addr))) _uatomic_exchange((addr), \
258 caa_cast_long_keep_sign(v), \
8760d94e
PB
259 sizeof(*(addr))))
260#endif /* #ifndef uatomic_xchg */
261
262#else /* #ifndef uatomic_cmpxchg */
263
bf33aaea
PB
264#ifndef uatomic_and
265/* uatomic_and */
266
267static inline __attribute__((always_inline))
268void _uatomic_and(void *addr, unsigned long val, int len)
269{
270 switch (len) {
271#ifdef UATOMIC_HAS_ATOMIC_BYTE
272 case 1:
273 {
274 unsigned char old, oldt;
275
276 oldt = uatomic_read((unsigned char *)addr);
277 do {
278 old = oldt;
279 oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
280 } while (oldt != old);
e6e5957d
MD
281
282 return;
bf33aaea
PB
283 }
284#endif
285#ifdef UATOMIC_HAS_ATOMIC_SHORT
286 case 2:
287 {
288 unsigned short old, oldt;
289
290 oldt = uatomic_read((unsigned short *)addr);
291 do {
292 old = oldt;
293 oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
294 } while (oldt != old);
295 }
296#endif
297 case 4:
298 {
299 unsigned int old, oldt;
300
301 oldt = uatomic_read((unsigned int *)addr);
302 do {
303 old = oldt;
304 oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
305 } while (oldt != old);
e6e5957d
MD
306
307 return;
bf33aaea
PB
308 }
309#if (CAA_BITS_PER_LONG == 64)
310 case 8:
311 {
312 unsigned long old, oldt;
313
314 oldt = uatomic_read((unsigned long *)addr);
315 do {
316 old = oldt;
317 oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
318 } while (oldt != old);
e6e5957d
MD
319
320 return;
bf33aaea
PB
321 }
322#endif
323 }
324 _uatomic_link_error();
bf33aaea
PB
325}
326
e56d99bf
MD
327#define uatomic_and(addr, v) \
328 (_uatomic_and((addr), \
329 caa_cast_long_keep_sign(v), \
330 sizeof(*(addr))))
bf33aaea
PB
331#endif /* #ifndef uatomic_and */
332
985b35b1
PB
333#ifndef uatomic_or
334/* uatomic_or */
335
336static inline __attribute__((always_inline))
337void _uatomic_or(void *addr, unsigned long val, int len)
338{
339 switch (len) {
340#ifdef UATOMIC_HAS_ATOMIC_BYTE
341 case 1:
342 {
343 unsigned char old, oldt;
344
345 oldt = uatomic_read((unsigned char *)addr);
346 do {
347 old = oldt;
348 oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
349 } while (oldt != old);
e6e5957d
MD
350
351 return;
985b35b1
PB
352 }
353#endif
354#ifdef UATOMIC_HAS_ATOMIC_SHORT
355 case 2:
356 {
357 unsigned short old, oldt;
358
359 oldt = uatomic_read((unsigned short *)addr);
360 do {
361 old = oldt;
362 oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
363 } while (oldt != old);
e6e5957d
MD
364
365 return;
985b35b1
PB
366 }
367#endif
368 case 4:
369 {
370 unsigned int old, oldt;
371
372 oldt = uatomic_read((unsigned int *)addr);
373 do {
374 old = oldt;
375 oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
376 } while (oldt != old);
e6e5957d
MD
377
378 return;
985b35b1
PB
379 }
380#if (CAA_BITS_PER_LONG == 64)
381 case 8:
382 {
383 unsigned long old, oldt;
384
385 oldt = uatomic_read((unsigned long *)addr);
386 do {
387 old = oldt;
388 oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
389 } while (oldt != old);
e6e5957d
MD
390
391 return;
985b35b1
PB
392 }
393#endif
394 }
395 _uatomic_link_error();
985b35b1
PB
396}
397
e56d99bf
MD
398#define uatomic_or(addr, v) \
399 (_uatomic_or((addr), \
400 caa_cast_long_keep_sign(v), \
401 sizeof(*(addr))))
985b35b1
PB
402#endif /* #ifndef uatomic_or */
403
8760d94e
PB
404#ifndef uatomic_add_return
405/* uatomic_add_return */
406
407static inline __attribute__((always_inline))
408unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
409{
410 switch (len) {
f469d839
PB
411#ifdef UATOMIC_HAS_ATOMIC_BYTE
412 case 1:
413 {
414 unsigned char old, oldt;
415
416 oldt = uatomic_read((unsigned char *)addr);
417 do {
418 old = oldt;
b4e6d540
PB
419 oldt = uatomic_cmpxchg((unsigned char *)addr,
420 old, old + val);
f469d839
PB
421 } while (oldt != old);
422
423 return old + val;
424 }
425#endif
426#ifdef UATOMIC_HAS_ATOMIC_SHORT
427 case 2:
428 {
429 unsigned short old, oldt;
430
431 oldt = uatomic_read((unsigned short *)addr);
432 do {
433 old = oldt;
b4e6d540
PB
434 oldt = uatomic_cmpxchg((unsigned short *)addr,
435 old, old + val);
f469d839
PB
436 } while (oldt != old);
437
438 return old + val;
439 }
440#endif
8760d94e
PB
441 case 4:
442 {
443 unsigned int old, oldt;
444
445 oldt = uatomic_read((unsigned int *)addr);
446 do {
447 old = oldt;
b4e6d540
PB
448 oldt = uatomic_cmpxchg((unsigned int *)addr,
449 old, old + val);
8760d94e
PB
450 } while (oldt != old);
451
452 return old + val;
453 }
b39e1761 454#if (CAA_BITS_PER_LONG == 64)
8760d94e
PB
455 case 8:
456 {
457 unsigned long old, oldt;
458
459 oldt = uatomic_read((unsigned long *)addr);
460 do {
461 old = oldt;
b4e6d540
PB
462 oldt = uatomic_cmpxchg((unsigned long *)addr,
463 old, old + val);
8760d94e
PB
464 } while (oldt != old);
465
466 return old + val;
467 }
468#endif
469 }
470 _uatomic_link_error();
471 return 0;
472}
473
e56d99bf
MD
474#define uatomic_add_return(addr, v) \
475 ((__typeof__(*(addr))) _uatomic_add_return((addr), \
476 caa_cast_long_keep_sign(v), \
477 sizeof(*(addr))))
8760d94e
PB
478#endif /* #ifndef uatomic_add_return */
479
480#ifndef uatomic_xchg
481/* xchg */
482
483static inline __attribute__((always_inline))
484unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
485{
486 switch (len) {
f469d839
PB
487#ifdef UATOMIC_HAS_ATOMIC_BYTE
488 case 1:
489 {
490 unsigned char old, oldt;
491
492 oldt = uatomic_read((unsigned char *)addr);
493 do {
494 old = oldt;
b4e6d540
PB
495 oldt = uatomic_cmpxchg((unsigned char *)addr,
496 old, val);
f469d839
PB
497 } while (oldt != old);
498
499 return old;
500 }
501#endif
502#ifdef UATOMIC_HAS_ATOMIC_SHORT
503 case 2:
504 {
505 unsigned short old, oldt;
506
507 oldt = uatomic_read((unsigned short *)addr);
508 do {
509 old = oldt;
b4e6d540
PB
510 oldt = uatomic_cmpxchg((unsigned short *)addr,
511 old, val);
f469d839
PB
512 } while (oldt != old);
513
514 return old;
515 }
516#endif
8760d94e
PB
517 case 4:
518 {
519 unsigned int old, oldt;
520
521 oldt = uatomic_read((unsigned int *)addr);
522 do {
523 old = oldt;
b4e6d540
PB
524 oldt = uatomic_cmpxchg((unsigned int *)addr,
525 old, val);
8760d94e
PB
526 } while (oldt != old);
527
528 return old;
529 }
b39e1761 530#if (CAA_BITS_PER_LONG == 64)
8760d94e
PB
531 case 8:
532 {
533 unsigned long old, oldt;
534
535 oldt = uatomic_read((unsigned long *)addr);
536 do {
537 old = oldt;
b4e6d540
PB
538 oldt = uatomic_cmpxchg((unsigned long *)addr,
539 old, val);
8760d94e
PB
540 } while (oldt != old);
541
542 return old;
543 }
544#endif
545 }
546 _uatomic_link_error();
547 return 0;
548}
549
550#define uatomic_xchg(addr, v) \
e56d99bf
MD
551 ((__typeof__(*(addr))) _uatomic_exchange((addr), \
552 caa_cast_long_keep_sign(v), \
8760d94e
PB
553 sizeof(*(addr))))
554#endif /* #ifndef uatomic_xchg */
555
556#endif /* #else #ifndef uatomic_cmpxchg */
557
558/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
559
560#ifndef uatomic_add
561#define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
562#endif
563
e56d99bf
MD
564#define uatomic_sub_return(addr, v) \
565 uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
566#define uatomic_sub(addr, v) \
567 uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
8760d94e
PB
568
569#ifndef uatomic_inc
570#define uatomic_inc(addr) uatomic_add((addr), 1)
571#endif
572
573#ifndef uatomic_dec
574#define uatomic_dec(addr) uatomic_add((addr), -1)
575#endif
576
577#ifdef __cplusplus
578}
579#endif
580
581#endif /* _URCU_UATOMIC_GENERIC_H */
This page took 0.048004 seconds and 4 git commands to generate.