Commit | Line | Data |
---|---|---|
ec4e58a3 MD |
1 | #ifndef _URCU_ARCH_UATOMIC_X86_H |
2 | #define _URCU_ARCH_UATOMIC_X86_H | |
0114ba7f | 3 | |
67ecffc0 | 4 | /* |
0114ba7f MD |
5 | * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. |
6 | * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. | |
7 | * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P. | |
8 | * Copyright (c) 2009 Mathieu Desnoyers | |
9 | * | |
10 | * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED | |
11 | * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. | |
12 | * | |
13 | * Permission is hereby granted to use or copy this program | |
14 | * for any purpose, provided the above notices are retained on all copies. | |
15 | * Permission to modify the code and to distribute modified code is granted, | |
16 | * provided the above notices are retained, and a notice that the code was | |
17 | * modified is included with the above copyright notice. | |
18 | * | |
ec4e58a3 | 19 | * Code inspired from libuatomic_ops-1.2, inherited in part from the |
0114ba7f MD |
20 | * Boehm-Demers-Weiser conservative garbage collector. |
21 | */ | |
22 | ||
0b1e236d | 23 | #include <urcu/arch.h> |
375db287 | 24 | #include <urcu/config.h> |
ec4e58a3 | 25 | #include <urcu/compiler.h> |
bf9de1b7 | 26 | #include <urcu/system.h> |
0fad128b | 27 | |
f469d839 PB |
28 | #define UATOMIC_HAS_ATOMIC_BYTE |
29 | #define UATOMIC_HAS_ATOMIC_SHORT | |
30 | ||
36bc70a8 MD |
31 | #ifdef __cplusplus |
32 | extern "C" { | |
67ecffc0 | 33 | #endif |
36bc70a8 | 34 | |
0114ba7f | 35 | /* |
0114ba7f MD |
36 | * Derived from AO_compare_and_swap() and AO_test_and_set_full(). |
37 | */ | |
38 | ||
835b9ab3 | 39 | /* |
71323499 | 40 | * The __hp() macro casts the void pointer @x to a pointer to a structure |
835b9ab3 MD |
41 | * containing an array of char of the specified size. This allows passing the |
42 | * @addr arguments of the following inline functions as "m" and "+m" operands | |
71323499 MD |
43 | * to the assembly. The @size parameter should be a constant to support |
44 | * compilers such as clang which do not support VLA. | |
835b9ab3 MD |
45 | */ |
46 | ||
47 | #define __hp(size, x) ((struct { char v[size]; } *)(x)) | |
cc1be41b | 48 | |
424d4ed5 | 49 | #define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v))) |
0fad128b | 50 | |
cc1be41b MD |
51 | /* cmpxchg */ |
52 | ||
5dba80f9 | 53 | static inline __attribute__((always_inline)) |
bf9de1b7 | 54 | unsigned long __uatomic_cmpxchg(void *addr, unsigned long old, |
0fad128b | 55 | unsigned long _new, int len) |
0114ba7f | 56 | { |
cc1be41b MD |
57 | switch (len) { |
58 | case 1: | |
59 | { | |
60 | unsigned char result = old; | |
0fad128b | 61 | |
cc1be41b MD |
62 | __asm__ __volatile__( |
63 | "lock; cmpxchgb %2, %1" | |
71323499 | 64 | : "+a"(result), "+m"(*__hp(1, addr)) |
cc1be41b | 65 | : "q"((unsigned char)_new) |
0114ba7f | 66 | : "memory"); |
cc1be41b MD |
67 | return result; |
68 | } | |
69 | case 2: | |
70 | { | |
71 | unsigned short result = old; | |
0fad128b | 72 | |
cc1be41b MD |
73 | __asm__ __volatile__( |
74 | "lock; cmpxchgw %2, %1" | |
71323499 | 75 | : "+a"(result), "+m"(*__hp(2, addr)) |
cc1be41b MD |
76 | : "r"((unsigned short)_new) |
77 | : "memory"); | |
78 | return result; | |
79 | } | |
80 | case 4: | |
81 | { | |
82 | unsigned int result = old; | |
0fad128b | 83 | |
cc1be41b MD |
84 | __asm__ __volatile__( |
85 | "lock; cmpxchgl %2, %1" | |
71323499 | 86 | : "+a"(result), "+m"(*__hp(4, addr)) |
cc1be41b MD |
87 | : "r"((unsigned int)_new) |
88 | : "memory"); | |
89 | return result; | |
90 | } | |
e040d717 | 91 | #if (CAA_BITS_PER_LONG == 64) |
cc1be41b MD |
92 | case 8: |
93 | { | |
6edb297e | 94 | unsigned long result = old; |
0fad128b | 95 | |
cc1be41b | 96 | __asm__ __volatile__( |
2c5e5fb3 | 97 | "lock; cmpxchgq %2, %1" |
71323499 | 98 | : "+a"(result), "+m"(*__hp(8, addr)) |
cc1be41b MD |
99 | : "r"((unsigned long)_new) |
100 | : "memory"); | |
101 | return result; | |
102 | } | |
103 | #endif | |
104 | } | |
d0bbd9c2 MD |
105 | /* |
106 | * generate an illegal instruction. Cannot catch this with | |
107 | * linker tricks when optimizations are disabled. | |
108 | */ | |
cc1be41b MD |
109 | __asm__ __volatile__("ud2"); |
110 | return 0; | |
0114ba7f MD |
111 | } |
112 | ||
bf9de1b7 | 113 | #define _uatomic_cmpxchg(addr, old, _new) \ |
e56d99bf MD |
114 | ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \ |
115 | caa_cast_long_keep_sign(old), \ | |
116 | caa_cast_long_keep_sign(_new),\ | |
cc1be41b MD |
117 | sizeof(*(addr)))) |
118 | ||
119 | /* xchg */ | |
0114ba7f | 120 | |
5dba80f9 | 121 | static inline __attribute__((always_inline)) |
bf9de1b7 | 122 | unsigned long __uatomic_exchange(void *addr, unsigned long val, int len) |
0114ba7f | 123 | { |
cc1be41b MD |
124 | /* Note: the "xchg" instruction does not need a "lock" prefix. */ |
125 | switch (len) { | |
126 | case 1: | |
127 | { | |
128 | unsigned char result; | |
129 | __asm__ __volatile__( | |
130 | "xchgb %0, %1" | |
71323499 | 131 | : "=q"(result), "+m"(*__hp(1, addr)) |
cc1be41b MD |
132 | : "0" ((unsigned char)val) |
133 | : "memory"); | |
134 | return result; | |
135 | } | |
136 | case 2: | |
137 | { | |
138 | unsigned short result; | |
139 | __asm__ __volatile__( | |
140 | "xchgw %0, %1" | |
71323499 | 141 | : "=r"(result), "+m"(*__hp(2, addr)) |
cc1be41b MD |
142 | : "0" ((unsigned short)val) |
143 | : "memory"); | |
144 | return result; | |
145 | } | |
146 | case 4: | |
147 | { | |
148 | unsigned int result; | |
149 | __asm__ __volatile__( | |
150 | "xchgl %0, %1" | |
71323499 | 151 | : "=r"(result), "+m"(*__hp(4, addr)) |
cc1be41b MD |
152 | : "0" ((unsigned int)val) |
153 | : "memory"); | |
154 | return result; | |
155 | } | |
e040d717 | 156 | #if (CAA_BITS_PER_LONG == 64) |
cc1be41b MD |
157 | case 8: |
158 | { | |
159 | unsigned long result; | |
160 | __asm__ __volatile__( | |
0114ba7f | 161 | "xchgq %0, %1" |
71323499 | 162 | : "=r"(result), "+m"(*__hp(8, addr)) |
cc1be41b | 163 | : "0" ((unsigned long)val) |
0114ba7f | 164 | : "memory"); |
cc1be41b MD |
165 | return result; |
166 | } | |
167 | #endif | |
168 | } | |
d0bbd9c2 MD |
169 | /* |
170 | * generate an illegal instruction. Cannot catch this with | |
171 | * linker tricks when optimizations are disabled. | |
172 | */ | |
cc1be41b MD |
173 | __asm__ __volatile__("ud2"); |
174 | return 0; | |
0114ba7f MD |
175 | } |
176 | ||
bf9de1b7 | 177 | #define _uatomic_xchg(addr, v) \ |
e56d99bf MD |
178 | ((__typeof__(*(addr))) __uatomic_exchange((addr), \ |
179 | caa_cast_long_keep_sign(v), \ | |
cc1be41b MD |
180 | sizeof(*(addr)))) |
181 | ||
8760d94e | 182 | /* uatomic_add_return */ |
0fad128b MD |
183 | |
184 | static inline __attribute__((always_inline)) | |
bf9de1b7 | 185 | unsigned long __uatomic_add_return(void *addr, unsigned long val, |
0fad128b MD |
186 | int len) |
187 | { | |
188 | switch (len) { | |
189 | case 1: | |
190 | { | |
191 | unsigned char result = val; | |
192 | ||
193 | __asm__ __volatile__( | |
194 | "lock; xaddb %1, %0" | |
71323499 | 195 | : "+m"(*__hp(1, addr)), "+q" (result) |
0fad128b MD |
196 | : |
197 | : "memory"); | |
198 | return result + (unsigned char)val; | |
199 | } | |
200 | case 2: | |
201 | { | |
202 | unsigned short result = val; | |
203 | ||
204 | __asm__ __volatile__( | |
205 | "lock; xaddw %1, %0" | |
71323499 | 206 | : "+m"(*__hp(2, addr)), "+r" (result) |
0fad128b MD |
207 | : |
208 | : "memory"); | |
209 | return result + (unsigned short)val; | |
210 | } | |
211 | case 4: | |
212 | { | |
213 | unsigned int result = val; | |
214 | ||
215 | __asm__ __volatile__( | |
216 | "lock; xaddl %1, %0" | |
71323499 | 217 | : "+m"(*__hp(4, addr)), "+r" (result) |
0fad128b MD |
218 | : |
219 | : "memory"); | |
220 | return result + (unsigned int)val; | |
221 | } | |
e040d717 | 222 | #if (CAA_BITS_PER_LONG == 64) |
0fad128b MD |
223 | case 8: |
224 | { | |
225 | unsigned long result = val; | |
226 | ||
227 | __asm__ __volatile__( | |
228 | "lock; xaddq %1, %0" | |
71323499 | 229 | : "+m"(*__hp(8, addr)), "+r" (result) |
0fad128b MD |
230 | : |
231 | : "memory"); | |
232 | return result + (unsigned long)val; | |
233 | } | |
234 | #endif | |
235 | } | |
d0bbd9c2 MD |
236 | /* |
237 | * generate an illegal instruction. Cannot catch this with | |
238 | * linker tricks when optimizations are disabled. | |
239 | */ | |
0fad128b MD |
240 | __asm__ __volatile__("ud2"); |
241 | return 0; | |
242 | } | |
243 | ||
e56d99bf MD |
244 | #define _uatomic_add_return(addr, v) \ |
245 | ((__typeof__(*(addr))) __uatomic_add_return((addr), \ | |
246 | caa_cast_long_keep_sign(v), \ | |
247 | sizeof(*(addr)))) | |
0fad128b | 248 | |
bf33aaea PB |
249 | /* uatomic_and */ |
250 | ||
251 | static inline __attribute__((always_inline)) | |
252 | void __uatomic_and(void *addr, unsigned long val, int len) | |
253 | { | |
254 | switch (len) { | |
255 | case 1: | |
256 | { | |
257 | __asm__ __volatile__( | |
258 | "lock; andb %1, %0" | |
71323499 | 259 | : "=m"(*__hp(1, addr)) |
bf33aaea PB |
260 | : "iq" ((unsigned char)val) |
261 | : "memory"); | |
262 | return; | |
263 | } | |
264 | case 2: | |
265 | { | |
266 | __asm__ __volatile__( | |
267 | "lock; andw %1, %0" | |
71323499 | 268 | : "=m"(*__hp(2, addr)) |
bf33aaea PB |
269 | : "ir" ((unsigned short)val) |
270 | : "memory"); | |
271 | return; | |
272 | } | |
273 | case 4: | |
274 | { | |
275 | __asm__ __volatile__( | |
276 | "lock; andl %1, %0" | |
71323499 | 277 | : "=m"(*__hp(4, addr)) |
bf33aaea PB |
278 | : "ir" ((unsigned int)val) |
279 | : "memory"); | |
280 | return; | |
281 | } | |
282 | #if (CAA_BITS_PER_LONG == 64) | |
283 | case 8: | |
284 | { | |
285 | __asm__ __volatile__( | |
286 | "lock; andq %1, %0" | |
71323499 | 287 | : "=m"(*__hp(8, addr)) |
bf33aaea PB |
288 | : "er" ((unsigned long)val) |
289 | : "memory"); | |
290 | return; | |
291 | } | |
292 | #endif | |
293 | } | |
d0bbd9c2 MD |
294 | /* |
295 | * generate an illegal instruction. Cannot catch this with | |
296 | * linker tricks when optimizations are disabled. | |
297 | */ | |
bf33aaea PB |
298 | __asm__ __volatile__("ud2"); |
299 | return; | |
300 | } | |
301 | ||
302 | #define _uatomic_and(addr, v) \ | |
e56d99bf | 303 | (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr)))) |
bf33aaea | 304 | |
985b35b1 PB |
305 | /* uatomic_or */ |
306 | ||
307 | static inline __attribute__((always_inline)) | |
308 | void __uatomic_or(void *addr, unsigned long val, int len) | |
309 | { | |
310 | switch (len) { | |
311 | case 1: | |
312 | { | |
313 | __asm__ __volatile__( | |
314 | "lock; orb %1, %0" | |
71323499 | 315 | : "=m"(*__hp(1, addr)) |
985b35b1 PB |
316 | : "iq" ((unsigned char)val) |
317 | : "memory"); | |
318 | return; | |
319 | } | |
320 | case 2: | |
321 | { | |
322 | __asm__ __volatile__( | |
323 | "lock; orw %1, %0" | |
71323499 | 324 | : "=m"(*__hp(2, addr)) |
985b35b1 PB |
325 | : "ir" ((unsigned short)val) |
326 | : "memory"); | |
327 | return; | |
328 | } | |
329 | case 4: | |
330 | { | |
331 | __asm__ __volatile__( | |
332 | "lock; orl %1, %0" | |
71323499 | 333 | : "=m"(*__hp(4, addr)) |
985b35b1 PB |
334 | : "ir" ((unsigned int)val) |
335 | : "memory"); | |
336 | return; | |
337 | } | |
338 | #if (CAA_BITS_PER_LONG == 64) | |
339 | case 8: | |
340 | { | |
341 | __asm__ __volatile__( | |
342 | "lock; orq %1, %0" | |
71323499 | 343 | : "=m"(*__hp(8, addr)) |
985b35b1 PB |
344 | : "er" ((unsigned long)val) |
345 | : "memory"); | |
346 | return; | |
347 | } | |
348 | #endif | |
349 | } | |
d0bbd9c2 MD |
350 | /* |
351 | * generate an illegal instruction. Cannot catch this with | |
352 | * linker tricks when optimizations are disabled. | |
353 | */ | |
985b35b1 PB |
354 | __asm__ __volatile__("ud2"); |
355 | return; | |
356 | } | |
357 | ||
358 | #define _uatomic_or(addr, v) \ | |
e56d99bf | 359 | (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr)))) |
985b35b1 | 360 | |
8760d94e | 361 | /* uatomic_add */ |
0114ba7f | 362 | |
5dba80f9 | 363 | static inline __attribute__((always_inline)) |
bf9de1b7 | 364 | void __uatomic_add(void *addr, unsigned long val, int len) |
0114ba7f MD |
365 | { |
366 | switch (len) { | |
cc1be41b MD |
367 | case 1: |
368 | { | |
369 | __asm__ __volatile__( | |
370 | "lock; addb %1, %0" | |
71323499 | 371 | : "=m"(*__hp(1, addr)) |
87322fe8 MD |
372 | : "iq" ((unsigned char)val) |
373 | : "memory"); | |
cc1be41b MD |
374 | return; |
375 | } | |
376 | case 2: | |
377 | { | |
378 | __asm__ __volatile__( | |
379 | "lock; addw %1, %0" | |
71323499 | 380 | : "=m"(*__hp(2, addr)) |
87322fe8 MD |
381 | : "ir" ((unsigned short)val) |
382 | : "memory"); | |
cc1be41b MD |
383 | return; |
384 | } | |
385 | case 4: | |
386 | { | |
387 | __asm__ __volatile__( | |
388 | "lock; addl %1, %0" | |
71323499 | 389 | : "=m"(*__hp(4, addr)) |
87322fe8 MD |
390 | : "ir" ((unsigned int)val) |
391 | : "memory"); | |
cc1be41b MD |
392 | return; |
393 | } | |
e040d717 | 394 | #if (CAA_BITS_PER_LONG == 64) |
cc1be41b MD |
395 | case 8: |
396 | { | |
397 | __asm__ __volatile__( | |
398 | "lock; addq %1, %0" | |
71323499 | 399 | : "=m"(*__hp(8, addr)) |
87322fe8 MD |
400 | : "er" ((unsigned long)val) |
401 | : "memory"); | |
cc1be41b MD |
402 | return; |
403 | } | |
0114ba7f MD |
404 | #endif |
405 | } | |
d0bbd9c2 MD |
406 | /* |
407 | * generate an illegal instruction. Cannot catch this with | |
408 | * linker tricks when optimizations are disabled. | |
409 | */ | |
0114ba7f | 410 | __asm__ __volatile__("ud2"); |
a81b8e5e | 411 | return; |
0114ba7f MD |
412 | } |
413 | ||
bf9de1b7 | 414 | #define _uatomic_add(addr, v) \ |
e56d99bf | 415 | (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr)))) |
0114ba7f | 416 | |
2c5e5fb3 | 417 | |
ec4e58a3 | 418 | /* uatomic_inc */ |
2c5e5fb3 MD |
419 | |
420 | static inline __attribute__((always_inline)) | |
bf9de1b7 | 421 | void __uatomic_inc(void *addr, int len) |
2c5e5fb3 MD |
422 | { |
423 | switch (len) { | |
424 | case 1: | |
425 | { | |
426 | __asm__ __volatile__( | |
427 | "lock; incb %0" | |
71323499 | 428 | : "=m"(*__hp(1, addr)) |
2c5e5fb3 MD |
429 | : |
430 | : "memory"); | |
431 | return; | |
432 | } | |
433 | case 2: | |
434 | { | |
435 | __asm__ __volatile__( | |
436 | "lock; incw %0" | |
71323499 | 437 | : "=m"(*__hp(2, addr)) |
2c5e5fb3 MD |
438 | : |
439 | : "memory"); | |
440 | return; | |
441 | } | |
442 | case 4: | |
443 | { | |
444 | __asm__ __volatile__( | |
445 | "lock; incl %0" | |
71323499 | 446 | : "=m"(*__hp(4, addr)) |
2c5e5fb3 MD |
447 | : |
448 | : "memory"); | |
449 | return; | |
450 | } | |
e040d717 | 451 | #if (CAA_BITS_PER_LONG == 64) |
2c5e5fb3 MD |
452 | case 8: |
453 | { | |
454 | __asm__ __volatile__( | |
455 | "lock; incq %0" | |
71323499 | 456 | : "=m"(*__hp(8, addr)) |
2c5e5fb3 MD |
457 | : |
458 | : "memory"); | |
459 | return; | |
460 | } | |
461 | #endif | |
462 | } | |
463 | /* generate an illegal instruction. Cannot catch this with linker tricks | |
464 | * when optimizations are disabled. */ | |
465 | __asm__ __volatile__("ud2"); | |
466 | return; | |
467 | } | |
468 | ||
bf9de1b7 | 469 | #define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr)))) |
2c5e5fb3 | 470 | |
ec4e58a3 | 471 | /* uatomic_dec */ |
2c5e5fb3 MD |
472 | |
473 | static inline __attribute__((always_inline)) | |
bf9de1b7 | 474 | void __uatomic_dec(void *addr, int len) |
2c5e5fb3 MD |
475 | { |
476 | switch (len) { | |
477 | case 1: | |
478 | { | |
479 | __asm__ __volatile__( | |
480 | "lock; decb %0" | |
71323499 | 481 | : "=m"(*__hp(1, addr)) |
2c5e5fb3 MD |
482 | : |
483 | : "memory"); | |
484 | return; | |
485 | } | |
486 | case 2: | |
487 | { | |
488 | __asm__ __volatile__( | |
489 | "lock; decw %0" | |
71323499 | 490 | : "=m"(*__hp(2, addr)) |
2c5e5fb3 MD |
491 | : |
492 | : "memory"); | |
493 | return; | |
494 | } | |
495 | case 4: | |
496 | { | |
497 | __asm__ __volatile__( | |
498 | "lock; decl %0" | |
71323499 | 499 | : "=m"(*__hp(4, addr)) |
2c5e5fb3 MD |
500 | : |
501 | : "memory"); | |
502 | return; | |
503 | } | |
e040d717 | 504 | #if (CAA_BITS_PER_LONG == 64) |
2c5e5fb3 MD |
505 | case 8: |
506 | { | |
507 | __asm__ __volatile__( | |
508 | "lock; decq %0" | |
71323499 | 509 | : "=m"(*__hp(8, addr)) |
2c5e5fb3 MD |
510 | : |
511 | : "memory"); | |
512 | return; | |
513 | } | |
514 | #endif | |
515 | } | |
d0bbd9c2 MD |
516 | /* |
517 | * generate an illegal instruction. Cannot catch this with | |
518 | * linker tricks when optimizations are disabled. | |
519 | */ | |
2c5e5fb3 MD |
520 | __asm__ __volatile__("ud2"); |
521 | return; | |
522 | } | |
523 | ||
bf9de1b7 | 524 | #define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr)))) |
0114ba7f | 525 | |
0b1e236d MJ |
526 | #if ((CAA_BITS_PER_LONG != 64) && defined(URCU_ARCH_I386)) |
527 | ||
528 | /* For backwards compat */ | |
529 | #define CONFIG_RCU_COMPAT_ARCH 1 | |
530 | ||
02be5561 MD |
531 | extern int __rcu_cas_avail; |
532 | extern int __rcu_cas_init(void); | |
bf9de1b7 MD |
533 | |
534 | #define UATOMIC_COMPAT(insn) \ | |
a0b7f7ea | 535 | ((caa_likely(__rcu_cas_avail > 0)) \ |
bf9de1b7 | 536 | ? (_uatomic_##insn) \ |
a0b7f7ea | 537 | : ((caa_unlikely(__rcu_cas_avail < 0) \ |
02be5561 | 538 | ? ((__rcu_cas_init() > 0) \ |
bf9de1b7 MD |
539 | ? (_uatomic_##insn) \ |
540 | : (compat_uatomic_##insn)) \ | |
541 | : (compat_uatomic_##insn)))) | |
542 | ||
424d4ed5 MD |
543 | /* |
544 | * We leave the return value so we don't break the ABI, but remove the | |
545 | * return value from the API. | |
546 | */ | |
bf9de1b7 MD |
547 | extern unsigned long _compat_uatomic_set(void *addr, |
548 | unsigned long _new, int len); | |
549 | #define compat_uatomic_set(addr, _new) \ | |
424d4ed5 MD |
550 | ((void) _compat_uatomic_set((addr), \ |
551 | caa_cast_long_keep_sign(_new), \ | |
552 | sizeof(*(addr)))) | |
bf9de1b7 MD |
553 | |
554 | ||
555 | extern unsigned long _compat_uatomic_xchg(void *addr, | |
556 | unsigned long _new, int len); | |
557 | #define compat_uatomic_xchg(addr, _new) \ | |
558 | ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \ | |
e56d99bf | 559 | caa_cast_long_keep_sign(_new), \ |
bf9de1b7 | 560 | sizeof(*(addr)))) |
7d413817 MD |
561 | |
562 | extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old, | |
bf9de1b7 MD |
563 | unsigned long _new, int len); |
564 | #define compat_uatomic_cmpxchg(addr, old, _new) \ | |
565 | ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \ | |
e56d99bf MD |
566 | caa_cast_long_keep_sign(old), \ |
567 | caa_cast_long_keep_sign(_new), \ | |
bf9de1b7 | 568 | sizeof(*(addr)))) |
7d413817 | 569 | |
8c43fe72 | 570 | extern void _compat_uatomic_and(void *addr, unsigned long _new, int len); |
bf33aaea | 571 | #define compat_uatomic_and(addr, v) \ |
8c43fe72 | 572 | (_compat_uatomic_and((addr), \ |
e56d99bf | 573 | caa_cast_long_keep_sign(v), \ |
8c43fe72 | 574 | sizeof(*(addr)))) |
bf33aaea | 575 | |
8c43fe72 | 576 | extern void _compat_uatomic_or(void *addr, unsigned long _new, int len); |
985b35b1 | 577 | #define compat_uatomic_or(addr, v) \ |
8c43fe72 | 578 | (_compat_uatomic_or((addr), \ |
e56d99bf | 579 | caa_cast_long_keep_sign(v), \ |
8c43fe72 | 580 | sizeof(*(addr)))) |
985b35b1 | 581 | |
28ca843d PB |
582 | extern unsigned long _compat_uatomic_add_return(void *addr, |
583 | unsigned long _new, int len); | |
e56d99bf MD |
584 | #define compat_uatomic_add_return(addr, v) \ |
585 | ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \ | |
586 | caa_cast_long_keep_sign(v), \ | |
587 | sizeof(*(addr)))) | |
bf9de1b7 | 588 | |
bf9de1b7 MD |
589 | #define compat_uatomic_add(addr, v) \ |
590 | ((void)compat_uatomic_add_return((addr), (v))) | |
bf9de1b7 MD |
591 | #define compat_uatomic_inc(addr) \ |
592 | (compat_uatomic_add((addr), 1)) | |
593 | #define compat_uatomic_dec(addr) \ | |
8760d94e | 594 | (compat_uatomic_add((addr), -1)) |
bf9de1b7 MD |
595 | |
596 | #else | |
597 | #define UATOMIC_COMPAT(insn) (_uatomic_##insn) | |
7d413817 MD |
598 | #endif |
599 | ||
bf9de1b7 | 600 | /* Read is atomic even in compat mode */ |
bf9de1b7 MD |
601 | #define uatomic_set(addr, v) \ |
602 | UATOMIC_COMPAT(set(addr, v)) | |
8760d94e | 603 | |
bf9de1b7 MD |
604 | #define uatomic_cmpxchg(addr, old, _new) \ |
605 | UATOMIC_COMPAT(cmpxchg(addr, old, _new)) | |
606 | #define uatomic_xchg(addr, v) \ | |
607 | UATOMIC_COMPAT(xchg(addr, v)) | |
2812a2d2 | 608 | |
bf33aaea PB |
609 | #define uatomic_and(addr, v) \ |
610 | UATOMIC_COMPAT(and(addr, v)) | |
42e83919 MD |
611 | #define cmm_smp_mb__before_uatomic_and() cmm_barrier() |
612 | #define cmm_smp_mb__after_uatomic_and() cmm_barrier() | |
2812a2d2 | 613 | |
985b35b1 PB |
614 | #define uatomic_or(addr, v) \ |
615 | UATOMIC_COMPAT(or(addr, v)) | |
42e83919 MD |
616 | #define cmm_smp_mb__before_uatomic_or() cmm_barrier() |
617 | #define cmm_smp_mb__after_uatomic_or() cmm_barrier() | |
2812a2d2 | 618 | |
bf9de1b7 MD |
619 | #define uatomic_add_return(addr, v) \ |
620 | UATOMIC_COMPAT(add_return(addr, v)) | |
8760d94e | 621 | |
bf9de1b7 | 622 | #define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v)) |
42e83919 MD |
623 | #define cmm_smp_mb__before_uatomic_add() cmm_barrier() |
624 | #define cmm_smp_mb__after_uatomic_add() cmm_barrier() | |
2812a2d2 | 625 | |
bf9de1b7 | 626 | #define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr)) |
42e83919 MD |
627 | #define cmm_smp_mb__before_uatomic_inc() cmm_barrier() |
628 | #define cmm_smp_mb__after_uatomic_inc() cmm_barrier() | |
2812a2d2 | 629 | |
bf9de1b7 | 630 | #define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr)) |
42e83919 MD |
631 | #define cmm_smp_mb__before_uatomic_dec() cmm_barrier() |
632 | #define cmm_smp_mb__after_uatomic_dec() cmm_barrier() | |
bf9de1b7 | 633 | |
67ecffc0 | 634 | #ifdef __cplusplus |
36bc70a8 MD |
635 | } |
636 | #endif | |
637 | ||
a2e7bf9c | 638 | #include <urcu/uatomic/generic.h> |
8760d94e | 639 | |
ec4e58a3 | 640 | #endif /* _URCU_ARCH_UATOMIC_X86_H */ |