add uatomic_generic.h, use it for common definitions
[urcu.git] / urcu / uatomic_arch_x86.h
CommitLineData
ec4e58a3
MD
1#ifndef _URCU_ARCH_UATOMIC_X86_H
2#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f
MD
3
4/*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 *
10 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
11 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
12 *
13 * Permission is hereby granted to use or copy this program
14 * for any purpose, provided the above notices are retained on all copies.
15 * Permission to modify the code and to distribute modified code is granted,
16 * provided the above notices are retained, and a notice that the code was
17 * modified is included with the above copyright notice.
18 *
ec4e58a3 19 * Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f
MD
20 * Boehm-Demers-Weiser conservative garbage collector.
21 */
22
ec4e58a3 23#include <urcu/compiler.h>
bf9de1b7 24#include <urcu/system.h>
0fad128b 25
36bc70a8
MD
26#ifdef __cplusplus
27extern "C" {
28#endif
29
618c4ae1
MD
30#ifndef __SIZEOF_LONG__
31#if defined(__x86_64__) || defined(__amd64__)
32#define __SIZEOF_LONG__ 8
33#else
34#define __SIZEOF_LONG__ 4
35#endif
36#endif
37
0114ba7f
MD
38#ifndef BITS_PER_LONG
39#define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
40#endif
41
0114ba7f 42/*
0114ba7f
MD
43 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
44 */
45
ec4e58a3 46struct __uatomic_dummy {
cc1be41b
MD
47 unsigned long v[10];
48};
ec4e58a3 49#define __hp(x) ((struct __uatomic_dummy *)(x))
cc1be41b 50
bf9de1b7 51#define _uatomic_set(addr, v) STORE_SHARED(*(addr), (v))
0fad128b 52
cc1be41b
MD
53/* cmpxchg */
54
5dba80f9 55static inline __attribute__((always_inline))
bf9de1b7 56unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b 57 unsigned long _new, int len)
0114ba7f 58{
cc1be41b
MD
59 switch (len) {
60 case 1:
61 {
62 unsigned char result = old;
0fad128b 63
cc1be41b
MD
64 __asm__ __volatile__(
65 "lock; cmpxchgb %2, %1"
66 : "+a"(result), "+m"(*__hp(addr))
67 : "q"((unsigned char)_new)
0114ba7f 68 : "memory");
cc1be41b
MD
69 return result;
70 }
71 case 2:
72 {
73 unsigned short result = old;
0fad128b 74
cc1be41b
MD
75 __asm__ __volatile__(
76 "lock; cmpxchgw %2, %1"
77 : "+a"(result), "+m"(*__hp(addr))
78 : "r"((unsigned short)_new)
79 : "memory");
80 return result;
81 }
82 case 4:
83 {
84 unsigned int result = old;
0fad128b 85
cc1be41b
MD
86 __asm__ __volatile__(
87 "lock; cmpxchgl %2, %1"
88 : "+a"(result), "+m"(*__hp(addr))
89 : "r"((unsigned int)_new)
90 : "memory");
91 return result;
92 }
93#if (BITS_PER_LONG == 64)
94 case 8:
95 {
6edb297e 96 unsigned long result = old;
0fad128b 97
cc1be41b 98 __asm__ __volatile__(
2c5e5fb3 99 "lock; cmpxchgq %2, %1"
cc1be41b
MD
100 : "+a"(result), "+m"(*__hp(addr))
101 : "r"((unsigned long)_new)
102 : "memory");
103 return result;
104 }
105#endif
106 }
107 /* generate an illegal instruction. Cannot catch this with linker tricks
108 * when optimizations are disabled. */
109 __asm__ __volatile__("ud2");
110 return 0;
0114ba7f
MD
111}
112
bf9de1b7
MD
113#define _uatomic_cmpxchg(addr, old, _new) \
114 ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), (unsigned long)(old),\
115 (unsigned long)(_new), \
cc1be41b
MD
116 sizeof(*(addr))))
117
118/* xchg */
0114ba7f 119
5dba80f9 120static inline __attribute__((always_inline))
bf9de1b7 121unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f 122{
cc1be41b
MD
123 /* Note: the "xchg" instruction does not need a "lock" prefix. */
124 switch (len) {
125 case 1:
126 {
127 unsigned char result;
128 __asm__ __volatile__(
129 "xchgb %0, %1"
130 : "=q"(result), "+m"(*__hp(addr))
131 : "0" ((unsigned char)val)
132 : "memory");
133 return result;
134 }
135 case 2:
136 {
137 unsigned short result;
138 __asm__ __volatile__(
139 "xchgw %0, %1"
140 : "=r"(result), "+m"(*__hp(addr))
141 : "0" ((unsigned short)val)
142 : "memory");
143 return result;
144 }
145 case 4:
146 {
147 unsigned int result;
148 __asm__ __volatile__(
149 "xchgl %0, %1"
150 : "=r"(result), "+m"(*__hp(addr))
151 : "0" ((unsigned int)val)
152 : "memory");
153 return result;
154 }
155#if (BITS_PER_LONG == 64)
156 case 8:
157 {
158 unsigned long result;
159 __asm__ __volatile__(
0114ba7f 160 "xchgq %0, %1"
cc1be41b
MD
161 : "=r"(result), "+m"(*__hp(addr))
162 : "0" ((unsigned long)val)
0114ba7f 163 : "memory");
cc1be41b
MD
164 return result;
165 }
166#endif
167 }
168 /* generate an illegal instruction. Cannot catch this with linker tricks
169 * when optimizations are disabled. */
170 __asm__ __volatile__("ud2");
171 return 0;
0114ba7f
MD
172}
173
bf9de1b7
MD
174#define _uatomic_xchg(addr, v) \
175 ((__typeof__(*(addr))) __uatomic_exchange((addr), (unsigned long)(v), \
cc1be41b
MD
176 sizeof(*(addr))))
177
8760d94e 178/* uatomic_add_return */
0fad128b
MD
179
180static inline __attribute__((always_inline))
bf9de1b7 181unsigned long __uatomic_add_return(void *addr, unsigned long val,
0fad128b
MD
182 int len)
183{
184 switch (len) {
185 case 1:
186 {
187 unsigned char result = val;
188
189 __asm__ __volatile__(
190 "lock; xaddb %1, %0"
191 : "+m"(*__hp(addr)), "+q" (result)
192 :
193 : "memory");
194 return result + (unsigned char)val;
195 }
196 case 2:
197 {
198 unsigned short result = val;
199
200 __asm__ __volatile__(
201 "lock; xaddw %1, %0"
202 : "+m"(*__hp(addr)), "+r" (result)
203 :
204 : "memory");
205 return result + (unsigned short)val;
206 }
207 case 4:
208 {
209 unsigned int result = val;
210
211 __asm__ __volatile__(
212 "lock; xaddl %1, %0"
213 : "+m"(*__hp(addr)), "+r" (result)
214 :
215 : "memory");
216 return result + (unsigned int)val;
217 }
218#if (BITS_PER_LONG == 64)
219 case 8:
220 {
221 unsigned long result = val;
222
223 __asm__ __volatile__(
224 "lock; xaddq %1, %0"
225 : "+m"(*__hp(addr)), "+r" (result)
226 :
227 : "memory");
228 return result + (unsigned long)val;
229 }
230#endif
231 }
232 /* generate an illegal instruction. Cannot catch this with linker tricks
233 * when optimizations are disabled. */
234 __asm__ __volatile__("ud2");
235 return 0;
236}
237
bf9de1b7
MD
238#define _uatomic_add_return(addr, v) \
239 ((__typeof__(*(addr))) __uatomic_add_return((addr), \
0fad128b
MD
240 (unsigned long)(v), \
241 sizeof(*(addr))))
242
8760d94e 243/* uatomic_add */
0114ba7f 244
5dba80f9 245static inline __attribute__((always_inline))
bf9de1b7 246void __uatomic_add(void *addr, unsigned long val, int len)
0114ba7f
MD
247{
248 switch (len) {
cc1be41b
MD
249 case 1:
250 {
251 __asm__ __volatile__(
252 "lock; addb %1, %0"
253 : "=m"(*__hp(addr))
87322fe8
MD
254 : "iq" ((unsigned char)val)
255 : "memory");
cc1be41b
MD
256 return;
257 }
258 case 2:
259 {
260 __asm__ __volatile__(
261 "lock; addw %1, %0"
262 : "=m"(*__hp(addr))
87322fe8
MD
263 : "ir" ((unsigned short)val)
264 : "memory");
cc1be41b
MD
265 return;
266 }
267 case 4:
268 {
269 __asm__ __volatile__(
270 "lock; addl %1, %0"
271 : "=m"(*__hp(addr))
87322fe8
MD
272 : "ir" ((unsigned int)val)
273 : "memory");
cc1be41b
MD
274 return;
275 }
0114ba7f 276#if (BITS_PER_LONG == 64)
cc1be41b
MD
277 case 8:
278 {
279 __asm__ __volatile__(
280 "lock; addq %1, %0"
281 : "=m"(*__hp(addr))
87322fe8
MD
282 : "er" ((unsigned long)val)
283 : "memory");
cc1be41b
MD
284 return;
285 }
0114ba7f
MD
286#endif
287 }
288 /* generate an illegal instruction. Cannot catch this with linker tricks
289 * when optimizations are disabled. */
290 __asm__ __volatile__("ud2");
a81b8e5e 291 return;
0114ba7f
MD
292}
293
bf9de1b7
MD
294#define _uatomic_add(addr, v) \
295 (__uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
0114ba7f 296
2c5e5fb3 297
ec4e58a3 298/* uatomic_inc */
2c5e5fb3
MD
299
300static inline __attribute__((always_inline))
bf9de1b7 301void __uatomic_inc(void *addr, int len)
2c5e5fb3
MD
302{
303 switch (len) {
304 case 1:
305 {
306 __asm__ __volatile__(
307 "lock; incb %0"
308 : "=m"(*__hp(addr))
309 :
310 : "memory");
311 return;
312 }
313 case 2:
314 {
315 __asm__ __volatile__(
316 "lock; incw %0"
317 : "=m"(*__hp(addr))
318 :
319 : "memory");
320 return;
321 }
322 case 4:
323 {
324 __asm__ __volatile__(
325 "lock; incl %0"
326 : "=m"(*__hp(addr))
327 :
328 : "memory");
329 return;
330 }
331#if (BITS_PER_LONG == 64)
332 case 8:
333 {
334 __asm__ __volatile__(
335 "lock; incq %0"
336 : "=m"(*__hp(addr))
337 :
338 : "memory");
339 return;
340 }
341#endif
342 }
343 /* generate an illegal instruction. Cannot catch this with linker tricks
344 * when optimizations are disabled. */
345 __asm__ __volatile__("ud2");
346 return;
347}
348
bf9de1b7 349#define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3 350
ec4e58a3 351/* uatomic_dec */
2c5e5fb3
MD
352
353static inline __attribute__((always_inline))
bf9de1b7 354void __uatomic_dec(void *addr, int len)
2c5e5fb3
MD
355{
356 switch (len) {
357 case 1:
358 {
359 __asm__ __volatile__(
360 "lock; decb %0"
361 : "=m"(*__hp(addr))
362 :
363 : "memory");
364 return;
365 }
366 case 2:
367 {
368 __asm__ __volatile__(
369 "lock; decw %0"
370 : "=m"(*__hp(addr))
371 :
372 : "memory");
373 return;
374 }
375 case 4:
376 {
377 __asm__ __volatile__(
378 "lock; decl %0"
379 : "=m"(*__hp(addr))
380 :
381 : "memory");
382 return;
383 }
384#if (BITS_PER_LONG == 64)
385 case 8:
386 {
387 __asm__ __volatile__(
388 "lock; decq %0"
389 : "=m"(*__hp(addr))
390 :
391 : "memory");
392 return;
393 }
394#endif
395 }
396 /* generate an illegal instruction. Cannot catch this with linker tricks
397 * when optimizations are disabled. */
398 __asm__ __volatile__("ud2");
399 return;
400}
401
bf9de1b7 402#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
0114ba7f 403
02be5561
MD
404#if ((BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
405extern int __rcu_cas_avail;
406extern int __rcu_cas_init(void);
bf9de1b7
MD
407
408#define UATOMIC_COMPAT(insn) \
02be5561 409 ((likely(__rcu_cas_avail > 0)) \
bf9de1b7 410 ? (_uatomic_##insn) \
02be5561
MD
411 : ((unlikely(__rcu_cas_avail < 0) \
412 ? ((__rcu_cas_init() > 0) \
bf9de1b7
MD
413 ? (_uatomic_##insn) \
414 : (compat_uatomic_##insn)) \
415 : (compat_uatomic_##insn))))
416
417extern unsigned long _compat_uatomic_set(void *addr,
418 unsigned long _new, int len);
419#define compat_uatomic_set(addr, _new) \
420 ((__typeof__(*(addr))) _compat_uatomic_set((addr), \
421 (unsigned long)(_new), \
422 sizeof(*(addr))))
423
424
425extern unsigned long _compat_uatomic_xchg(void *addr,
426 unsigned long _new, int len);
427#define compat_uatomic_xchg(addr, _new) \
428 ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
429 (unsigned long)(_new), \
430 sizeof(*(addr))))
7d413817
MD
431
432extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
bf9de1b7
MD
433 unsigned long _new, int len);
434#define compat_uatomic_cmpxchg(addr, old, _new) \
435 ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
436 (unsigned long)(old), \
437 (unsigned long)(_new), \
438 sizeof(*(addr))))
7d413817 439
bf9de1b7
MD
440extern unsigned long _compat_uatomic_xchg(void *addr,
441 unsigned long _new, int len);
442#define compat_uatomic_add_return(addr, v) \
443 ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
444 (unsigned long)(v), \
7d413817 445 sizeof(*(addr))))
bf9de1b7 446
bf9de1b7
MD
447#define compat_uatomic_add(addr, v) \
448 ((void)compat_uatomic_add_return((addr), (v)))
bf9de1b7
MD
449#define compat_uatomic_inc(addr) \
450 (compat_uatomic_add((addr), 1))
451#define compat_uatomic_dec(addr) \
8760d94e 452 (compat_uatomic_add((addr), -1))
bf9de1b7
MD
453
454#else
455#define UATOMIC_COMPAT(insn) (_uatomic_##insn)
7d413817
MD
456#endif
457
bf9de1b7 458/* Read is atomic even in compat mode */
bf9de1b7
MD
459#define uatomic_set(addr, v) \
460 UATOMIC_COMPAT(set(addr, v))
8760d94e 461
bf9de1b7
MD
462#define uatomic_cmpxchg(addr, old, _new) \
463 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
464#define uatomic_xchg(addr, v) \
465 UATOMIC_COMPAT(xchg(addr, v))
466#define uatomic_add_return(addr, v) \
467 UATOMIC_COMPAT(add_return(addr, v))
8760d94e 468
bf9de1b7 469#define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
bf9de1b7
MD
470#define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
471#define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
472
36bc70a8
MD
473#ifdef __cplusplus
474}
475#endif
476
8760d94e
PB
477#include <urcu/uatomic_generic.h>
478
ec4e58a3 479#endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.069731 seconds and 4 git commands to generate.