e358be3673552dd9919ca6579e2c83071410157a
[urcu.git] / urcu / uatomic_arch_x86.h
1 #ifndef _URCU_ARCH_UATOMIC_X86_H
2 #define _URCU_ARCH_UATOMIC_X86_H
3
4 /*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 *
10 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
11 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
12 *
13 * Permission is hereby granted to use or copy this program
14 * for any purpose, provided the above notices are retained on all copies.
15 * Permission to modify the code and to distribute modified code is granted,
16 * provided the above notices are retained, and a notice that the code was
17 * modified is included with the above copyright notice.
18 *
19 * Code inspired from libuatomic_ops-1.2, inherited in part from the
20 * Boehm-Demers-Weiser conservative garbage collector.
21 */
22
23 #include <urcu/compiler.h>
24 #include <urcu/system.h>
25
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29
30 #ifndef __SIZEOF_LONG__
31 #if defined(__x86_64__) || defined(__amd64__)
32 #define __SIZEOF_LONG__ 8
33 #else
34 #define __SIZEOF_LONG__ 4
35 #endif
36 #endif
37
38 #ifndef BITS_PER_LONG
39 #define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
40 #endif
41
42 /*
43 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
44 */
45
46 struct __uatomic_dummy {
47 unsigned long v[10];
48 };
49 #define __hp(x) ((struct __uatomic_dummy *)(x))
50
51 #define _uatomic_set(addr, v) STORE_SHARED(*(addr), (v))
52
53 /* cmpxchg */
54
55 static inline __attribute__((always_inline))
56 unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
57 unsigned long _new, int len)
58 {
59 switch (len) {
60 case 1:
61 {
62 unsigned char result = old;
63
64 __asm__ __volatile__(
65 "lock; cmpxchgb %2, %1"
66 : "+a"(result), "+m"(*__hp(addr))
67 : "q"((unsigned char)_new)
68 : "memory");
69 return result;
70 }
71 case 2:
72 {
73 unsigned short result = old;
74
75 __asm__ __volatile__(
76 "lock; cmpxchgw %2, %1"
77 : "+a"(result), "+m"(*__hp(addr))
78 : "r"((unsigned short)_new)
79 : "memory");
80 return result;
81 }
82 case 4:
83 {
84 unsigned int result = old;
85
86 __asm__ __volatile__(
87 "lock; cmpxchgl %2, %1"
88 : "+a"(result), "+m"(*__hp(addr))
89 : "r"((unsigned int)_new)
90 : "memory");
91 return result;
92 }
93 #if (BITS_PER_LONG == 64)
94 case 8:
95 {
96 unsigned long result = old;
97
98 __asm__ __volatile__(
99 "lock; cmpxchgq %2, %1"
100 : "+a"(result), "+m"(*__hp(addr))
101 : "r"((unsigned long)_new)
102 : "memory");
103 return result;
104 }
105 #endif
106 }
107 /* generate an illegal instruction. Cannot catch this with linker tricks
108 * when optimizations are disabled. */
109 __asm__ __volatile__("ud2");
110 return 0;
111 }
112
113 #define _uatomic_cmpxchg(addr, old, _new) \
114 ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), (unsigned long)(old),\
115 (unsigned long)(_new), \
116 sizeof(*(addr))))
117
118 /* xchg */
119
120 static inline __attribute__((always_inline))
121 unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
122 {
123 /* Note: the "xchg" instruction does not need a "lock" prefix. */
124 switch (len) {
125 case 1:
126 {
127 unsigned char result;
128 __asm__ __volatile__(
129 "xchgb %0, %1"
130 : "=q"(result), "+m"(*__hp(addr))
131 : "0" ((unsigned char)val)
132 : "memory");
133 return result;
134 }
135 case 2:
136 {
137 unsigned short result;
138 __asm__ __volatile__(
139 "xchgw %0, %1"
140 : "=r"(result), "+m"(*__hp(addr))
141 : "0" ((unsigned short)val)
142 : "memory");
143 return result;
144 }
145 case 4:
146 {
147 unsigned int result;
148 __asm__ __volatile__(
149 "xchgl %0, %1"
150 : "=r"(result), "+m"(*__hp(addr))
151 : "0" ((unsigned int)val)
152 : "memory");
153 return result;
154 }
155 #if (BITS_PER_LONG == 64)
156 case 8:
157 {
158 unsigned long result;
159 __asm__ __volatile__(
160 "xchgq %0, %1"
161 : "=r"(result), "+m"(*__hp(addr))
162 : "0" ((unsigned long)val)
163 : "memory");
164 return result;
165 }
166 #endif
167 }
168 /* generate an illegal instruction. Cannot catch this with linker tricks
169 * when optimizations are disabled. */
170 __asm__ __volatile__("ud2");
171 return 0;
172 }
173
174 #define _uatomic_xchg(addr, v) \
175 ((__typeof__(*(addr))) __uatomic_exchange((addr), (unsigned long)(v), \
176 sizeof(*(addr))))
177
178 /* uatomic_add_return */
179
180 static inline __attribute__((always_inline))
181 unsigned long __uatomic_add_return(void *addr, unsigned long val,
182 int len)
183 {
184 switch (len) {
185 case 1:
186 {
187 unsigned char result = val;
188
189 __asm__ __volatile__(
190 "lock; xaddb %1, %0"
191 : "+m"(*__hp(addr)), "+q" (result)
192 :
193 : "memory");
194 return result + (unsigned char)val;
195 }
196 case 2:
197 {
198 unsigned short result = val;
199
200 __asm__ __volatile__(
201 "lock; xaddw %1, %0"
202 : "+m"(*__hp(addr)), "+r" (result)
203 :
204 : "memory");
205 return result + (unsigned short)val;
206 }
207 case 4:
208 {
209 unsigned int result = val;
210
211 __asm__ __volatile__(
212 "lock; xaddl %1, %0"
213 : "+m"(*__hp(addr)), "+r" (result)
214 :
215 : "memory");
216 return result + (unsigned int)val;
217 }
218 #if (BITS_PER_LONG == 64)
219 case 8:
220 {
221 unsigned long result = val;
222
223 __asm__ __volatile__(
224 "lock; xaddq %1, %0"
225 : "+m"(*__hp(addr)), "+r" (result)
226 :
227 : "memory");
228 return result + (unsigned long)val;
229 }
230 #endif
231 }
232 /* generate an illegal instruction. Cannot catch this with linker tricks
233 * when optimizations are disabled. */
234 __asm__ __volatile__("ud2");
235 return 0;
236 }
237
238 #define _uatomic_add_return(addr, v) \
239 ((__typeof__(*(addr))) __uatomic_add_return((addr), \
240 (unsigned long)(v), \
241 sizeof(*(addr))))
242
243 /* uatomic_add */
244
245 static inline __attribute__((always_inline))
246 void __uatomic_add(void *addr, unsigned long val, int len)
247 {
248 switch (len) {
249 case 1:
250 {
251 __asm__ __volatile__(
252 "lock; addb %1, %0"
253 : "=m"(*__hp(addr))
254 : "iq" ((unsigned char)val)
255 : "memory");
256 return;
257 }
258 case 2:
259 {
260 __asm__ __volatile__(
261 "lock; addw %1, %0"
262 : "=m"(*__hp(addr))
263 : "ir" ((unsigned short)val)
264 : "memory");
265 return;
266 }
267 case 4:
268 {
269 __asm__ __volatile__(
270 "lock; addl %1, %0"
271 : "=m"(*__hp(addr))
272 : "ir" ((unsigned int)val)
273 : "memory");
274 return;
275 }
276 #if (BITS_PER_LONG == 64)
277 case 8:
278 {
279 __asm__ __volatile__(
280 "lock; addq %1, %0"
281 : "=m"(*__hp(addr))
282 : "er" ((unsigned long)val)
283 : "memory");
284 return;
285 }
286 #endif
287 }
288 /* generate an illegal instruction. Cannot catch this with linker tricks
289 * when optimizations are disabled. */
290 __asm__ __volatile__("ud2");
291 return;
292 }
293
294 #define _uatomic_add(addr, v) \
295 (__uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
296
297
298 /* uatomic_inc */
299
300 static inline __attribute__((always_inline))
301 void __uatomic_inc(void *addr, int len)
302 {
303 switch (len) {
304 case 1:
305 {
306 __asm__ __volatile__(
307 "lock; incb %0"
308 : "=m"(*__hp(addr))
309 :
310 : "memory");
311 return;
312 }
313 case 2:
314 {
315 __asm__ __volatile__(
316 "lock; incw %0"
317 : "=m"(*__hp(addr))
318 :
319 : "memory");
320 return;
321 }
322 case 4:
323 {
324 __asm__ __volatile__(
325 "lock; incl %0"
326 : "=m"(*__hp(addr))
327 :
328 : "memory");
329 return;
330 }
331 #if (BITS_PER_LONG == 64)
332 case 8:
333 {
334 __asm__ __volatile__(
335 "lock; incq %0"
336 : "=m"(*__hp(addr))
337 :
338 : "memory");
339 return;
340 }
341 #endif
342 }
343 /* generate an illegal instruction. Cannot catch this with linker tricks
344 * when optimizations are disabled. */
345 __asm__ __volatile__("ud2");
346 return;
347 }
348
349 #define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
350
351 /* uatomic_dec */
352
353 static inline __attribute__((always_inline))
354 void __uatomic_dec(void *addr, int len)
355 {
356 switch (len) {
357 case 1:
358 {
359 __asm__ __volatile__(
360 "lock; decb %0"
361 : "=m"(*__hp(addr))
362 :
363 : "memory");
364 return;
365 }
366 case 2:
367 {
368 __asm__ __volatile__(
369 "lock; decw %0"
370 : "=m"(*__hp(addr))
371 :
372 : "memory");
373 return;
374 }
375 case 4:
376 {
377 __asm__ __volatile__(
378 "lock; decl %0"
379 : "=m"(*__hp(addr))
380 :
381 : "memory");
382 return;
383 }
384 #if (BITS_PER_LONG == 64)
385 case 8:
386 {
387 __asm__ __volatile__(
388 "lock; decq %0"
389 : "=m"(*__hp(addr))
390 :
391 : "memory");
392 return;
393 }
394 #endif
395 }
396 /* generate an illegal instruction. Cannot catch this with linker tricks
397 * when optimizations are disabled. */
398 __asm__ __volatile__("ud2");
399 return;
400 }
401
402 #define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
403
404 #if ((BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
405 extern int __rcu_cas_avail;
406 extern int __rcu_cas_init(void);
407
408 #define UATOMIC_COMPAT(insn) \
409 ((likely(__rcu_cas_avail > 0)) \
410 ? (_uatomic_##insn) \
411 : ((unlikely(__rcu_cas_avail < 0) \
412 ? ((__rcu_cas_init() > 0) \
413 ? (_uatomic_##insn) \
414 : (compat_uatomic_##insn)) \
415 : (compat_uatomic_##insn))))
416
417 extern unsigned long _compat_uatomic_set(void *addr,
418 unsigned long _new, int len);
419 #define compat_uatomic_set(addr, _new) \
420 ((__typeof__(*(addr))) _compat_uatomic_set((addr), \
421 (unsigned long)(_new), \
422 sizeof(*(addr))))
423
424
425 extern unsigned long _compat_uatomic_xchg(void *addr,
426 unsigned long _new, int len);
427 #define compat_uatomic_xchg(addr, _new) \
428 ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
429 (unsigned long)(_new), \
430 sizeof(*(addr))))
431
432 extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
433 unsigned long _new, int len);
434 #define compat_uatomic_cmpxchg(addr, old, _new) \
435 ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
436 (unsigned long)(old), \
437 (unsigned long)(_new), \
438 sizeof(*(addr))))
439
440 extern unsigned long _compat_uatomic_xchg(void *addr,
441 unsigned long _new, int len);
442 #define compat_uatomic_add_return(addr, v) \
443 ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
444 (unsigned long)(v), \
445 sizeof(*(addr))))
446
447 #define compat_uatomic_add(addr, v) \
448 ((void)compat_uatomic_add_return((addr), (v)))
449 #define compat_uatomic_inc(addr) \
450 (compat_uatomic_add((addr), 1))
451 #define compat_uatomic_dec(addr) \
452 (compat_uatomic_add((addr), -1))
453
454 #else
455 #define UATOMIC_COMPAT(insn) (_uatomic_##insn)
456 #endif
457
458 /* Read is atomic even in compat mode */
459 #define uatomic_set(addr, v) \
460 UATOMIC_COMPAT(set(addr, v))
461
462 #define uatomic_cmpxchg(addr, old, _new) \
463 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
464 #define uatomic_xchg(addr, v) \
465 UATOMIC_COMPAT(xchg(addr, v))
466 #define uatomic_add_return(addr, v) \
467 UATOMIC_COMPAT(add_return(addr, v))
468
469 #define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
470 #define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
471 #define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
472
473 #ifdef __cplusplus
474 }
475 #endif
476
477 #include <urcu/uatomic_generic.h>
478
479 #endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.038229 seconds and 4 git commands to generate.