uatomic: update atomic set/read, use STORE_SHARED/LOAD_SHARED
[urcu.git] / urcu / uatomic_arch_x86.h
CommitLineData
ec4e58a3
MD
1#ifndef _URCU_ARCH_UATOMIC_X86_H
2#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f
MD
3
4/*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 *
10 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
11 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
12 *
13 * Permission is hereby granted to use or copy this program
14 * for any purpose, provided the above notices are retained on all copies.
15 * Permission to modify the code and to distribute modified code is granted,
16 * provided the above notices are retained, and a notice that the code was
17 * modified is included with the above copyright notice.
18 *
ec4e58a3 19 * Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f
MD
20 * Boehm-Demers-Weiser conservative garbage collector.
21 */
22
ec4e58a3 23#include <urcu/compiler.h>
0fad128b 24
618c4ae1
MD
25#ifndef __SIZEOF_LONG__
26#if defined(__x86_64__) || defined(__amd64__)
27#define __SIZEOF_LONG__ 8
28#else
29#define __SIZEOF_LONG__ 4
30#endif
31#endif
32
0114ba7f
MD
33#ifndef BITS_PER_LONG
34#define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
35#endif
36
0114ba7f 37/*
0114ba7f
MD
38 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
39 */
40
ec4e58a3 41struct __uatomic_dummy {
cc1be41b
MD
42 unsigned long v[10];
43};
ec4e58a3 44#define __hp(x) ((struct __uatomic_dummy *)(x))
cc1be41b 45
ec4e58a3 46#define uatomic_set(addr, v) \
0fad128b
MD
47do { \
48 ACCESS_ONCE(*(addr)) = (v); \
49} while (0)
50
ec4e58a3 51#define uatomic_read(addr) ACCESS_ONCE(*(addr))
0fad128b 52
cc1be41b
MD
53/* cmpxchg */
54
5dba80f9 55static inline __attribute__((always_inline))
ec4e58a3 56unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b 57 unsigned long _new, int len)
0114ba7f 58{
cc1be41b
MD
59 switch (len) {
60 case 1:
61 {
62 unsigned char result = old;
0fad128b 63
cc1be41b
MD
64 __asm__ __volatile__(
65 "lock; cmpxchgb %2, %1"
66 : "+a"(result), "+m"(*__hp(addr))
67 : "q"((unsigned char)_new)
0114ba7f 68 : "memory");
cc1be41b
MD
69 return result;
70 }
71 case 2:
72 {
73 unsigned short result = old;
0fad128b 74
cc1be41b
MD
75 __asm__ __volatile__(
76 "lock; cmpxchgw %2, %1"
77 : "+a"(result), "+m"(*__hp(addr))
78 : "r"((unsigned short)_new)
79 : "memory");
80 return result;
81 }
82 case 4:
83 {
84 unsigned int result = old;
0fad128b 85
cc1be41b
MD
86 __asm__ __volatile__(
87 "lock; cmpxchgl %2, %1"
88 : "+a"(result), "+m"(*__hp(addr))
89 : "r"((unsigned int)_new)
90 : "memory");
91 return result;
92 }
93#if (BITS_PER_LONG == 64)
94 case 8:
95 {
6edb297e 96 unsigned long result = old;
0fad128b 97
cc1be41b 98 __asm__ __volatile__(
2c5e5fb3 99 "lock; cmpxchgq %2, %1"
cc1be41b
MD
100 : "+a"(result), "+m"(*__hp(addr))
101 : "r"((unsigned long)_new)
102 : "memory");
103 return result;
104 }
105#endif
106 }
107 /* generate an illegal instruction. Cannot catch this with linker tricks
108 * when optimizations are disabled. */
109 __asm__ __volatile__("ud2");
110 return 0;
0114ba7f
MD
111}
112
ec4e58a3
MD
113#define uatomic_cmpxchg(addr, old, _new) \
114 ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
cc1be41b
MD
115 (unsigned long)(_new), \
116 sizeof(*(addr))))
117
118/* xchg */
0114ba7f 119
5dba80f9 120static inline __attribute__((always_inline))
ec4e58a3 121unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f 122{
cc1be41b
MD
123 /* Note: the "xchg" instruction does not need a "lock" prefix. */
124 switch (len) {
125 case 1:
126 {
127 unsigned char result;
128 __asm__ __volatile__(
129 "xchgb %0, %1"
130 : "=q"(result), "+m"(*__hp(addr))
131 : "0" ((unsigned char)val)
132 : "memory");
133 return result;
134 }
135 case 2:
136 {
137 unsigned short result;
138 __asm__ __volatile__(
139 "xchgw %0, %1"
140 : "=r"(result), "+m"(*__hp(addr))
141 : "0" ((unsigned short)val)
142 : "memory");
143 return result;
144 }
145 case 4:
146 {
147 unsigned int result;
148 __asm__ __volatile__(
149 "xchgl %0, %1"
150 : "=r"(result), "+m"(*__hp(addr))
151 : "0" ((unsigned int)val)
152 : "memory");
153 return result;
154 }
155#if (BITS_PER_LONG == 64)
156 case 8:
157 {
158 unsigned long result;
159 __asm__ __volatile__(
0114ba7f 160 "xchgq %0, %1"
cc1be41b
MD
161 : "=r"(result), "+m"(*__hp(addr))
162 : "0" ((unsigned long)val)
0114ba7f 163 : "memory");
cc1be41b
MD
164 return result;
165 }
166#endif
167 }
168 /* generate an illegal instruction. Cannot catch this with linker tricks
169 * when optimizations are disabled. */
170 __asm__ __volatile__("ud2");
171 return 0;
0114ba7f
MD
172}
173
ec4e58a3
MD
174#define uatomic_xchg(addr, v) \
175 ((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
cc1be41b
MD
176 sizeof(*(addr))))
177
ec4e58a3 178/* uatomic_add_return, uatomic_sub_return */
0fad128b
MD
179
180static inline __attribute__((always_inline))
ec4e58a3 181unsigned long _uatomic_add_return(void *addr, unsigned long val,
0fad128b
MD
182 int len)
183{
184 switch (len) {
185 case 1:
186 {
187 unsigned char result = val;
188
189 __asm__ __volatile__(
190 "lock; xaddb %1, %0"
191 : "+m"(*__hp(addr)), "+q" (result)
192 :
193 : "memory");
194 return result + (unsigned char)val;
195 }
196 case 2:
197 {
198 unsigned short result = val;
199
200 __asm__ __volatile__(
201 "lock; xaddw %1, %0"
202 : "+m"(*__hp(addr)), "+r" (result)
203 :
204 : "memory");
205 return result + (unsigned short)val;
206 }
207 case 4:
208 {
209 unsigned int result = val;
210
211 __asm__ __volatile__(
212 "lock; xaddl %1, %0"
213 : "+m"(*__hp(addr)), "+r" (result)
214 :
215 : "memory");
216 return result + (unsigned int)val;
217 }
218#if (BITS_PER_LONG == 64)
219 case 8:
220 {
221 unsigned long result = val;
222
223 __asm__ __volatile__(
224 "lock; xaddq %1, %0"
225 : "+m"(*__hp(addr)), "+r" (result)
226 :
227 : "memory");
228 return result + (unsigned long)val;
229 }
230#endif
231 }
232 /* generate an illegal instruction. Cannot catch this with linker tricks
233 * when optimizations are disabled. */
234 __asm__ __volatile__("ud2");
235 return 0;
236}
237
ec4e58a3
MD
238#define uatomic_add_return(addr, v) \
239 ((__typeof__(*(addr))) _uatomic_add_return((addr), \
0fad128b
MD
240 (unsigned long)(v), \
241 sizeof(*(addr))))
242
ec4e58a3 243#define uatomic_sub_return(addr, v) uatomic_add_return((addr), -(v))
0fad128b 244
ec4e58a3 245/* uatomic_add, uatomic_sub */
0114ba7f 246
5dba80f9 247static inline __attribute__((always_inline))
ec4e58a3 248void _uatomic_add(void *addr, unsigned long val, int len)
0114ba7f
MD
249{
250 switch (len) {
cc1be41b
MD
251 case 1:
252 {
253 __asm__ __volatile__(
254 "lock; addb %1, %0"
255 : "=m"(*__hp(addr))
87322fe8
MD
256 : "iq" ((unsigned char)val)
257 : "memory");
cc1be41b
MD
258 return;
259 }
260 case 2:
261 {
262 __asm__ __volatile__(
263 "lock; addw %1, %0"
264 : "=m"(*__hp(addr))
87322fe8
MD
265 : "ir" ((unsigned short)val)
266 : "memory");
cc1be41b
MD
267 return;
268 }
269 case 4:
270 {
271 __asm__ __volatile__(
272 "lock; addl %1, %0"
273 : "=m"(*__hp(addr))
87322fe8
MD
274 : "ir" ((unsigned int)val)
275 : "memory");
cc1be41b
MD
276 return;
277 }
0114ba7f 278#if (BITS_PER_LONG == 64)
cc1be41b
MD
279 case 8:
280 {
281 __asm__ __volatile__(
282 "lock; addq %1, %0"
283 : "=m"(*__hp(addr))
87322fe8
MD
284 : "er" ((unsigned long)val)
285 : "memory");
cc1be41b
MD
286 return;
287 }
0114ba7f
MD
288#endif
289 }
290 /* generate an illegal instruction. Cannot catch this with linker tricks
291 * when optimizations are disabled. */
292 __asm__ __volatile__("ud2");
a81b8e5e 293 return;
0114ba7f
MD
294}
295
ec4e58a3
MD
296#define uatomic_add(addr, v) \
297 (_uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
0114ba7f 298
ec4e58a3 299#define uatomic_sub(addr, v) uatomic_add((addr), -(v))
2c5e5fb3
MD
300
301
ec4e58a3 302/* uatomic_inc */
2c5e5fb3
MD
303
304static inline __attribute__((always_inline))
ec4e58a3 305void _uatomic_inc(void *addr, int len)
2c5e5fb3
MD
306{
307 switch (len) {
308 case 1:
309 {
310 __asm__ __volatile__(
311 "lock; incb %0"
312 : "=m"(*__hp(addr))
313 :
314 : "memory");
315 return;
316 }
317 case 2:
318 {
319 __asm__ __volatile__(
320 "lock; incw %0"
321 : "=m"(*__hp(addr))
322 :
323 : "memory");
324 return;
325 }
326 case 4:
327 {
328 __asm__ __volatile__(
329 "lock; incl %0"
330 : "=m"(*__hp(addr))
331 :
332 : "memory");
333 return;
334 }
335#if (BITS_PER_LONG == 64)
336 case 8:
337 {
338 __asm__ __volatile__(
339 "lock; incq %0"
340 : "=m"(*__hp(addr))
341 :
342 : "memory");
343 return;
344 }
345#endif
346 }
347 /* generate an illegal instruction. Cannot catch this with linker tricks
348 * when optimizations are disabled. */
349 __asm__ __volatile__("ud2");
350 return;
351}
352
ec4e58a3 353#define uatomic_inc(addr) (_uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3 354
ec4e58a3 355/* uatomic_dec */
2c5e5fb3
MD
356
357static inline __attribute__((always_inline))
ec4e58a3 358void _uatomic_dec(void *addr, int len)
2c5e5fb3
MD
359{
360 switch (len) {
361 case 1:
362 {
363 __asm__ __volatile__(
364 "lock; decb %0"
365 : "=m"(*__hp(addr))
366 :
367 : "memory");
368 return;
369 }
370 case 2:
371 {
372 __asm__ __volatile__(
373 "lock; decw %0"
374 : "=m"(*__hp(addr))
375 :
376 : "memory");
377 return;
378 }
379 case 4:
380 {
381 __asm__ __volatile__(
382 "lock; decl %0"
383 : "=m"(*__hp(addr))
384 :
385 : "memory");
386 return;
387 }
388#if (BITS_PER_LONG == 64)
389 case 8:
390 {
391 __asm__ __volatile__(
392 "lock; decq %0"
393 : "=m"(*__hp(addr))
394 :
395 : "memory");
396 return;
397 }
398#endif
399 }
400 /* generate an illegal instruction. Cannot catch this with linker tricks
401 * when optimizations are disabled. */
402 __asm__ __volatile__("ud2");
403 return;
404}
405
ec4e58a3 406#define uatomic_dec(addr) (_uatomic_dec((addr), sizeof(*(addr))))
0114ba7f 407
7d413817
MD
408#if (BITS_PER_LONG == 64)
409#define URCU_CAS_AVAIL() 1
410#define compat_uatomic_cmpxchg(ptr, old, _new) uatomic_cmpxchg(ptr, old, _new)
411#else
412extern int __urcu_cas_avail;
e1562850
MD
413extern int __urcu_cas_init(void);
414#define URCU_CAS_AVAIL() \
415 ((likely(__urcu_cas_avail > 0)) ? \
416 (1) : \
417 ((unlikely(__urcu_cas_avail < 0) ? \
418 (__urcu_cas_init()) : \
419 (0))))
7d413817
MD
420
421extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
422 unsigned long _new, int len);
423
424#define compat_uatomic_cmpxchg(addr, old, _new) \
425 ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
426 (unsigned long)(_new), \
427 sizeof(*(addr))))
428#endif
429
ec4e58a3 430#endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.040888 seconds and 4 git commands to generate.