Update 386 cmpxchg support
[urcu.git] / urcu / uatomic_arch_x86.h
CommitLineData
ec4e58a3
MD
1#ifndef _URCU_ARCH_UATOMIC_X86_H
2#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f
MD
3
4/*
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
9 *
10 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
11 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
12 *
13 * Permission is hereby granted to use or copy this program
14 * for any purpose, provided the above notices are retained on all copies.
15 * Permission to modify the code and to distribute modified code is granted,
16 * provided the above notices are retained, and a notice that the code was
17 * modified is included with the above copyright notice.
18 *
ec4e58a3 19 * Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f
MD
20 * Boehm-Demers-Weiser conservative garbage collector.
21 */
22
ec4e58a3 23#include <urcu/compiler.h>
0fad128b 24
0114ba7f
MD
25#ifndef BITS_PER_LONG
26#define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
27#endif
28
0114ba7f 29/*
0114ba7f
MD
30 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
31 */
32
ec4e58a3 33struct __uatomic_dummy {
cc1be41b
MD
34 unsigned long v[10];
35};
ec4e58a3 36#define __hp(x) ((struct __uatomic_dummy *)(x))
cc1be41b 37
ec4e58a3 38#define uatomic_set(addr, v) \
0fad128b
MD
39do { \
40 ACCESS_ONCE(*(addr)) = (v); \
41} while (0)
42
ec4e58a3 43#define uatomic_read(addr) ACCESS_ONCE(*(addr))
0fad128b 44
cc1be41b
MD
45/* cmpxchg */
46
5dba80f9 47static inline __attribute__((always_inline))
ec4e58a3 48unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b 49 unsigned long _new, int len)
0114ba7f 50{
cc1be41b
MD
51 switch (len) {
52 case 1:
53 {
54 unsigned char result = old;
0fad128b 55
cc1be41b
MD
56 __asm__ __volatile__(
57 "lock; cmpxchgb %2, %1"
58 : "+a"(result), "+m"(*__hp(addr))
59 : "q"((unsigned char)_new)
0114ba7f 60 : "memory");
cc1be41b
MD
61 return result;
62 }
63 case 2:
64 {
65 unsigned short result = old;
0fad128b 66
cc1be41b
MD
67 __asm__ __volatile__(
68 "lock; cmpxchgw %2, %1"
69 : "+a"(result), "+m"(*__hp(addr))
70 : "r"((unsigned short)_new)
71 : "memory");
72 return result;
73 }
74 case 4:
75 {
76 unsigned int result = old;
0fad128b 77
cc1be41b
MD
78 __asm__ __volatile__(
79 "lock; cmpxchgl %2, %1"
80 : "+a"(result), "+m"(*__hp(addr))
81 : "r"((unsigned int)_new)
82 : "memory");
83 return result;
84 }
85#if (BITS_PER_LONG == 64)
86 case 8:
87 {
6edb297e 88 unsigned long result = old;
0fad128b 89
cc1be41b 90 __asm__ __volatile__(
2c5e5fb3 91 "lock; cmpxchgq %2, %1"
cc1be41b
MD
92 : "+a"(result), "+m"(*__hp(addr))
93 : "r"((unsigned long)_new)
94 : "memory");
95 return result;
96 }
97#endif
98 }
99 /* generate an illegal instruction. Cannot catch this with linker tricks
100 * when optimizations are disabled. */
101 __asm__ __volatile__("ud2");
102 return 0;
0114ba7f
MD
103}
104
ec4e58a3
MD
105#define uatomic_cmpxchg(addr, old, _new) \
106 ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
cc1be41b
MD
107 (unsigned long)(_new), \
108 sizeof(*(addr))))
109
110/* xchg */
0114ba7f 111
5dba80f9 112static inline __attribute__((always_inline))
ec4e58a3 113unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f 114{
cc1be41b
MD
115 /* Note: the "xchg" instruction does not need a "lock" prefix. */
116 switch (len) {
117 case 1:
118 {
119 unsigned char result;
120 __asm__ __volatile__(
121 "xchgb %0, %1"
122 : "=q"(result), "+m"(*__hp(addr))
123 : "0" ((unsigned char)val)
124 : "memory");
125 return result;
126 }
127 case 2:
128 {
129 unsigned short result;
130 __asm__ __volatile__(
131 "xchgw %0, %1"
132 : "=r"(result), "+m"(*__hp(addr))
133 : "0" ((unsigned short)val)
134 : "memory");
135 return result;
136 }
137 case 4:
138 {
139 unsigned int result;
140 __asm__ __volatile__(
141 "xchgl %0, %1"
142 : "=r"(result), "+m"(*__hp(addr))
143 : "0" ((unsigned int)val)
144 : "memory");
145 return result;
146 }
147#if (BITS_PER_LONG == 64)
148 case 8:
149 {
150 unsigned long result;
151 __asm__ __volatile__(
0114ba7f 152 "xchgq %0, %1"
cc1be41b
MD
153 : "=r"(result), "+m"(*__hp(addr))
154 : "0" ((unsigned long)val)
0114ba7f 155 : "memory");
cc1be41b
MD
156 return result;
157 }
158#endif
159 }
160 /* generate an illegal instruction. Cannot catch this with linker tricks
161 * when optimizations are disabled. */
162 __asm__ __volatile__("ud2");
163 return 0;
0114ba7f
MD
164}
165
ec4e58a3
MD
166#define uatomic_xchg(addr, v) \
167 ((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
cc1be41b
MD
168 sizeof(*(addr))))
169
ec4e58a3 170/* uatomic_add_return, uatomic_sub_return */
0fad128b
MD
171
172static inline __attribute__((always_inline))
ec4e58a3 173unsigned long _uatomic_add_return(void *addr, unsigned long val,
0fad128b
MD
174 int len)
175{
176 switch (len) {
177 case 1:
178 {
179 unsigned char result = val;
180
181 __asm__ __volatile__(
182 "lock; xaddb %1, %0"
183 : "+m"(*__hp(addr)), "+q" (result)
184 :
185 : "memory");
186 return result + (unsigned char)val;
187 }
188 case 2:
189 {
190 unsigned short result = val;
191
192 __asm__ __volatile__(
193 "lock; xaddw %1, %0"
194 : "+m"(*__hp(addr)), "+r" (result)
195 :
196 : "memory");
197 return result + (unsigned short)val;
198 }
199 case 4:
200 {
201 unsigned int result = val;
202
203 __asm__ __volatile__(
204 "lock; xaddl %1, %0"
205 : "+m"(*__hp(addr)), "+r" (result)
206 :
207 : "memory");
208 return result + (unsigned int)val;
209 }
210#if (BITS_PER_LONG == 64)
211 case 8:
212 {
213 unsigned long result = val;
214
215 __asm__ __volatile__(
216 "lock; xaddq %1, %0"
217 : "+m"(*__hp(addr)), "+r" (result)
218 :
219 : "memory");
220 return result + (unsigned long)val;
221 }
222#endif
223 }
224 /* generate an illegal instruction. Cannot catch this with linker tricks
225 * when optimizations are disabled. */
226 __asm__ __volatile__("ud2");
227 return 0;
228}
229
ec4e58a3
MD
230#define uatomic_add_return(addr, v) \
231 ((__typeof__(*(addr))) _uatomic_add_return((addr), \
0fad128b
MD
232 (unsigned long)(v), \
233 sizeof(*(addr))))
234
ec4e58a3 235#define uatomic_sub_return(addr, v) uatomic_add_return((addr), -(v))
0fad128b 236
ec4e58a3 237/* uatomic_add, uatomic_sub */
0114ba7f 238
5dba80f9 239static inline __attribute__((always_inline))
ec4e58a3 240void _uatomic_add(void *addr, unsigned long val, int len)
0114ba7f
MD
241{
242 switch (len) {
cc1be41b
MD
243 case 1:
244 {
245 __asm__ __volatile__(
246 "lock; addb %1, %0"
247 : "=m"(*__hp(addr))
87322fe8
MD
248 : "iq" ((unsigned char)val)
249 : "memory");
cc1be41b
MD
250 return;
251 }
252 case 2:
253 {
254 __asm__ __volatile__(
255 "lock; addw %1, %0"
256 : "=m"(*__hp(addr))
87322fe8
MD
257 : "ir" ((unsigned short)val)
258 : "memory");
cc1be41b
MD
259 return;
260 }
261 case 4:
262 {
263 __asm__ __volatile__(
264 "lock; addl %1, %0"
265 : "=m"(*__hp(addr))
87322fe8
MD
266 : "ir" ((unsigned int)val)
267 : "memory");
cc1be41b
MD
268 return;
269 }
0114ba7f 270#if (BITS_PER_LONG == 64)
cc1be41b
MD
271 case 8:
272 {
273 __asm__ __volatile__(
274 "lock; addq %1, %0"
275 : "=m"(*__hp(addr))
87322fe8
MD
276 : "er" ((unsigned long)val)
277 : "memory");
cc1be41b
MD
278 return;
279 }
0114ba7f
MD
280#endif
281 }
282 /* generate an illegal instruction. Cannot catch this with linker tricks
283 * when optimizations are disabled. */
284 __asm__ __volatile__("ud2");
a81b8e5e 285 return;
0114ba7f
MD
286}
287
ec4e58a3
MD
288#define uatomic_add(addr, v) \
289 (_uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
0114ba7f 290
ec4e58a3 291#define uatomic_sub(addr, v) uatomic_add((addr), -(v))
2c5e5fb3
MD
292
293
ec4e58a3 294/* uatomic_inc */
2c5e5fb3
MD
295
296static inline __attribute__((always_inline))
ec4e58a3 297void _uatomic_inc(void *addr, int len)
2c5e5fb3
MD
298{
299 switch (len) {
300 case 1:
301 {
302 __asm__ __volatile__(
303 "lock; incb %0"
304 : "=m"(*__hp(addr))
305 :
306 : "memory");
307 return;
308 }
309 case 2:
310 {
311 __asm__ __volatile__(
312 "lock; incw %0"
313 : "=m"(*__hp(addr))
314 :
315 : "memory");
316 return;
317 }
318 case 4:
319 {
320 __asm__ __volatile__(
321 "lock; incl %0"
322 : "=m"(*__hp(addr))
323 :
324 : "memory");
325 return;
326 }
327#if (BITS_PER_LONG == 64)
328 case 8:
329 {
330 __asm__ __volatile__(
331 "lock; incq %0"
332 : "=m"(*__hp(addr))
333 :
334 : "memory");
335 return;
336 }
337#endif
338 }
339 /* generate an illegal instruction. Cannot catch this with linker tricks
340 * when optimizations are disabled. */
341 __asm__ __volatile__("ud2");
342 return;
343}
344
ec4e58a3 345#define uatomic_inc(addr) (_uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3 346
ec4e58a3 347/* uatomic_dec */
2c5e5fb3
MD
348
349static inline __attribute__((always_inline))
ec4e58a3 350void _uatomic_dec(void *addr, int len)
2c5e5fb3
MD
351{
352 switch (len) {
353 case 1:
354 {
355 __asm__ __volatile__(
356 "lock; decb %0"
357 : "=m"(*__hp(addr))
358 :
359 : "memory");
360 return;
361 }
362 case 2:
363 {
364 __asm__ __volatile__(
365 "lock; decw %0"
366 : "=m"(*__hp(addr))
367 :
368 : "memory");
369 return;
370 }
371 case 4:
372 {
373 __asm__ __volatile__(
374 "lock; decl %0"
375 : "=m"(*__hp(addr))
376 :
377 : "memory");
378 return;
379 }
380#if (BITS_PER_LONG == 64)
381 case 8:
382 {
383 __asm__ __volatile__(
384 "lock; decq %0"
385 : "=m"(*__hp(addr))
386 :
387 : "memory");
388 return;
389 }
390#endif
391 }
392 /* generate an illegal instruction. Cannot catch this with linker tricks
393 * when optimizations are disabled. */
394 __asm__ __volatile__("ud2");
395 return;
396}
397
ec4e58a3 398#define uatomic_dec(addr) (_uatomic_dec((addr), sizeof(*(addr))))
0114ba7f 399
7d413817
MD
400#if (BITS_PER_LONG == 64)
401#define URCU_CAS_AVAIL() 1
402#define compat_uatomic_cmpxchg(ptr, old, _new) uatomic_cmpxchg(ptr, old, _new)
403#else
404extern int __urcu_cas_avail;
e1562850
MD
405extern int __urcu_cas_init(void);
406#define URCU_CAS_AVAIL() \
407 ((likely(__urcu_cas_avail > 0)) ? \
408 (1) : \
409 ((unlikely(__urcu_cas_avail < 0) ? \
410 (__urcu_cas_init()) : \
411 (0))))
7d413817
MD
412
413extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
414 unsigned long _new, int len);
415
416#define compat_uatomic_cmpxchg(addr, old, _new) \
417 ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
418 (unsigned long)(_new), \
419 sizeof(*(addr))))
420#endif
421
ec4e58a3 422#endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.063635 seconds and 4 git commands to generate.