urcu/uatomic_arch_x86.h

   1 #ifndef _URCU_ARCH_UATOMIC_X86_H
   2 #define _URCU_ARCH_UATOMIC_X86_H
   3
   4 /*
   5  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
   6  * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
   7  * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
   8  * Copyright (c) 2009      Mathieu Desnoyers
   9  *
  10  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  11  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  12  *
  13  * Permission is hereby granted to use or copy this program
  14  * for any purpose,  provided the above notices are retained on all copies.
  15  * Permission to modify the code and to distribute modified code is granted,
  16  * provided the above notices are retained, and a notice that the code was
  17  * modified is included with the above copyright notice.
  18  *
  19  * Code inspired from libuatomic_ops-1.2, inherited in part from the
  20  * Boehm-Demers-Weiser conservative garbage collector.
  21  */
  22
  23 #include <urcu/compiler.h>
  24
  25 #ifndef BITS_PER_LONG
  26 #define BITS_PER_LONG   (__SIZEOF_LONG__ * 8)
  27 #endif
  28
  29 /*
  30  * Derived from AO_compare_and_swap() and AO_test_and_set_full().
  31  */
  32
  33 struct __uatomic_dummy {
  34         unsigned long v[10];
  35 };
  36 #define __hp(x) ((struct __uatomic_dummy *)(x))
  37
  38 #define uatomic_set(addr, v)                            \
  39 do {                                                    \
  40         ACCESS_ONCE(*(addr)) = (v);                     \
  41 } while (0)
  42
  43 #define uatomic_read(addr)      ACCESS_ONCE(*(addr))
  44
  45 /* cmpxchg */
  46
  47 static inline __attribute__((always_inline))
  48 unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
  49                               unsigned long _new, int len)
  50 {
  51         switch (len) {
  52         case 1:
  53         {
  54                 unsigned char result = old;
  55
  56                 __asm__ __volatile__(
  57                 "lock; cmpxchgb %2, %1"
  58                         : "+a"(result), "+m"(*__hp(addr))
  59                         : "q"((unsigned char)_new)
  60                         : "memory");
  61                 return result;
  62         }
  63         case 2:
  64         {
  65                 unsigned short result = old;
  66
  67                 __asm__ __volatile__(
  68                 "lock; cmpxchgw %2, %1"
  69                         : "+a"(result), "+m"(*__hp(addr))
  70                         : "r"((unsigned short)_new)
  71                         : "memory");
  72                 return result;
  73         }
  74         case 4:
  75         {
  76                 unsigned int result = old;
  77
  78                 __asm__ __volatile__(
  79                 "lock; cmpxchgl %2, %1"
  80                         : "+a"(result), "+m"(*__hp(addr))
  81                         : "r"((unsigned int)_new)
  82                         : "memory");
  83                 return result;
  84         }
  85 #if (BITS_PER_LONG == 64)
  86         case 8:
  87         {
  88                 unsigned long result = old;
  89
  90                 __asm__ __volatile__(
  91                 "lock; cmpxchgq %2, %1"
  92                         : "+a"(result), "+m"(*__hp(addr))
  93                         : "r"((unsigned long)_new)
  94                         : "memory");
  95                 return result;
  96         }
  97 #endif
  98         }
  99         /* generate an illegal instruction. Cannot catch this with linker tricks
 100          * when optimizations are disabled. */
 101         __asm__ __volatile__("ud2");
 102         return 0;
 103 }
 104
 105 #define uatomic_cmpxchg(addr, old, _new)                                    \
 106         ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
 107                                                 (unsigned long)(_new),      \
 108                                                 sizeof(*(addr))))
 109
 110 /* xchg */
 111
 112 static inline __attribute__((always_inline))
 113 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
 114 {
 115         /* Note: the "xchg" instruction does not need a "lock" prefix. */
 116         switch (len) {
 117         case 1:
 118         {
 119                 unsigned char result;
 120                 __asm__ __volatile__(
 121                 "xchgb %0, %1"
 122                         : "=q"(result), "+m"(*__hp(addr))
 123                         : "0" ((unsigned char)val)
 124                         : "memory");
 125                 return result;
 126         }
 127         case 2:
 128         {
 129                 unsigned short result;
 130                 __asm__ __volatile__(
 131                 "xchgw %0, %1"
 132                         : "=r"(result), "+m"(*__hp(addr))
 133                         : "0" ((unsigned short)val)
 134                         : "memory");
 135                 return result;
 136         }
 137         case 4:
 138         {
 139                 unsigned int result;
 140                 __asm__ __volatile__(
 141                 "xchgl %0, %1"
 142                         : "=r"(result), "+m"(*__hp(addr))
 143                         : "0" ((unsigned int)val)
 144                         : "memory");
 145                 return result;
 146         }
 147 #if (BITS_PER_LONG == 64)
 148         case 8:
 149         {
 150                 unsigned long result;
 151                 __asm__ __volatile__(
 152                 "xchgq %0, %1"
 153                         : "=r"(result), "+m"(*__hp(addr))
 154                         : "0" ((unsigned long)val)
 155                         : "memory");
 156                 return result;
 157         }
 158 #endif
 159         }
 160         /* generate an illegal instruction. Cannot catch this with linker tricks
 161          * when optimizations are disabled. */
 162         __asm__ __volatile__("ud2");
 163         return 0;
 164 }
 165
 166 #define uatomic_xchg(addr, v)                                               \
 167         ((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
 168                                                 sizeof(*(addr))))
 169
 170 /* uatomic_add_return, uatomic_sub_return */
 171
 172 static inline __attribute__((always_inline))
 173 unsigned long _uatomic_add_return(void *addr, unsigned long val,
 174                                  int len)
 175 {
 176         switch (len) {
 177         case 1:
 178         {
 179                 unsigned char result = val;
 180
 181                 __asm__ __volatile__(
 182                 "lock; xaddb %1, %0"
 183                         : "+m"(*__hp(addr)), "+q" (result)
 184                         :
 185                         : "memory");
 186                 return result + (unsigned char)val;
 187         }
 188         case 2:
 189         {
 190                 unsigned short result = val;
 191
 192                 __asm__ __volatile__(
 193                 "lock; xaddw %1, %0"
 194                         : "+m"(*__hp(addr)), "+r" (result)
 195                         :
 196                         : "memory");
 197                 return result + (unsigned short)val;
 198         }
 199         case 4:
 200         {
 201                 unsigned int result = val;
 202
 203                 __asm__ __volatile__(
 204                 "lock; xaddl %1, %0"
 205                         : "+m"(*__hp(addr)), "+r" (result)
 206                         :
 207                         : "memory");
 208                 return result + (unsigned int)val;
 209         }
 210 #if (BITS_PER_LONG == 64)
 211         case 8:
 212         {
 213                 unsigned long result = val;
 214
 215                 __asm__ __volatile__(
 216                 "lock; xaddq %1, %0"
 217                         : "+m"(*__hp(addr)), "+r" (result)
 218                         :
 219                         : "memory");
 220                 return result + (unsigned long)val;
 221         }
 222 #endif
 223         }
 224         /* generate an illegal instruction. Cannot catch this with linker tricks
 225          * when optimizations are disabled. */
 226         __asm__ __volatile__("ud2");
 227         return 0;
 228 }
 229
 230 #define uatomic_add_return(addr, v)                                     \
 231         ((__typeof__(*(addr))) _uatomic_add_return((addr),              \
 232                                                   (unsigned long)(v),   \
 233                                                   sizeof(*(addr))))
 234
 235 #define uatomic_sub_return(addr, v)     uatomic_add_return((addr), -(v))
 236
 237 /* uatomic_add, uatomic_sub */
 238
 239 static inline __attribute__((always_inline))
 240 void _uatomic_add(void *addr, unsigned long val, int len)
 241 {
 242         switch (len) {
 243         case 1:
 244         {
 245                 __asm__ __volatile__(
 246                 "lock; addb %1, %0"
 247                         : "=m"(*__hp(addr))
 248                         : "iq" ((unsigned char)val)
 249                         : "memory");
 250                 return;
 251         }
 252         case 2:
 253         {
 254                 __asm__ __volatile__(
 255                 "lock; addw %1, %0"
 256                         : "=m"(*__hp(addr))
 257                         : "ir" ((unsigned short)val)
 258                         : "memory");
 259                 return;
 260         }
 261         case 4:
 262         {
 263                 __asm__ __volatile__(
 264                 "lock; addl %1, %0"
 265                         : "=m"(*__hp(addr))
 266                         : "ir" ((unsigned int)val)
 267                         : "memory");
 268                 return;
 269         }
 270 #if (BITS_PER_LONG == 64)
 271         case 8:
 272         {
 273                 __asm__ __volatile__(
 274                 "lock; addq %1, %0"
 275                         : "=m"(*__hp(addr))
 276                         : "er" ((unsigned long)val)
 277                         : "memory");
 278                 return;
 279         }
 280 #endif
 281         }
 282         /* generate an illegal instruction. Cannot catch this with linker tricks
 283          * when optimizations are disabled. */
 284         __asm__ __volatile__("ud2");
 285         return;
 286 }
 287
 288 #define uatomic_add(addr, v)                                               \
 289         (_uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
 290
 291 #define uatomic_sub(addr, v)    uatomic_add((addr), -(v))
 292
 293
 294 /* uatomic_inc */
 295
 296 static inline __attribute__((always_inline))
 297 void _uatomic_inc(void *addr, int len)
 298 {
 299         switch (len) {
 300         case 1:
 301         {
 302                 __asm__ __volatile__(
 303                 "lock; incb %0"
 304                         : "=m"(*__hp(addr))
 305                         :
 306                         : "memory");
 307                 return;
 308         }
 309         case 2:
 310         {
 311                 __asm__ __volatile__(
 312                 "lock; incw %0"
 313                         : "=m"(*__hp(addr))
 314                         :
 315                         : "memory");
 316                 return;
 317         }
 318         case 4:
 319         {
 320                 __asm__ __volatile__(
 321                 "lock; incl %0"
 322                         : "=m"(*__hp(addr))
 323                         :
 324                         : "memory");
 325                 return;
 326         }
 327 #if (BITS_PER_LONG == 64)
 328         case 8:
 329         {
 330                 __asm__ __volatile__(
 331                 "lock; incq %0"
 332                         : "=m"(*__hp(addr))
 333                         :
 334                         : "memory");
 335                 return;
 336         }
 337 #endif
 338         }
 339         /* generate an illegal instruction. Cannot catch this with linker tricks
 340          * when optimizations are disabled. */
 341         __asm__ __volatile__("ud2");
 342         return;
 343 }
 344
 345 #define uatomic_inc(addr)       (_uatomic_inc((addr), sizeof(*(addr))))
 346
 347 /* uatomic_dec */
 348
 349 static inline __attribute__((always_inline))
 350 void _uatomic_dec(void *addr, int len)
 351 {
 352         switch (len) {
 353         case 1:
 354         {
 355                 __asm__ __volatile__(
 356                 "lock; decb %0"
 357                         : "=m"(*__hp(addr))
 358                         :
 359                         : "memory");
 360                 return;
 361         }
 362         case 2:
 363         {
 364                 __asm__ __volatile__(
 365                 "lock; decw %0"
 366                         : "=m"(*__hp(addr))
 367                         :
 368                         : "memory");
 369                 return;
 370         }
 371         case 4:
 372         {
 373                 __asm__ __volatile__(
 374                 "lock; decl %0"
 375                         : "=m"(*__hp(addr))
 376                         :
 377                         : "memory");
 378                 return;
 379         }
 380 #if (BITS_PER_LONG == 64)
 381         case 8:
 382         {
 383                 __asm__ __volatile__(
 384                 "lock; decq %0"
 385                         : "=m"(*__hp(addr))
 386                         :
 387                         : "memory");
 388                 return;
 389         }
 390 #endif
 391         }
 392         /* generate an illegal instruction. Cannot catch this with linker tricks
 393          * when optimizations are disabled. */
 394         __asm__ __volatile__("ud2");
 395         return;
 396 }
 397
 398 #define uatomic_dec(addr)       (_uatomic_dec((addr), sizeof(*(addr))))
 399
 400 #endif /* _URCU_ARCH_UATOMIC_X86_H */