urcu/uatomic_arch_x86.h

   1 #ifndef _URCU_ARCH_UATOMIC_X86_H
   2 #define _URCU_ARCH_UATOMIC_X86_H
   3
   4 /*
   5  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
   6  * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
   7  * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
   8  * Copyright (c) 2009      Mathieu Desnoyers
   9  *
  10  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  11  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  12  *
  13  * Permission is hereby granted to use or copy this program
  14  * for any purpose,  provided the above notices are retained on all copies.
  15  * Permission to modify the code and to distribute modified code is granted,
  16  * provided the above notices are retained, and a notice that the code was
  17  * modified is included with the above copyright notice.
  18  *
  19  * Code inspired from libuatomic_ops-1.2, inherited in part from the
  20  * Boehm-Demers-Weiser conservative garbage collector.
  21  */
  22
  23 #include <urcu/compiler.h>
  24 #include <urcu/system.h>
  25
  26 #define UATOMIC_HAS_ATOMIC_BYTE
  27 #define UATOMIC_HAS_ATOMIC_SHORT
  28
  29 #ifdef __cplusplus
  30 extern "C" {
  31 #endif
  32
  33 #ifndef __SIZEOF_LONG__
  34 #if defined(__x86_64__) || defined(__amd64__)
  35 #define __SIZEOF_LONG__ 8
  36 #else
  37 #define __SIZEOF_LONG__ 4
  38 #endif
  39 #endif
  40
  41 #ifndef BITS_PER_LONG
  42 #define BITS_PER_LONG   (__SIZEOF_LONG__ * 8)
  43 #endif
  44
  45 /*
  46  * Derived from AO_compare_and_swap() and AO_test_and_set_full().
  47  */
  48
  49 struct __uatomic_dummy {
  50         unsigned long v[10];
  51 };
  52 #define __hp(x) ((struct __uatomic_dummy *)(x))
  53
  54 #define _uatomic_set(addr, v)   STORE_SHARED(*(addr), (v))
  55
  56 /* cmpxchg */
  57
  58 static inline __attribute__((always_inline))
  59 unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
  60                               unsigned long _new, int len)
  61 {
  62         switch (len) {
  63         case 1:
  64         {
  65                 unsigned char result = old;
  66
  67                 __asm__ __volatile__(
  68                 "lock; cmpxchgb %2, %1"
  69                         : "+a"(result), "+m"(*__hp(addr))
  70                         : "q"((unsigned char)_new)
  71                         : "memory");
  72                 return result;
  73         }
  74         case 2:
  75         {
  76                 unsigned short result = old;
  77
  78                 __asm__ __volatile__(
  79                 "lock; cmpxchgw %2, %1"
  80                         : "+a"(result), "+m"(*__hp(addr))
  81                         : "r"((unsigned short)_new)
  82                         : "memory");
  83                 return result;
  84         }
  85         case 4:
  86         {
  87                 unsigned int result = old;
  88
  89                 __asm__ __volatile__(
  90                 "lock; cmpxchgl %2, %1"
  91                         : "+a"(result), "+m"(*__hp(addr))
  92                         : "r"((unsigned int)_new)
  93                         : "memory");
  94                 return result;
  95         }
  96 #if (BITS_PER_LONG == 64)
  97         case 8:
  98         {
  99                 unsigned long result = old;
 100
 101                 __asm__ __volatile__(
 102                 "lock; cmpxchgq %2, %1"
 103                         : "+a"(result), "+m"(*__hp(addr))
 104                         : "r"((unsigned long)_new)
 105                         : "memory");
 106                 return result;
 107         }
 108 #endif
 109         }
 110         /* generate an illegal instruction. Cannot catch this with linker tricks
 111          * when optimizations are disabled. */
 112         __asm__ __volatile__("ud2");
 113         return 0;
 114 }
 115
 116 #define _uatomic_cmpxchg(addr, old, _new)                                     \
 117         ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), (unsigned long)(old),\
 118                                                 (unsigned long)(_new),        \
 119                                                 sizeof(*(addr))))
 120
 121 /* xchg */
 122
 123 static inline __attribute__((always_inline))
 124 unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
 125 {
 126         /* Note: the "xchg" instruction does not need a "lock" prefix. */
 127         switch (len) {
 128         case 1:
 129         {
 130                 unsigned char result;
 131                 __asm__ __volatile__(
 132                 "xchgb %0, %1"
 133                         : "=q"(result), "+m"(*__hp(addr))
 134                         : "0" ((unsigned char)val)
 135                         : "memory");
 136                 return result;
 137         }
 138         case 2:
 139         {
 140                 unsigned short result;
 141                 __asm__ __volatile__(
 142                 "xchgw %0, %1"
 143                         : "=r"(result), "+m"(*__hp(addr))
 144                         : "0" ((unsigned short)val)
 145                         : "memory");
 146                 return result;
 147         }
 148         case 4:
 149         {
 150                 unsigned int result;
 151                 __asm__ __volatile__(
 152                 "xchgl %0, %1"
 153                         : "=r"(result), "+m"(*__hp(addr))
 154                         : "0" ((unsigned int)val)
 155                         : "memory");
 156                 return result;
 157         }
 158 #if (BITS_PER_LONG == 64)
 159         case 8:
 160         {
 161                 unsigned long result;
 162                 __asm__ __volatile__(
 163                 "xchgq %0, %1"
 164                         : "=r"(result), "+m"(*__hp(addr))
 165                         : "0" ((unsigned long)val)
 166                         : "memory");
 167                 return result;
 168         }
 169 #endif
 170         }
 171         /* generate an illegal instruction. Cannot catch this with linker tricks
 172          * when optimizations are disabled. */
 173         __asm__ __volatile__("ud2");
 174         return 0;
 175 }
 176
 177 #define _uatomic_xchg(addr, v)                                                \
 178         ((__typeof__(*(addr))) __uatomic_exchange((addr), (unsigned long)(v), \
 179                                                 sizeof(*(addr))))
 180
 181 /* uatomic_add_return */
 182
 183 static inline __attribute__((always_inline))
 184 unsigned long __uatomic_add_return(void *addr, unsigned long val,
 185                                  int len)
 186 {
 187         switch (len) {
 188         case 1:
 189         {
 190                 unsigned char result = val;
 191
 192                 __asm__ __volatile__(
 193                 "lock; xaddb %1, %0"
 194                         : "+m"(*__hp(addr)), "+q" (result)
 195                         :
 196                         : "memory");
 197                 return result + (unsigned char)val;
 198         }
 199         case 2:
 200         {
 201                 unsigned short result = val;
 202
 203                 __asm__ __volatile__(
 204                 "lock; xaddw %1, %0"
 205                         : "+m"(*__hp(addr)), "+r" (result)
 206                         :
 207                         : "memory");
 208                 return result + (unsigned short)val;
 209         }
 210         case 4:
 211         {
 212                 unsigned int result = val;
 213
 214                 __asm__ __volatile__(
 215                 "lock; xaddl %1, %0"
 216                         : "+m"(*__hp(addr)), "+r" (result)
 217                         :
 218                         : "memory");
 219                 return result + (unsigned int)val;
 220         }
 221 #if (BITS_PER_LONG == 64)
 222         case 8:
 223         {
 224                 unsigned long result = val;
 225
 226                 __asm__ __volatile__(
 227                 "lock; xaddq %1, %0"
 228                         : "+m"(*__hp(addr)), "+r" (result)
 229                         :
 230                         : "memory");
 231                 return result + (unsigned long)val;
 232         }
 233 #endif
 234         }
 235         /* generate an illegal instruction. Cannot catch this with linker tricks
 236          * when optimizations are disabled. */
 237         __asm__ __volatile__("ud2");
 238         return 0;
 239 }
 240
 241 #define _uatomic_add_return(addr, v)                                    \
 242         ((__typeof__(*(addr))) __uatomic_add_return((addr),             \
 243                                                   (unsigned long)(v),   \
 244                                                   sizeof(*(addr))))
 245
 246 /* uatomic_add */
 247
 248 static inline __attribute__((always_inline))
 249 void __uatomic_add(void *addr, unsigned long val, int len)
 250 {
 251         switch (len) {
 252         case 1:
 253         {
 254                 __asm__ __volatile__(
 255                 "lock; addb %1, %0"
 256                         : "=m"(*__hp(addr))
 257                         : "iq" ((unsigned char)val)
 258                         : "memory");
 259                 return;
 260         }
 261         case 2:
 262         {
 263                 __asm__ __volatile__(
 264                 "lock; addw %1, %0"
 265                         : "=m"(*__hp(addr))
 266                         : "ir" ((unsigned short)val)
 267                         : "memory");
 268                 return;
 269         }
 270         case 4:
 271         {
 272                 __asm__ __volatile__(
 273                 "lock; addl %1, %0"
 274                         : "=m"(*__hp(addr))
 275                         : "ir" ((unsigned int)val)
 276                         : "memory");
 277                 return;
 278         }
 279 #if (BITS_PER_LONG == 64)
 280         case 8:
 281         {
 282                 __asm__ __volatile__(
 283                 "lock; addq %1, %0"
 284                         : "=m"(*__hp(addr))
 285                         : "er" ((unsigned long)val)
 286                         : "memory");
 287                 return;
 288         }
 289 #endif
 290         }
 291         /* generate an illegal instruction. Cannot catch this with linker tricks
 292          * when optimizations are disabled. */
 293         __asm__ __volatile__("ud2");
 294         return;
 295 }
 296
 297 #define _uatomic_add(addr, v)                                              \
 298         (__uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
 299
 300
 301 /* uatomic_inc */
 302
 303 static inline __attribute__((always_inline))
 304 void __uatomic_inc(void *addr, int len)
 305 {
 306         switch (len) {
 307         case 1:
 308         {
 309                 __asm__ __volatile__(
 310                 "lock; incb %0"
 311                         : "=m"(*__hp(addr))
 312                         :
 313                         : "memory");
 314                 return;
 315         }
 316         case 2:
 317         {
 318                 __asm__ __volatile__(
 319                 "lock; incw %0"
 320                         : "=m"(*__hp(addr))
 321                         :
 322                         : "memory");
 323                 return;
 324         }
 325         case 4:
 326         {
 327                 __asm__ __volatile__(
 328                 "lock; incl %0"
 329                         : "=m"(*__hp(addr))
 330                         :
 331                         : "memory");
 332                 return;
 333         }
 334 #if (BITS_PER_LONG == 64)
 335         case 8:
 336         {
 337                 __asm__ __volatile__(
 338                 "lock; incq %0"
 339                         : "=m"(*__hp(addr))
 340                         :
 341                         : "memory");
 342                 return;
 343         }
 344 #endif
 345         }
 346         /* generate an illegal instruction. Cannot catch this with linker tricks
 347          * when optimizations are disabled. */
 348         __asm__ __volatile__("ud2");
 349         return;
 350 }
 351
 352 #define _uatomic_inc(addr)      (__uatomic_inc((addr), sizeof(*(addr))))
 353
 354 /* uatomic_dec */
 355
 356 static inline __attribute__((always_inline))
 357 void __uatomic_dec(void *addr, int len)
 358 {
 359         switch (len) {
 360         case 1:
 361         {
 362                 __asm__ __volatile__(
 363                 "lock; decb %0"
 364                         : "=m"(*__hp(addr))
 365                         :
 366                         : "memory");
 367                 return;
 368         }
 369         case 2:
 370         {
 371                 __asm__ __volatile__(
 372                 "lock; decw %0"
 373                         : "=m"(*__hp(addr))
 374                         :
 375                         : "memory");
 376                 return;
 377         }
 378         case 4:
 379         {
 380                 __asm__ __volatile__(
 381                 "lock; decl %0"
 382                         : "=m"(*__hp(addr))
 383                         :
 384                         : "memory");
 385                 return;
 386         }
 387 #if (BITS_PER_LONG == 64)
 388         case 8:
 389         {
 390                 __asm__ __volatile__(
 391                 "lock; decq %0"
 392                         : "=m"(*__hp(addr))
 393                         :
 394                         : "memory");
 395                 return;
 396         }
 397 #endif
 398         }
 399         /* generate an illegal instruction. Cannot catch this with linker tricks
 400          * when optimizations are disabled. */
 401         __asm__ __volatile__("ud2");
 402         return;
 403 }
 404
 405 #define _uatomic_dec(addr)      (__uatomic_dec((addr), sizeof(*(addr))))
 406
 407 #if ((BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
 408 extern int __rcu_cas_avail;
 409 extern int __rcu_cas_init(void);
 410
 411 #define UATOMIC_COMPAT(insn)                                                    \
 412         ((likely(__rcu_cas_avail > 0))                                          \
 413         ? (_uatomic_##insn)                                                     \
 414                 : ((unlikely(__rcu_cas_avail < 0)                               \
 415                         ? ((__rcu_cas_init() > 0)                               \
 416                                 ? (_uatomic_##insn)                             \
 417                                 : (compat_uatomic_##insn))                      \
 418                         : (compat_uatomic_##insn))))
 419
 420 extern unsigned long _compat_uatomic_set(void *addr,
 421                                          unsigned long _new, int len);
 422 #define compat_uatomic_set(addr, _new)                                         \
 423         ((__typeof__(*(addr))) _compat_uatomic_set((addr),                     \
 424                                                 (unsigned long)(_new),         \
 425                                                 sizeof(*(addr))))
 426
 427
 428 extern unsigned long _compat_uatomic_xchg(void *addr,
 429                                           unsigned long _new, int len);
 430 #define compat_uatomic_xchg(addr, _new)                                        \
 431         ((__typeof__(*(addr))) _compat_uatomic_xchg((addr),                    \
 432                                                 (unsigned long)(_new),         \
 433                                                 sizeof(*(addr))))
 434
 435 extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
 436                                              unsigned long _new, int len);
 437 #define compat_uatomic_cmpxchg(addr, old, _new)                                \
 438         ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),                 \
 439                                                 (unsigned long)(old),          \
 440                                                 (unsigned long)(_new),         \
 441                                                 sizeof(*(addr))))
 442
 443 extern unsigned long _compat_uatomic_xchg(void *addr,
 444                                           unsigned long _new, int len);
 445 #define compat_uatomic_add_return(addr, v)                                     \
 446         ((__typeof__(*(addr))) _compat_uatomic_add_return((addr),              \
 447                                                 (unsigned long)(v),            \
 448                                                 sizeof(*(addr))))
 449
 450 #define compat_uatomic_add(addr, v)                                            \
 451                 ((void)compat_uatomic_add_return((addr), (v)))
 452 #define compat_uatomic_inc(addr)                                               \
 453                 (compat_uatomic_add((addr), 1))
 454 #define compat_uatomic_dec(addr)                                               \
 455                 (compat_uatomic_add((addr), -1))
 456
 457 #else
 458 #define UATOMIC_COMPAT(insn)    (_uatomic_##insn)
 459 #endif
 460
 461 /* Read is atomic even in compat mode */
 462 #define uatomic_set(addr, v)                    \
 463                 UATOMIC_COMPAT(set(addr, v))
 464
 465 #define uatomic_cmpxchg(addr, old, _new)        \
 466                 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
 467 #define uatomic_xchg(addr, v)                   \
 468                 UATOMIC_COMPAT(xchg(addr, v))
 469 #define uatomic_add_return(addr, v)             \
 470                 UATOMIC_COMPAT(add_return(addr, v))
 471
 472 #define uatomic_add(addr, v)    UATOMIC_COMPAT(add(addr, v))
 473 #define uatomic_inc(addr)       UATOMIC_COMPAT(inc(addr))
 474 #define uatomic_dec(addr)       UATOMIC_COMPAT(dec(addr))
 475
 476 #ifdef __cplusplus
 477 }
 478 #endif
 479
 480 #include <urcu/uatomic_generic.h>
 481
 482 #endif /* _URCU_ARCH_UATOMIC_X86_H */