urcu/uatomic_arch_x86.h

   1 #ifndef _URCU_ARCH_UATOMIC_X86_H
   2 #define _URCU_ARCH_UATOMIC_X86_H
   3
   4 /*
   5  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
   6  * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
   7  * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
   8  * Copyright (c) 2009      Mathieu Desnoyers
   9  *
  10  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  11  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  12  *
  13  * Permission is hereby granted to use or copy this program
  14  * for any purpose,  provided the above notices are retained on all copies.
  15  * Permission to modify the code and to distribute modified code is granted,
  16  * provided the above notices are retained, and a notice that the code was
  17  * modified is included with the above copyright notice.
  18  *
  19  * Code inspired from libuatomic_ops-1.2, inherited in part from the
  20  * Boehm-Demers-Weiser conservative garbage collector.
  21  */
  22
  23 #include <urcu/compiler.h>
  24 #include <urcu/system.h>
  25
  26 #ifdef __cplusplus
  27 extern "C" {
  28 #endif
  29
  30 #ifndef __SIZEOF_LONG__
  31 #if defined(__x86_64__) || defined(__amd64__)
  32 #define __SIZEOF_LONG__ 8
  33 #else
  34 #define __SIZEOF_LONG__ 4
  35 #endif
  36 #endif
  37
  38 #ifndef BITS_PER_LONG
  39 #define BITS_PER_LONG   (__SIZEOF_LONG__ * 8)
  40 #endif
  41
  42 /*
  43  * Derived from AO_compare_and_swap() and AO_test_and_set_full().
  44  */
  45
  46 struct __uatomic_dummy {
  47         unsigned long v[10];
  48 };
  49 #define __hp(x) ((struct __uatomic_dummy *)(x))
  50
  51 #define _uatomic_set(addr, v)   STORE_SHARED(*(addr), (v))
  52
  53 /* cmpxchg */
  54
  55 static inline __attribute__((always_inline))
  56 unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
  57                               unsigned long _new, int len)
  58 {
  59         switch (len) {
  60         case 1:
  61         {
  62                 unsigned char result = old;
  63
  64                 __asm__ __volatile__(
  65                 "lock; cmpxchgb %2, %1"
  66                         : "+a"(result), "+m"(*__hp(addr))
  67                         : "q"((unsigned char)_new)
  68                         : "memory");
  69                 return result;
  70         }
  71         case 2:
  72         {
  73                 unsigned short result = old;
  74
  75                 __asm__ __volatile__(
  76                 "lock; cmpxchgw %2, %1"
  77                         : "+a"(result), "+m"(*__hp(addr))
  78                         : "r"((unsigned short)_new)
  79                         : "memory");
  80                 return result;
  81         }
  82         case 4:
  83         {
  84                 unsigned int result = old;
  85
  86                 __asm__ __volatile__(
  87                 "lock; cmpxchgl %2, %1"
  88                         : "+a"(result), "+m"(*__hp(addr))
  89                         : "r"((unsigned int)_new)
  90                         : "memory");
  91                 return result;
  92         }
  93 #if (BITS_PER_LONG == 64)
  94         case 8:
  95         {
  96                 unsigned long result = old;
  97
  98                 __asm__ __volatile__(
  99                 "lock; cmpxchgq %2, %1"
 100                         : "+a"(result), "+m"(*__hp(addr))
 101                         : "r"((unsigned long)_new)
 102                         : "memory");
 103                 return result;
 104         }
 105 #endif
 106         }
 107         /* generate an illegal instruction. Cannot catch this with linker tricks
 108          * when optimizations are disabled. */
 109         __asm__ __volatile__("ud2");
 110         return 0;
 111 }
 112
 113 #define _uatomic_cmpxchg(addr, old, _new)                                     \
 114         ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), (unsigned long)(old),\
 115                                                 (unsigned long)(_new),        \
 116                                                 sizeof(*(addr))))
 117
 118 /* xchg */
 119
 120 static inline __attribute__((always_inline))
 121 unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
 122 {
 123         /* Note: the "xchg" instruction does not need a "lock" prefix. */
 124         switch (len) {
 125         case 1:
 126         {
 127                 unsigned char result;
 128                 __asm__ __volatile__(
 129                 "xchgb %0, %1"
 130                         : "=q"(result), "+m"(*__hp(addr))
 131                         : "0" ((unsigned char)val)
 132                         : "memory");
 133                 return result;
 134         }
 135         case 2:
 136         {
 137                 unsigned short result;
 138                 __asm__ __volatile__(
 139                 "xchgw %0, %1"
 140                         : "=r"(result), "+m"(*__hp(addr))
 141                         : "0" ((unsigned short)val)
 142                         : "memory");
 143                 return result;
 144         }
 145         case 4:
 146         {
 147                 unsigned int result;
 148                 __asm__ __volatile__(
 149                 "xchgl %0, %1"
 150                         : "=r"(result), "+m"(*__hp(addr))
 151                         : "0" ((unsigned int)val)
 152                         : "memory");
 153                 return result;
 154         }
 155 #if (BITS_PER_LONG == 64)
 156         case 8:
 157         {
 158                 unsigned long result;
 159                 __asm__ __volatile__(
 160                 "xchgq %0, %1"
 161                         : "=r"(result), "+m"(*__hp(addr))
 162                         : "0" ((unsigned long)val)
 163                         : "memory");
 164                 return result;
 165         }
 166 #endif
 167         }
 168         /* generate an illegal instruction. Cannot catch this with linker tricks
 169          * when optimizations are disabled. */
 170         __asm__ __volatile__("ud2");
 171         return 0;
 172 }
 173
 174 #define _uatomic_xchg(addr, v)                                                \
 175         ((__typeof__(*(addr))) __uatomic_exchange((addr), (unsigned long)(v), \
 176                                                 sizeof(*(addr))))
 177
 178 /* uatomic_add_return */
 179
 180 static inline __attribute__((always_inline))
 181 unsigned long __uatomic_add_return(void *addr, unsigned long val,
 182                                  int len)
 183 {
 184         switch (len) {
 185         case 1:
 186         {
 187                 unsigned char result = val;
 188
 189                 __asm__ __volatile__(
 190                 "lock; xaddb %1, %0"
 191                         : "+m"(*__hp(addr)), "+q" (result)
 192                         :
 193                         : "memory");
 194                 return result + (unsigned char)val;
 195         }
 196         case 2:
 197         {
 198                 unsigned short result = val;
 199
 200                 __asm__ __volatile__(
 201                 "lock; xaddw %1, %0"
 202                         : "+m"(*__hp(addr)), "+r" (result)
 203                         :
 204                         : "memory");
 205                 return result + (unsigned short)val;
 206         }
 207         case 4:
 208         {
 209                 unsigned int result = val;
 210
 211                 __asm__ __volatile__(
 212                 "lock; xaddl %1, %0"
 213                         : "+m"(*__hp(addr)), "+r" (result)
 214                         :
 215                         : "memory");
 216                 return result + (unsigned int)val;
 217         }
 218 #if (BITS_PER_LONG == 64)
 219         case 8:
 220         {
 221                 unsigned long result = val;
 222
 223                 __asm__ __volatile__(
 224                 "lock; xaddq %1, %0"
 225                         : "+m"(*__hp(addr)), "+r" (result)
 226                         :
 227                         : "memory");
 228                 return result + (unsigned long)val;
 229         }
 230 #endif
 231         }
 232         /* generate an illegal instruction. Cannot catch this with linker tricks
 233          * when optimizations are disabled. */
 234         __asm__ __volatile__("ud2");
 235         return 0;
 236 }
 237
 238 #define _uatomic_add_return(addr, v)                                    \
 239         ((__typeof__(*(addr))) __uatomic_add_return((addr),             \
 240                                                   (unsigned long)(v),   \
 241                                                   sizeof(*(addr))))
 242
 243 /* uatomic_add */
 244
 245 static inline __attribute__((always_inline))
 246 void __uatomic_add(void *addr, unsigned long val, int len)
 247 {
 248         switch (len) {
 249         case 1:
 250         {
 251                 __asm__ __volatile__(
 252                 "lock; addb %1, %0"
 253                         : "=m"(*__hp(addr))
 254                         : "iq" ((unsigned char)val)
 255                         : "memory");
 256                 return;
 257         }
 258         case 2:
 259         {
 260                 __asm__ __volatile__(
 261                 "lock; addw %1, %0"
 262                         : "=m"(*__hp(addr))
 263                         : "ir" ((unsigned short)val)
 264                         : "memory");
 265                 return;
 266         }
 267         case 4:
 268         {
 269                 __asm__ __volatile__(
 270                 "lock; addl %1, %0"
 271                         : "=m"(*__hp(addr))
 272                         : "ir" ((unsigned int)val)
 273                         : "memory");
 274                 return;
 275         }
 276 #if (BITS_PER_LONG == 64)
 277         case 8:
 278         {
 279                 __asm__ __volatile__(
 280                 "lock; addq %1, %0"
 281                         : "=m"(*__hp(addr))
 282                         : "er" ((unsigned long)val)
 283                         : "memory");
 284                 return;
 285         }
 286 #endif
 287         }
 288         /* generate an illegal instruction. Cannot catch this with linker tricks
 289          * when optimizations are disabled. */
 290         __asm__ __volatile__("ud2");
 291         return;
 292 }
 293
 294 #define _uatomic_add(addr, v)                                              \
 295         (__uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
 296
 297
 298 /* uatomic_inc */
 299
 300 static inline __attribute__((always_inline))
 301 void __uatomic_inc(void *addr, int len)
 302 {
 303         switch (len) {
 304         case 1:
 305         {
 306                 __asm__ __volatile__(
 307                 "lock; incb %0"
 308                         : "=m"(*__hp(addr))
 309                         :
 310                         : "memory");
 311                 return;
 312         }
 313         case 2:
 314         {
 315                 __asm__ __volatile__(
 316                 "lock; incw %0"
 317                         : "=m"(*__hp(addr))
 318                         :
 319                         : "memory");
 320                 return;
 321         }
 322         case 4:
 323         {
 324                 __asm__ __volatile__(
 325                 "lock; incl %0"
 326                         : "=m"(*__hp(addr))
 327                         :
 328                         : "memory");
 329                 return;
 330         }
 331 #if (BITS_PER_LONG == 64)
 332         case 8:
 333         {
 334                 __asm__ __volatile__(
 335                 "lock; incq %0"
 336                         : "=m"(*__hp(addr))
 337                         :
 338                         : "memory");
 339                 return;
 340         }
 341 #endif
 342         }
 343         /* generate an illegal instruction. Cannot catch this with linker tricks
 344          * when optimizations are disabled. */
 345         __asm__ __volatile__("ud2");
 346         return;
 347 }
 348
 349 #define _uatomic_inc(addr)      (__uatomic_inc((addr), sizeof(*(addr))))
 350
 351 /* uatomic_dec */
 352
 353 static inline __attribute__((always_inline))
 354 void __uatomic_dec(void *addr, int len)
 355 {
 356         switch (len) {
 357         case 1:
 358         {
 359                 __asm__ __volatile__(
 360                 "lock; decb %0"
 361                         : "=m"(*__hp(addr))
 362                         :
 363                         : "memory");
 364                 return;
 365         }
 366         case 2:
 367         {
 368                 __asm__ __volatile__(
 369                 "lock; decw %0"
 370                         : "=m"(*__hp(addr))
 371                         :
 372                         : "memory");
 373                 return;
 374         }
 375         case 4:
 376         {
 377                 __asm__ __volatile__(
 378                 "lock; decl %0"
 379                         : "=m"(*__hp(addr))
 380                         :
 381                         : "memory");
 382                 return;
 383         }
 384 #if (BITS_PER_LONG == 64)
 385         case 8:
 386         {
 387                 __asm__ __volatile__(
 388                 "lock; decq %0"
 389                         : "=m"(*__hp(addr))
 390                         :
 391                         : "memory");
 392                 return;
 393         }
 394 #endif
 395         }
 396         /* generate an illegal instruction. Cannot catch this with linker tricks
 397          * when optimizations are disabled. */
 398         __asm__ __volatile__("ud2");
 399         return;
 400 }
 401
 402 #define _uatomic_dec(addr)      (__uatomic_dec((addr), sizeof(*(addr))))
 403
 404 #if ((BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
 405 extern int __rcu_cas_avail;
 406 extern int __rcu_cas_init(void);
 407
 408 #define UATOMIC_COMPAT(insn)                                                    \
 409         ((likely(__rcu_cas_avail > 0))                                          \
 410         ? (_uatomic_##insn)                                                     \
 411                 : ((unlikely(__rcu_cas_avail < 0)                               \
 412                         ? ((__rcu_cas_init() > 0)                               \
 413                                 ? (_uatomic_##insn)                             \
 414                                 : (compat_uatomic_##insn))                      \
 415                         : (compat_uatomic_##insn))))
 416
 417 extern unsigned long _compat_uatomic_set(void *addr,
 418                                          unsigned long _new, int len);
 419 #define compat_uatomic_set(addr, _new)                                         \
 420         ((__typeof__(*(addr))) _compat_uatomic_set((addr),                     \
 421                                                 (unsigned long)(_new),         \
 422                                                 sizeof(*(addr))))
 423
 424
 425 extern unsigned long _compat_uatomic_xchg(void *addr,
 426                                           unsigned long _new, int len);
 427 #define compat_uatomic_xchg(addr, _new)                                        \
 428         ((__typeof__(*(addr))) _compat_uatomic_xchg((addr),                    \
 429                                                 (unsigned long)(_new),         \
 430                                                 sizeof(*(addr))))
 431
 432 extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
 433                                              unsigned long _new, int len);
 434 #define compat_uatomic_cmpxchg(addr, old, _new)                                \
 435         ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),                 \
 436                                                 (unsigned long)(old),          \
 437                                                 (unsigned long)(_new),         \
 438                                                 sizeof(*(addr))))
 439
 440 extern unsigned long _compat_uatomic_xchg(void *addr,
 441                                           unsigned long _new, int len);
 442 #define compat_uatomic_add_return(addr, v)                                     \
 443         ((__typeof__(*(addr))) _compat_uatomic_add_return((addr),              \
 444                                                 (unsigned long)(v),            \
 445                                                 sizeof(*(addr))))
 446
 447 #define compat_uatomic_add(addr, v)                                            \
 448                 ((void)compat_uatomic_add_return((addr), (v)))
 449 #define compat_uatomic_inc(addr)                                               \
 450                 (compat_uatomic_add((addr), 1))
 451 #define compat_uatomic_dec(addr)                                               \
 452                 (compat_uatomic_add((addr), -1))
 453
 454 #else
 455 #define UATOMIC_COMPAT(insn)    (_uatomic_##insn)
 456 #endif
 457
 458 /* Read is atomic even in compat mode */
 459 #define uatomic_set(addr, v)                    \
 460                 UATOMIC_COMPAT(set(addr, v))
 461
 462 #define uatomic_cmpxchg(addr, old, _new)        \
 463                 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
 464 #define uatomic_xchg(addr, v)                   \
 465                 UATOMIC_COMPAT(xchg(addr, v))
 466 #define uatomic_add_return(addr, v)             \
 467                 UATOMIC_COMPAT(add_return(addr, v))
 468
 469 #define uatomic_add(addr, v)    UATOMIC_COMPAT(add(addr, v))
 470 #define uatomic_inc(addr)       UATOMIC_COMPAT(inc(addr))
 471 #define uatomic_dec(addr)       UATOMIC_COMPAT(dec(addr))
 472
 473 #ifdef __cplusplus
 474 }
 475 #endif
 476
 477 #include <urcu/uatomic_generic.h>
 478
 479 #endif /* _URCU_ARCH_UATOMIC_X86_H */