urcu/uatomic/x86.h

   1 #ifndef _URCU_ARCH_UATOMIC_X86_H
   2 #define _URCU_ARCH_UATOMIC_X86_H
   3
   4 /*
   5  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
   6  * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
   7  * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
   8  * Copyright (c) 2009      Mathieu Desnoyers
   9  *
  10  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  11  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  12  *
  13  * Permission is hereby granted to use or copy this program
  14  * for any purpose,  provided the above notices are retained on all copies.
  15  * Permission to modify the code and to distribute modified code is granted,
  16  * provided the above notices are retained, and a notice that the code was
  17  * modified is included with the above copyright notice.
  18  *
  19  * Code inspired from libuatomic_ops-1.2, inherited in part from the
  20  * Boehm-Demers-Weiser conservative garbage collector.
  21  */
  22
  23 #include <urcu/compiler.h>
  24 #include <urcu/system.h>
  25
  26 #define UATOMIC_HAS_ATOMIC_BYTE
  27 #define UATOMIC_HAS_ATOMIC_SHORT
  28
  29 #ifdef __cplusplus
  30 extern "C" {
  31 #endif
  32
  33 /*
  34  * Derived from AO_compare_and_swap() and AO_test_and_set_full().
  35  */
  36
  37 struct __uatomic_dummy {
  38         unsigned long v[10];
  39 };
  40 #define __hp(x) ((struct __uatomic_dummy *)(x))
  41
  42 #define _uatomic_set(addr, v)   CMM_STORE_SHARED(*(addr), (v))
  43
  44 /* cmpxchg */
  45
  46 static inline __attribute__((always_inline))
  47 unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
  48                               unsigned long _new, int len)
  49 {
  50         switch (len) {
  51         case 1:
  52         {
  53                 unsigned char result = old;
  54
  55                 __asm__ __volatile__(
  56                 "lock; cmpxchgb %2, %1"
  57                         : "+a"(result), "+m"(*__hp(addr))
  58                         : "q"((unsigned char)_new)
  59                         : "memory");
  60                 return result;
  61         }
  62         case 2:
  63         {
  64                 unsigned short result = old;
  65
  66                 __asm__ __volatile__(
  67                 "lock; cmpxchgw %2, %1"
  68                         : "+a"(result), "+m"(*__hp(addr))
  69                         : "r"((unsigned short)_new)
  70                         : "memory");
  71                 return result;
  72         }
  73         case 4:
  74         {
  75                 unsigned int result = old;
  76
  77                 __asm__ __volatile__(
  78                 "lock; cmpxchgl %2, %1"
  79                         : "+a"(result), "+m"(*__hp(addr))
  80                         : "r"((unsigned int)_new)
  81                         : "memory");
  82                 return result;
  83         }
  84 #if (CAA_BITS_PER_LONG == 64)
  85         case 8:
  86         {
  87                 unsigned long result = old;
  88
  89                 __asm__ __volatile__(
  90                 "lock; cmpxchgq %2, %1"
  91                         : "+a"(result), "+m"(*__hp(addr))
  92                         : "r"((unsigned long)_new)
  93                         : "memory");
  94                 return result;
  95         }
  96 #endif
  97         }
  98         /* generate an illegal instruction. Cannot catch this with linker tricks
  99          * when optimizations are disabled. */
 100         __asm__ __volatile__("ud2");
 101         return 0;
 102 }
 103
 104 #define _uatomic_cmpxchg(addr, old, _new)                                     \
 105         ((__typeof__(*(addr))) __uatomic_cmpxchg((addr),                      \
 106                                                 caa_cast_long_keep_sign(old), \
 107                                                 caa_cast_long_keep_sign(_new),\
 108                                                 sizeof(*(addr))))
 109
 110 /* xchg */
 111
 112 static inline __attribute__((always_inline))
 113 unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
 114 {
 115         /* Note: the "xchg" instruction does not need a "lock" prefix. */
 116         switch (len) {
 117         case 1:
 118         {
 119                 unsigned char result;
 120                 __asm__ __volatile__(
 121                 "xchgb %0, %1"
 122                         : "=q"(result), "+m"(*__hp(addr))
 123                         : "0" ((unsigned char)val)
 124                         : "memory");
 125                 return result;
 126         }
 127         case 2:
 128         {
 129                 unsigned short result;
 130                 __asm__ __volatile__(
 131                 "xchgw %0, %1"
 132                         : "=r"(result), "+m"(*__hp(addr))
 133                         : "0" ((unsigned short)val)
 134                         : "memory");
 135                 return result;
 136         }
 137         case 4:
 138         {
 139                 unsigned int result;
 140                 __asm__ __volatile__(
 141                 "xchgl %0, %1"
 142                         : "=r"(result), "+m"(*__hp(addr))
 143                         : "0" ((unsigned int)val)
 144                         : "memory");
 145                 return result;
 146         }
 147 #if (CAA_BITS_PER_LONG == 64)
 148         case 8:
 149         {
 150                 unsigned long result;
 151                 __asm__ __volatile__(
 152                 "xchgq %0, %1"
 153                         : "=r"(result), "+m"(*__hp(addr))
 154                         : "0" ((unsigned long)val)
 155                         : "memory");
 156                 return result;
 157         }
 158 #endif
 159         }
 160         /* generate an illegal instruction. Cannot catch this with linker tricks
 161          * when optimizations are disabled. */
 162         __asm__ __volatile__("ud2");
 163         return 0;
 164 }
 165
 166 #define _uatomic_xchg(addr, v)                                                \
 167         ((__typeof__(*(addr))) __uatomic_exchange((addr),                     \
 168                                                 caa_cast_long_keep_sign(v),   \
 169                                                 sizeof(*(addr))))
 170
 171 /* uatomic_add_return */
 172
 173 static inline __attribute__((always_inline))
 174 unsigned long __uatomic_add_return(void *addr, unsigned long val,
 175                                  int len)
 176 {
 177         switch (len) {
 178         case 1:
 179         {
 180                 unsigned char result = val;
 181
 182                 __asm__ __volatile__(
 183                 "lock; xaddb %1, %0"
 184                         : "+m"(*__hp(addr)), "+q" (result)
 185                         :
 186                         : "memory");
 187                 return result + (unsigned char)val;
 188         }
 189         case 2:
 190         {
 191                 unsigned short result = val;
 192
 193                 __asm__ __volatile__(
 194                 "lock; xaddw %1, %0"
 195                         : "+m"(*__hp(addr)), "+r" (result)
 196                         :
 197                         : "memory");
 198                 return result + (unsigned short)val;
 199         }
 200         case 4:
 201         {
 202                 unsigned int result = val;
 203
 204                 __asm__ __volatile__(
 205                 "lock; xaddl %1, %0"
 206                         : "+m"(*__hp(addr)), "+r" (result)
 207                         :
 208                         : "memory");
 209                 return result + (unsigned int)val;
 210         }
 211 #if (CAA_BITS_PER_LONG == 64)
 212         case 8:
 213         {
 214                 unsigned long result = val;
 215
 216                 __asm__ __volatile__(
 217                 "lock; xaddq %1, %0"
 218                         : "+m"(*__hp(addr)), "+r" (result)
 219                         :
 220                         : "memory");
 221                 return result + (unsigned long)val;
 222         }
 223 #endif
 224         }
 225         /* generate an illegal instruction. Cannot catch this with linker tricks
 226          * when optimizations are disabled. */
 227         __asm__ __volatile__("ud2");
 228         return 0;
 229 }
 230
 231 #define _uatomic_add_return(addr, v)                                        \
 232         ((__typeof__(*(addr))) __uatomic_add_return((addr),                 \
 233                                                 caa_cast_long_keep_sign(v), \
 234                                                 sizeof(*(addr))))
 235
 236 /* uatomic_and */
 237
 238 static inline __attribute__((always_inline))
 239 void __uatomic_and(void *addr, unsigned long val, int len)
 240 {
 241         switch (len) {
 242         case 1:
 243         {
 244                 __asm__ __volatile__(
 245                 "lock; andb %1, %0"
 246                         : "=m"(*__hp(addr))
 247                         : "iq" ((unsigned char)val)
 248                         : "memory");
 249                 return;
 250         }
 251         case 2:
 252         {
 253                 __asm__ __volatile__(
 254                 "lock; andw %1, %0"
 255                         : "=m"(*__hp(addr))
 256                         : "ir" ((unsigned short)val)
 257                         : "memory");
 258                 return;
 259         }
 260         case 4:
 261         {
 262                 __asm__ __volatile__(
 263                 "lock; andl %1, %0"
 264                         : "=m"(*__hp(addr))
 265                         : "ir" ((unsigned int)val)
 266                         : "memory");
 267                 return;
 268         }
 269 #if (CAA_BITS_PER_LONG == 64)
 270         case 8:
 271         {
 272                 __asm__ __volatile__(
 273                 "lock; andq %1, %0"
 274                         : "=m"(*__hp(addr))
 275                         : "er" ((unsigned long)val)
 276                         : "memory");
 277                 return;
 278         }
 279 #endif
 280         }
 281         /* generate an illegal instruction. Cannot catch this with linker tricks
 282          * when optimizations are disabled. */
 283         __asm__ __volatile__("ud2");
 284         return;
 285 }
 286
 287 #define _uatomic_and(addr, v)                                              \
 288         (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 289
 290 /* uatomic_or */
 291
 292 static inline __attribute__((always_inline))
 293 void __uatomic_or(void *addr, unsigned long val, int len)
 294 {
 295         switch (len) {
 296         case 1:
 297         {
 298                 __asm__ __volatile__(
 299                 "lock; orb %1, %0"
 300                         : "=m"(*__hp(addr))
 301                         : "iq" ((unsigned char)val)
 302                         : "memory");
 303                 return;
 304         }
 305         case 2:
 306         {
 307                 __asm__ __volatile__(
 308                 "lock; orw %1, %0"
 309                         : "=m"(*__hp(addr))
 310                         : "ir" ((unsigned short)val)
 311                         : "memory");
 312                 return;
 313         }
 314         case 4:
 315         {
 316                 __asm__ __volatile__(
 317                 "lock; orl %1, %0"
 318                         : "=m"(*__hp(addr))
 319                         : "ir" ((unsigned int)val)
 320                         : "memory");
 321                 return;
 322         }
 323 #if (CAA_BITS_PER_LONG == 64)
 324         case 8:
 325         {
 326                 __asm__ __volatile__(
 327                 "lock; orq %1, %0"
 328                         : "=m"(*__hp(addr))
 329                         : "er" ((unsigned long)val)
 330                         : "memory");
 331                 return;
 332         }
 333 #endif
 334         }
 335         /* generate an illegal instruction. Cannot catch this with linker tricks
 336          * when optimizations are disabled. */
 337         __asm__ __volatile__("ud2");
 338         return;
 339 }
 340
 341 #define _uatomic_or(addr, v)                                               \
 342         (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 343
 344 /* uatomic_add */
 345
 346 static inline __attribute__((always_inline))
 347 void __uatomic_add(void *addr, unsigned long val, int len)
 348 {
 349         switch (len) {
 350         case 1:
 351         {
 352                 __asm__ __volatile__(
 353                 "lock; addb %1, %0"
 354                         : "=m"(*__hp(addr))
 355                         : "iq" ((unsigned char)val)
 356                         : "memory");
 357                 return;
 358         }
 359         case 2:
 360         {
 361                 __asm__ __volatile__(
 362                 "lock; addw %1, %0"
 363                         : "=m"(*__hp(addr))
 364                         : "ir" ((unsigned short)val)
 365                         : "memory");
 366                 return;
 367         }
 368         case 4:
 369         {
 370                 __asm__ __volatile__(
 371                 "lock; addl %1, %0"
 372                         : "=m"(*__hp(addr))
 373                         : "ir" ((unsigned int)val)
 374                         : "memory");
 375                 return;
 376         }
 377 #if (CAA_BITS_PER_LONG == 64)
 378         case 8:
 379         {
 380                 __asm__ __volatile__(
 381                 "lock; addq %1, %0"
 382                         : "=m"(*__hp(addr))
 383                         : "er" ((unsigned long)val)
 384                         : "memory");
 385                 return;
 386         }
 387 #endif
 388         }
 389         /* generate an illegal instruction. Cannot catch this with linker tricks
 390          * when optimizations are disabled. */
 391         __asm__ __volatile__("ud2");
 392         return;
 393 }
 394
 395 #define _uatomic_add(addr, v)                                              \
 396         (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 397
 398
 399 /* uatomic_inc */
 400
 401 static inline __attribute__((always_inline))
 402 void __uatomic_inc(void *addr, int len)
 403 {
 404         switch (len) {
 405         case 1:
 406         {
 407                 __asm__ __volatile__(
 408                 "lock; incb %0"
 409                         : "=m"(*__hp(addr))
 410                         :
 411                         : "memory");
 412                 return;
 413         }
 414         case 2:
 415         {
 416                 __asm__ __volatile__(
 417                 "lock; incw %0"
 418                         : "=m"(*__hp(addr))
 419                         :
 420                         : "memory");
 421                 return;
 422         }
 423         case 4:
 424         {
 425                 __asm__ __volatile__(
 426                 "lock; incl %0"
 427                         : "=m"(*__hp(addr))
 428                         :
 429                         : "memory");
 430                 return;
 431         }
 432 #if (CAA_BITS_PER_LONG == 64)
 433         case 8:
 434         {
 435                 __asm__ __volatile__(
 436                 "lock; incq %0"
 437                         : "=m"(*__hp(addr))
 438                         :
 439                         : "memory");
 440                 return;
 441         }
 442 #endif
 443         }
 444         /* generate an illegal instruction. Cannot catch this with linker tricks
 445          * when optimizations are disabled. */
 446         __asm__ __volatile__("ud2");
 447         return;
 448 }
 449
 450 #define _uatomic_inc(addr)      (__uatomic_inc((addr), sizeof(*(addr))))
 451
 452 /* uatomic_dec */
 453
 454 static inline __attribute__((always_inline))
 455 void __uatomic_dec(void *addr, int len)
 456 {
 457         switch (len) {
 458         case 1:
 459         {
 460                 __asm__ __volatile__(
 461                 "lock; decb %0"
 462                         : "=m"(*__hp(addr))
 463                         :
 464                         : "memory");
 465                 return;
 466         }
 467         case 2:
 468         {
 469                 __asm__ __volatile__(
 470                 "lock; decw %0"
 471                         : "=m"(*__hp(addr))
 472                         :
 473                         : "memory");
 474                 return;
 475         }
 476         case 4:
 477         {
 478                 __asm__ __volatile__(
 479                 "lock; decl %0"
 480                         : "=m"(*__hp(addr))
 481                         :
 482                         : "memory");
 483                 return;
 484         }
 485 #if (CAA_BITS_PER_LONG == 64)
 486         case 8:
 487         {
 488                 __asm__ __volatile__(
 489                 "lock; decq %0"
 490                         : "=m"(*__hp(addr))
 491                         :
 492                         : "memory");
 493                 return;
 494         }
 495 #endif
 496         }
 497         /* generate an illegal instruction. Cannot catch this with linker tricks
 498          * when optimizations are disabled. */
 499         __asm__ __volatile__("ud2");
 500         return;
 501 }
 502
 503 #define _uatomic_dec(addr)      (__uatomic_dec((addr), sizeof(*(addr))))
 504
 505 #if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
 506 extern int __rcu_cas_avail;
 507 extern int __rcu_cas_init(void);
 508
 509 #define UATOMIC_COMPAT(insn)                                                    \
 510         ((caa_likely(__rcu_cas_avail > 0))                                              \
 511         ? (_uatomic_##insn)                                                     \
 512                 : ((caa_unlikely(__rcu_cas_avail < 0)                           \
 513                         ? ((__rcu_cas_init() > 0)                               \
 514                                 ? (_uatomic_##insn)                             \
 515                                 : (compat_uatomic_##insn))                      \
 516                         : (compat_uatomic_##insn))))
 517
 518 extern unsigned long _compat_uatomic_set(void *addr,
 519                                          unsigned long _new, int len);
 520 #define compat_uatomic_set(addr, _new)                                         \
 521         ((__typeof__(*(addr))) _compat_uatomic_set((addr),                     \
 522                                                 caa_cast_long_keep_sign(_new), \
 523                                                 sizeof(*(addr))))
 524
 525
 526 extern unsigned long _compat_uatomic_xchg(void *addr,
 527                                           unsigned long _new, int len);
 528 #define compat_uatomic_xchg(addr, _new)                                        \
 529         ((__typeof__(*(addr))) _compat_uatomic_xchg((addr),                    \
 530                                                 caa_cast_long_keep_sign(_new), \
 531                                                 sizeof(*(addr))))
 532
 533 extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
 534                                              unsigned long _new, int len);
 535 #define compat_uatomic_cmpxchg(addr, old, _new)                                \
 536         ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),                 \
 537                                                 caa_cast_long_keep_sign(old),  \
 538                                                 caa_cast_long_keep_sign(_new), \
 539                                                 sizeof(*(addr))))
 540
 541 extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
 542 #define compat_uatomic_and(addr, v)                                    \
 543         (_compat_uatomic_and((addr),                                   \
 544                         caa_cast_long_keep_sign(v),                    \
 545                         sizeof(*(addr))))
 546
 547 extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
 548 #define compat_uatomic_or(addr, v)                                     \
 549         (_compat_uatomic_or((addr),                                    \
 550                           caa_cast_long_keep_sign(v),                  \
 551                           sizeof(*(addr))))
 552
 553 extern unsigned long _compat_uatomic_add_return(void *addr,
 554                                                 unsigned long _new, int len);
 555 #define compat_uatomic_add_return(addr, v)                                  \
 556         ((__typeof__(*(addr))) _compat_uatomic_add_return((addr),           \
 557                                                 caa_cast_long_keep_sign(v), \
 558                                                 sizeof(*(addr))))
 559
 560 #define compat_uatomic_add(addr, v)                                            \
 561                 ((void)compat_uatomic_add_return((addr), (v)))
 562 #define compat_uatomic_inc(addr)                                               \
 563                 (compat_uatomic_add((addr), 1))
 564 #define compat_uatomic_dec(addr)                                               \
 565                 (compat_uatomic_add((addr), -1))
 566
 567 #else
 568 #define UATOMIC_COMPAT(insn)    (_uatomic_##insn)
 569 #endif
 570
 571 /* Read is atomic even in compat mode */
 572 #define uatomic_set(addr, v)                    \
 573                 UATOMIC_COMPAT(set(addr, v))
 574
 575 #define uatomic_cmpxchg(addr, old, _new)        \
 576                 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
 577 #define uatomic_xchg(addr, v)                   \
 578                 UATOMIC_COMPAT(xchg(addr, v))
 579
 580 #define uatomic_and(addr, v)            \
 581                 UATOMIC_COMPAT(and(addr, v))
 582 #define cmm_smp_mb__before_uatomic_and()        cmm_barrier()
 583 #define cmm_smp_mb__after_uatomic_and()         cmm_barrier()
 584
 585 #define uatomic_or(addr, v)             \
 586                 UATOMIC_COMPAT(or(addr, v))
 587 #define cmm_smp_mb__before_uatomic_or()         cmm_barrier()
 588 #define cmm_smp_mb__after_uatomic_or()          cmm_barrier()
 589
 590 #define uatomic_add_return(addr, v)             \
 591                 UATOMIC_COMPAT(add_return(addr, v))
 592
 593 #define uatomic_add(addr, v)    UATOMIC_COMPAT(add(addr, v))
 594 #define cmm_smp_mb__before_uatomic_add()        cmm_barrier()
 595 #define cmm_smp_mb__after_uatomic_add()         cmm_barrier()
 596
 597 #define uatomic_inc(addr)       UATOMIC_COMPAT(inc(addr))
 598 #define cmm_smp_mb__before_uatomic_inc()        cmm_barrier()
 599 #define cmm_smp_mb__after_uatomic_inc()         cmm_barrier()
 600
 601 #define uatomic_dec(addr)       UATOMIC_COMPAT(dec(addr))
 602 #define cmm_smp_mb__before_uatomic_dec()        cmm_barrier()
 603 #define cmm_smp_mb__after_uatomic_dec()         cmm_barrier()
 604
 605 #ifdef __cplusplus
 606 }
 607 #endif
 608
 609 #include <urcu/uatomic/generic.h>
 610
 611 #endif /* _URCU_ARCH_UATOMIC_X86_H */