include/urcu/uatomic/x86.h

   1 #ifndef _URCU_ARCH_UATOMIC_X86_H
   2 #define _URCU_ARCH_UATOMIC_X86_H
   3
   4 /*
   5  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
   6  * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
   7  * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
   8  * Copyright (c) 2009      Mathieu Desnoyers
   9  *
  10  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  11  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  12  *
  13  * Permission is hereby granted to use or copy this program
  14  * for any purpose,  provided the above notices are retained on all copies.
  15  * Permission to modify the code and to distribute modified code is granted,
  16  * provided the above notices are retained, and a notice that the code was
  17  * modified is included with the above copyright notice.
  18  *
  19  * Code inspired from libuatomic_ops-1.2, inherited in part from the
  20  * Boehm-Demers-Weiser conservative garbage collector.
  21  */
  22
  23 #include <urcu/config.h>
  24 #include <urcu/compiler.h>
  25 #include <urcu/system.h>
  26
  27 #define UATOMIC_HAS_ATOMIC_BYTE
  28 #define UATOMIC_HAS_ATOMIC_SHORT
  29
  30 #ifdef __cplusplus
  31 extern "C" {
  32 #endif
  33
  34 /*
  35  * Derived from AO_compare_and_swap() and AO_test_and_set_full().
  36  */
  37
  38 /*
  39  * The __hp() macro casts the void pointer @x to a pointer to a structure
  40  * containing an array of char of the specified size. This allows passing the
  41  * @addr arguments of the following inline functions as "m" and "+m" operands
  42  * to the assembly. The @size parameter should be a constant to support
  43  * compilers such as clang which do not support VLA. Create typedefs because
  44  * C++ does not allow types be defined in casts.
  45  */
  46
  47 typedef struct { char v[1]; } __hp_1;
  48 typedef struct { char v[2]; } __hp_2;
  49 typedef struct { char v[4]; } __hp_4;
  50 typedef struct { char v[8]; } __hp_8;
  51
  52 #define __hp(size, x)   ((__hp_##size *)(x))
  53
  54 #define _uatomic_set(addr, v)   ((void) CMM_STORE_SHARED(*(addr), (v)))
  55
  56 /* cmpxchg */
  57
  58 static inline __attribute__((always_inline))
  59 unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
  60                               unsigned long _new, int len)
  61 {
  62         switch (len) {
  63         case 1:
  64         {
  65                 unsigned char result = old;
  66
  67                 __asm__ __volatile__(
  68                 "lock; cmpxchgb %2, %1"
  69                         : "+a"(result), "+m"(*__hp(1, addr))
  70                         : "q"((unsigned char)_new)
  71                         : "memory");
  72                 return result;
  73         }
  74         case 2:
  75         {
  76                 unsigned short result = old;
  77
  78                 __asm__ __volatile__(
  79                 "lock; cmpxchgw %2, %1"
  80                         : "+a"(result), "+m"(*__hp(2, addr))
  81                         : "r"((unsigned short)_new)
  82                         : "memory");
  83                 return result;
  84         }
  85         case 4:
  86         {
  87                 unsigned int result = old;
  88
  89                 __asm__ __volatile__(
  90                 "lock; cmpxchgl %2, %1"
  91                         : "+a"(result), "+m"(*__hp(4, addr))
  92                         : "r"((unsigned int)_new)
  93                         : "memory");
  94                 return result;
  95         }
  96 #if (CAA_BITS_PER_LONG == 64)
  97         case 8:
  98         {
  99                 unsigned long result = old;
 100
 101                 __asm__ __volatile__(
 102                 "lock; cmpxchgq %2, %1"
 103                         : "+a"(result), "+m"(*__hp(8, addr))
 104                         : "r"((unsigned long)_new)
 105                         : "memory");
 106                 return result;
 107         }
 108 #endif
 109         }
 110         /*
 111          * generate an illegal instruction. Cannot catch this with
 112          * linker tricks when optimizations are disabled.
 113          */
 114         __asm__ __volatile__("ud2");
 115         return 0;
 116 }
 117
 118 #define _uatomic_cmpxchg(addr, old, _new)                                     \
 119         ((__typeof__(*(addr))) __uatomic_cmpxchg((addr),                      \
 120                                                 caa_cast_long_keep_sign(old), \
 121                                                 caa_cast_long_keep_sign(_new),\
 122                                                 sizeof(*(addr))))
 123
 124 /* xchg */
 125
 126 static inline __attribute__((always_inline))
 127 unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
 128 {
 129         /* Note: the "xchg" instruction does not need a "lock" prefix. */
 130         switch (len) {
 131         case 1:
 132         {
 133                 unsigned char result;
 134                 __asm__ __volatile__(
 135                 "xchgb %0, %1"
 136                         : "=q"(result), "+m"(*__hp(1, addr))
 137                         : "0" ((unsigned char)val)
 138                         : "memory");
 139                 return result;
 140         }
 141         case 2:
 142         {
 143                 unsigned short result;
 144                 __asm__ __volatile__(
 145                 "xchgw %0, %1"
 146                         : "=r"(result), "+m"(*__hp(2, addr))
 147                         : "0" ((unsigned short)val)
 148                         : "memory");
 149                 return result;
 150         }
 151         case 4:
 152         {
 153                 unsigned int result;
 154                 __asm__ __volatile__(
 155                 "xchgl %0, %1"
 156                         : "=r"(result), "+m"(*__hp(4, addr))
 157                         : "0" ((unsigned int)val)
 158                         : "memory");
 159                 return result;
 160         }
 161 #if (CAA_BITS_PER_LONG == 64)
 162         case 8:
 163         {
 164                 unsigned long result;
 165                 __asm__ __volatile__(
 166                 "xchgq %0, %1"
 167                         : "=r"(result), "+m"(*__hp(8, addr))
 168                         : "0" ((unsigned long)val)
 169                         : "memory");
 170                 return result;
 171         }
 172 #endif
 173         }
 174         /*
 175          * generate an illegal instruction. Cannot catch this with
 176          * linker tricks when optimizations are disabled.
 177          */
 178         __asm__ __volatile__("ud2");
 179         return 0;
 180 }
 181
 182 #define _uatomic_xchg(addr, v)                                                \
 183         ((__typeof__(*(addr))) __uatomic_exchange((addr),                     \
 184                                                 caa_cast_long_keep_sign(v),   \
 185                                                 sizeof(*(addr))))
 186
 187 /* uatomic_add_return */
 188
 189 static inline __attribute__((always_inline))
 190 unsigned long __uatomic_add_return(void *addr, unsigned long val,
 191                                  int len)
 192 {
 193         switch (len) {
 194         case 1:
 195         {
 196                 unsigned char result = val;
 197
 198                 __asm__ __volatile__(
 199                 "lock; xaddb %1, %0"
 200                         : "+m"(*__hp(1, addr)), "+q" (result)
 201                         :
 202                         : "memory");
 203                 return result + (unsigned char)val;
 204         }
 205         case 2:
 206         {
 207                 unsigned short result = val;
 208
 209                 __asm__ __volatile__(
 210                 "lock; xaddw %1, %0"
 211                         : "+m"(*__hp(2, addr)), "+r" (result)
 212                         :
 213                         : "memory");
 214                 return result + (unsigned short)val;
 215         }
 216         case 4:
 217         {
 218                 unsigned int result = val;
 219
 220                 __asm__ __volatile__(
 221                 "lock; xaddl %1, %0"
 222                         : "+m"(*__hp(4, addr)), "+r" (result)
 223                         :
 224                         : "memory");
 225                 return result + (unsigned int)val;
 226         }
 227 #if (CAA_BITS_PER_LONG == 64)
 228         case 8:
 229         {
 230                 unsigned long result = val;
 231
 232                 __asm__ __volatile__(
 233                 "lock; xaddq %1, %0"
 234                         : "+m"(*__hp(8, addr)), "+r" (result)
 235                         :
 236                         : "memory");
 237                 return result + (unsigned long)val;
 238         }
 239 #endif
 240         }
 241         /*
 242          * generate an illegal instruction. Cannot catch this with
 243          * linker tricks when optimizations are disabled.
 244          */
 245         __asm__ __volatile__("ud2");
 246         return 0;
 247 }
 248
 249 #define _uatomic_add_return(addr, v)                                        \
 250         ((__typeof__(*(addr))) __uatomic_add_return((addr),                 \
 251                                                 caa_cast_long_keep_sign(v), \
 252                                                 sizeof(*(addr))))
 253
 254 /* uatomic_and */
 255
 256 static inline __attribute__((always_inline))
 257 void __uatomic_and(void *addr, unsigned long val, int len)
 258 {
 259         switch (len) {
 260         case 1:
 261         {
 262                 __asm__ __volatile__(
 263                 "lock; andb %1, %0"
 264                         : "=m"(*__hp(1, addr))
 265                         : "iq" ((unsigned char)val)
 266                         : "memory");
 267                 return;
 268         }
 269         case 2:
 270         {
 271                 __asm__ __volatile__(
 272                 "lock; andw %1, %0"
 273                         : "=m"(*__hp(2, addr))
 274                         : "ir" ((unsigned short)val)
 275                         : "memory");
 276                 return;
 277         }
 278         case 4:
 279         {
 280                 __asm__ __volatile__(
 281                 "lock; andl %1, %0"
 282                         : "=m"(*__hp(4, addr))
 283                         : "ir" ((unsigned int)val)
 284                         : "memory");
 285                 return;
 286         }
 287 #if (CAA_BITS_PER_LONG == 64)
 288         case 8:
 289         {
 290                 __asm__ __volatile__(
 291                 "lock; andq %1, %0"
 292                         : "=m"(*__hp(8, addr))
 293                         : "er" ((unsigned long)val)
 294                         : "memory");
 295                 return;
 296         }
 297 #endif
 298         }
 299         /*
 300          * generate an illegal instruction. Cannot catch this with
 301          * linker tricks when optimizations are disabled.
 302          */
 303         __asm__ __volatile__("ud2");
 304         return;
 305 }
 306
 307 #define _uatomic_and(addr, v)                                              \
 308         (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 309
 310 /* uatomic_or */
 311
 312 static inline __attribute__((always_inline))
 313 void __uatomic_or(void *addr, unsigned long val, int len)
 314 {
 315         switch (len) {
 316         case 1:
 317         {
 318                 __asm__ __volatile__(
 319                 "lock; orb %1, %0"
 320                         : "=m"(*__hp(1, addr))
 321                         : "iq" ((unsigned char)val)
 322                         : "memory");
 323                 return;
 324         }
 325         case 2:
 326         {
 327                 __asm__ __volatile__(
 328                 "lock; orw %1, %0"
 329                         : "=m"(*__hp(2, addr))
 330                         : "ir" ((unsigned short)val)
 331                         : "memory");
 332                 return;
 333         }
 334         case 4:
 335         {
 336                 __asm__ __volatile__(
 337                 "lock; orl %1, %0"
 338                         : "=m"(*__hp(4, addr))
 339                         : "ir" ((unsigned int)val)
 340                         : "memory");
 341                 return;
 342         }
 343 #if (CAA_BITS_PER_LONG == 64)
 344         case 8:
 345         {
 346                 __asm__ __volatile__(
 347                 "lock; orq %1, %0"
 348                         : "=m"(*__hp(8, addr))
 349                         : "er" ((unsigned long)val)
 350                         : "memory");
 351                 return;
 352         }
 353 #endif
 354         }
 355         /*
 356          * generate an illegal instruction. Cannot catch this with
 357          * linker tricks when optimizations are disabled.
 358          */
 359         __asm__ __volatile__("ud2");
 360         return;
 361 }
 362
 363 #define _uatomic_or(addr, v)                                               \
 364         (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 365
 366 /* uatomic_add */
 367
 368 static inline __attribute__((always_inline))
 369 void __uatomic_add(void *addr, unsigned long val, int len)
 370 {
 371         switch (len) {
 372         case 1:
 373         {
 374                 __asm__ __volatile__(
 375                 "lock; addb %1, %0"
 376                         : "=m"(*__hp(1, addr))
 377                         : "iq" ((unsigned char)val)
 378                         : "memory");
 379                 return;
 380         }
 381         case 2:
 382         {
 383                 __asm__ __volatile__(
 384                 "lock; addw %1, %0"
 385                         : "=m"(*__hp(2, addr))
 386                         : "ir" ((unsigned short)val)
 387                         : "memory");
 388                 return;
 389         }
 390         case 4:
 391         {
 392                 __asm__ __volatile__(
 393                 "lock; addl %1, %0"
 394                         : "=m"(*__hp(4, addr))
 395                         : "ir" ((unsigned int)val)
 396                         : "memory");
 397                 return;
 398         }
 399 #if (CAA_BITS_PER_LONG == 64)
 400         case 8:
 401         {
 402                 __asm__ __volatile__(
 403                 "lock; addq %1, %0"
 404                         : "=m"(*__hp(8, addr))
 405                         : "er" ((unsigned long)val)
 406                         : "memory");
 407                 return;
 408         }
 409 #endif
 410         }
 411         /*
 412          * generate an illegal instruction. Cannot catch this with
 413          * linker tricks when optimizations are disabled.
 414          */
 415         __asm__ __volatile__("ud2");
 416         return;
 417 }
 418
 419 #define _uatomic_add(addr, v)                                              \
 420         (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 421
 422
 423 /* uatomic_inc */
 424
 425 static inline __attribute__((always_inline))
 426 void __uatomic_inc(void *addr, int len)
 427 {
 428         switch (len) {
 429         case 1:
 430         {
 431                 __asm__ __volatile__(
 432                 "lock; incb %0"
 433                         : "=m"(*__hp(1, addr))
 434                         :
 435                         : "memory");
 436                 return;
 437         }
 438         case 2:
 439         {
 440                 __asm__ __volatile__(
 441                 "lock; incw %0"
 442                         : "=m"(*__hp(2, addr))
 443                         :
 444                         : "memory");
 445                 return;
 446         }
 447         case 4:
 448         {
 449                 __asm__ __volatile__(
 450                 "lock; incl %0"
 451                         : "=m"(*__hp(4, addr))
 452                         :
 453                         : "memory");
 454                 return;
 455         }
 456 #if (CAA_BITS_PER_LONG == 64)
 457         case 8:
 458         {
 459                 __asm__ __volatile__(
 460                 "lock; incq %0"
 461                         : "=m"(*__hp(8, addr))
 462                         :
 463                         : "memory");
 464                 return;
 465         }
 466 #endif
 467         }
 468         /* generate an illegal instruction. Cannot catch this with linker tricks
 469          * when optimizations are disabled. */
 470         __asm__ __volatile__("ud2");
 471         return;
 472 }
 473
 474 #define _uatomic_inc(addr)      (__uatomic_inc((addr), sizeof(*(addr))))
 475
 476 /* uatomic_dec */
 477
 478 static inline __attribute__((always_inline))
 479 void __uatomic_dec(void *addr, int len)
 480 {
 481         switch (len) {
 482         case 1:
 483         {
 484                 __asm__ __volatile__(
 485                 "lock; decb %0"
 486                         : "=m"(*__hp(1, addr))
 487                         :
 488                         : "memory");
 489                 return;
 490         }
 491         case 2:
 492         {
 493                 __asm__ __volatile__(
 494                 "lock; decw %0"
 495                         : "=m"(*__hp(2, addr))
 496                         :
 497                         : "memory");
 498                 return;
 499         }
 500         case 4:
 501         {
 502                 __asm__ __volatile__(
 503                 "lock; decl %0"
 504                         : "=m"(*__hp(4, addr))
 505                         :
 506                         : "memory");
 507                 return;
 508         }
 509 #if (CAA_BITS_PER_LONG == 64)
 510         case 8:
 511         {
 512                 __asm__ __volatile__(
 513                 "lock; decq %0"
 514                         : "=m"(*__hp(8, addr))
 515                         :
 516                         : "memory");
 517                 return;
 518         }
 519 #endif
 520         }
 521         /*
 522          * generate an illegal instruction. Cannot catch this with
 523          * linker tricks when optimizations are disabled.
 524          */
 525         __asm__ __volatile__("ud2");
 526         return;
 527 }
 528
 529 #define _uatomic_dec(addr)      (__uatomic_dec((addr), sizeof(*(addr))))
 530
 531 #if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
 532 extern int __rcu_cas_avail;
 533 extern int __rcu_cas_init(void);
 534
 535 #define UATOMIC_COMPAT(insn)                                                    \
 536         ((caa_likely(__rcu_cas_avail > 0))                                              \
 537         ? (_uatomic_##insn)                                                     \
 538                 : ((caa_unlikely(__rcu_cas_avail < 0)                           \
 539                         ? ((__rcu_cas_init() > 0)                               \
 540                                 ? (_uatomic_##insn)                             \
 541                                 : (compat_uatomic_##insn))                      \
 542                         : (compat_uatomic_##insn))))
 543
 544 /*
 545  * We leave the return value so we don't break the ABI, but remove the
 546  * return value from the API.
 547  */
 548 extern unsigned long _compat_uatomic_set(void *addr,
 549                                          unsigned long _new, int len);
 550 #define compat_uatomic_set(addr, _new)                                         \
 551         ((void) _compat_uatomic_set((addr),                                    \
 552                                 caa_cast_long_keep_sign(_new),                 \
 553                                 sizeof(*(addr))))
 554
 555
 556 extern unsigned long _compat_uatomic_xchg(void *addr,
 557                                           unsigned long _new, int len);
 558 #define compat_uatomic_xchg(addr, _new)                                        \
 559         ((__typeof__(*(addr))) _compat_uatomic_xchg((addr),                    \
 560                                                 caa_cast_long_keep_sign(_new), \
 561                                                 sizeof(*(addr))))
 562
 563 extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
 564                                              unsigned long _new, int len);
 565 #define compat_uatomic_cmpxchg(addr, old, _new)                                \
 566         ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),                 \
 567                                                 caa_cast_long_keep_sign(old),  \
 568                                                 caa_cast_long_keep_sign(_new), \
 569                                                 sizeof(*(addr))))
 570
 571 extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
 572 #define compat_uatomic_and(addr, v)                                    \
 573         (_compat_uatomic_and((addr),                                   \
 574                         caa_cast_long_keep_sign(v),                    \
 575                         sizeof(*(addr))))
 576
 577 extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
 578 #define compat_uatomic_or(addr, v)                                     \
 579         (_compat_uatomic_or((addr),                                    \
 580                           caa_cast_long_keep_sign(v),                  \
 581                           sizeof(*(addr))))
 582
 583 extern unsigned long _compat_uatomic_add_return(void *addr,
 584                                                 unsigned long _new, int len);
 585 #define compat_uatomic_add_return(addr, v)                                  \
 586         ((__typeof__(*(addr))) _compat_uatomic_add_return((addr),           \
 587                                                 caa_cast_long_keep_sign(v), \
 588                                                 sizeof(*(addr))))
 589
 590 #define compat_uatomic_add(addr, v)                                            \
 591                 ((void)compat_uatomic_add_return((addr), (v)))
 592 #define compat_uatomic_inc(addr)                                               \
 593                 (compat_uatomic_add((addr), 1))
 594 #define compat_uatomic_dec(addr)                                               \
 595                 (compat_uatomic_add((addr), -1))
 596
 597 #else
 598 #define UATOMIC_COMPAT(insn)    (_uatomic_##insn)
 599 #endif
 600
 601 /* Read is atomic even in compat mode */
 602 #define uatomic_set(addr, v)                    \
 603                 UATOMIC_COMPAT(set(addr, v))
 604
 605 #define uatomic_cmpxchg(addr, old, _new)        \
 606                 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
 607 #define uatomic_xchg(addr, v)                   \
 608                 UATOMIC_COMPAT(xchg(addr, v))
 609
 610 #define uatomic_and(addr, v)            \
 611                 UATOMIC_COMPAT(and(addr, v))
 612 #define cmm_smp_mb__before_uatomic_and()        cmm_barrier()
 613 #define cmm_smp_mb__after_uatomic_and()         cmm_barrier()
 614
 615 #define uatomic_or(addr, v)             \
 616                 UATOMIC_COMPAT(or(addr, v))
 617 #define cmm_smp_mb__before_uatomic_or()         cmm_barrier()
 618 #define cmm_smp_mb__after_uatomic_or()          cmm_barrier()
 619
 620 #define uatomic_add_return(addr, v)             \
 621                 UATOMIC_COMPAT(add_return(addr, v))
 622
 623 #define uatomic_add(addr, v)    UATOMIC_COMPAT(add(addr, v))
 624 #define cmm_smp_mb__before_uatomic_add()        cmm_barrier()
 625 #define cmm_smp_mb__after_uatomic_add()         cmm_barrier()
 626
 627 #define uatomic_inc(addr)       UATOMIC_COMPAT(inc(addr))
 628 #define cmm_smp_mb__before_uatomic_inc()        cmm_barrier()
 629 #define cmm_smp_mb__after_uatomic_inc()         cmm_barrier()
 630
 631 #define uatomic_dec(addr)       UATOMIC_COMPAT(dec(addr))
 632 #define cmm_smp_mb__before_uatomic_dec()        cmm_barrier()
 633 #define cmm_smp_mb__after_uatomic_dec()         cmm_barrier()
 634
 635 #ifdef __cplusplus
 636 }
 637 #endif
 638
 639 #include <urcu/uatomic/generic.h>
 640
 641 #endif /* _URCU_ARCH_UATOMIC_X86_H */