include/urcu/uatomic/x86.h

   1 #ifndef _URCU_ARCH_UATOMIC_X86_H
   2 #define _URCU_ARCH_UATOMIC_X86_H
   3
   4 /*
   5  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
   6  * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
   7  * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
   8  * Copyright (c) 2009      Mathieu Desnoyers
   9  *
  10  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  11  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  12  *
  13  * Permission is hereby granted to use or copy this program
  14  * for any purpose,  provided the above notices are retained on all copies.
  15  * Permission to modify the code and to distribute modified code is granted,
  16  * provided the above notices are retained, and a notice that the code was
  17  * modified is included with the above copyright notice.
  18  *
  19  * Code inspired from libuatomic_ops-1.2, inherited in part from the
  20  * Boehm-Demers-Weiser conservative garbage collector.
  21  */
  22
  23 #include <urcu/arch.h>
  24 #include <urcu/config.h>
  25 #include <urcu/compiler.h>
  26 #include <urcu/system.h>
  27
  28 #define UATOMIC_HAS_ATOMIC_BYTE
  29 #define UATOMIC_HAS_ATOMIC_SHORT
  30
  31 #ifdef __cplusplus
  32 extern "C" {
  33 #endif
  34
  35 /*
  36  * Derived from AO_compare_and_swap() and AO_test_and_set_full().
  37  */
  38
  39 /*
  40  * The __hp() macro casts the void pointer @x to a pointer to a structure
  41  * containing an array of char of the specified size. This allows passing the
  42  * @addr arguments of the following inline functions as "m" and "+m" operands
  43  * to the assembly. The @size parameter should be a constant to support
  44  * compilers such as clang which do not support VLA.
  45  */
  46
  47 #define __hp(size, x)   ((struct { char v[size]; } *)(x))
  48
  49 #define _uatomic_set(addr, v)   ((void) CMM_STORE_SHARED(*(addr), (v)))
  50
  51 /* cmpxchg */
  52
  53 static inline __attribute__((always_inline))
  54 unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
  55                               unsigned long _new, int len)
  56 {
  57         switch (len) {
  58         case 1:
  59         {
  60                 unsigned char result = old;
  61
  62                 __asm__ __volatile__(
  63                 "lock; cmpxchgb %2, %1"
  64                         : "+a"(result), "+m"(*__hp(1, addr))
  65                         : "q"((unsigned char)_new)
  66                         : "memory");
  67                 return result;
  68         }
  69         case 2:
  70         {
  71                 unsigned short result = old;
  72
  73                 __asm__ __volatile__(
  74                 "lock; cmpxchgw %2, %1"
  75                         : "+a"(result), "+m"(*__hp(2, addr))
  76                         : "r"((unsigned short)_new)
  77                         : "memory");
  78                 return result;
  79         }
  80         case 4:
  81         {
  82                 unsigned int result = old;
  83
  84                 __asm__ __volatile__(
  85                 "lock; cmpxchgl %2, %1"
  86                         : "+a"(result), "+m"(*__hp(4, addr))
  87                         : "r"((unsigned int)_new)
  88                         : "memory");
  89                 return result;
  90         }
  91 #if (CAA_BITS_PER_LONG == 64)
  92         case 8:
  93         {
  94                 unsigned long result = old;
  95
  96                 __asm__ __volatile__(
  97                 "lock; cmpxchgq %2, %1"
  98                         : "+a"(result), "+m"(*__hp(8, addr))
  99                         : "r"((unsigned long)_new)
 100                         : "memory");
 101                 return result;
 102         }
 103 #endif
 104         }
 105         /*
 106          * generate an illegal instruction. Cannot catch this with
 107          * linker tricks when optimizations are disabled.
 108          */
 109         __asm__ __volatile__("ud2");
 110         return 0;
 111 }
 112
 113 #define _uatomic_cmpxchg(addr, old, _new)                                     \
 114         ((__typeof__(*(addr))) __uatomic_cmpxchg((addr),                      \
 115                                                 caa_cast_long_keep_sign(old), \
 116                                                 caa_cast_long_keep_sign(_new),\
 117                                                 sizeof(*(addr))))
 118
 119 /* xchg */
 120
 121 static inline __attribute__((always_inline))
 122 unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
 123 {
 124         /* Note: the "xchg" instruction does not need a "lock" prefix. */
 125         switch (len) {
 126         case 1:
 127         {
 128                 unsigned char result;
 129                 __asm__ __volatile__(
 130                 "xchgb %0, %1"
 131                         : "=q"(result), "+m"(*__hp(1, addr))
 132                         : "0" ((unsigned char)val)
 133                         : "memory");
 134                 return result;
 135         }
 136         case 2:
 137         {
 138                 unsigned short result;
 139                 __asm__ __volatile__(
 140                 "xchgw %0, %1"
 141                         : "=r"(result), "+m"(*__hp(2, addr))
 142                         : "0" ((unsigned short)val)
 143                         : "memory");
 144                 return result;
 145         }
 146         case 4:
 147         {
 148                 unsigned int result;
 149                 __asm__ __volatile__(
 150                 "xchgl %0, %1"
 151                         : "=r"(result), "+m"(*__hp(4, addr))
 152                         : "0" ((unsigned int)val)
 153                         : "memory");
 154                 return result;
 155         }
 156 #if (CAA_BITS_PER_LONG == 64)
 157         case 8:
 158         {
 159                 unsigned long result;
 160                 __asm__ __volatile__(
 161                 "xchgq %0, %1"
 162                         : "=r"(result), "+m"(*__hp(8, addr))
 163                         : "0" ((unsigned long)val)
 164                         : "memory");
 165                 return result;
 166         }
 167 #endif
 168         }
 169         /*
 170          * generate an illegal instruction. Cannot catch this with
 171          * linker tricks when optimizations are disabled.
 172          */
 173         __asm__ __volatile__("ud2");
 174         return 0;
 175 }
 176
 177 #define _uatomic_xchg(addr, v)                                                \
 178         ((__typeof__(*(addr))) __uatomic_exchange((addr),                     \
 179                                                 caa_cast_long_keep_sign(v),   \
 180                                                 sizeof(*(addr))))
 181
 182 /* uatomic_add_return */
 183
 184 static inline __attribute__((always_inline))
 185 unsigned long __uatomic_add_return(void *addr, unsigned long val,
 186                                  int len)
 187 {
 188         switch (len) {
 189         case 1:
 190         {
 191                 unsigned char result = val;
 192
 193                 __asm__ __volatile__(
 194                 "lock; xaddb %1, %0"
 195                         : "+m"(*__hp(1, addr)), "+q" (result)
 196                         :
 197                         : "memory");
 198                 return result + (unsigned char)val;
 199         }
 200         case 2:
 201         {
 202                 unsigned short result = val;
 203
 204                 __asm__ __volatile__(
 205                 "lock; xaddw %1, %0"
 206                         : "+m"(*__hp(2, addr)), "+r" (result)
 207                         :
 208                         : "memory");
 209                 return result + (unsigned short)val;
 210         }
 211         case 4:
 212         {
 213                 unsigned int result = val;
 214
 215                 __asm__ __volatile__(
 216                 "lock; xaddl %1, %0"
 217                         : "+m"(*__hp(4, addr)), "+r" (result)
 218                         :
 219                         : "memory");
 220                 return result + (unsigned int)val;
 221         }
 222 #if (CAA_BITS_PER_LONG == 64)
 223         case 8:
 224         {
 225                 unsigned long result = val;
 226
 227                 __asm__ __volatile__(
 228                 "lock; xaddq %1, %0"
 229                         : "+m"(*__hp(8, addr)), "+r" (result)
 230                         :
 231                         : "memory");
 232                 return result + (unsigned long)val;
 233         }
 234 #endif
 235         }
 236         /*
 237          * generate an illegal instruction. Cannot catch this with
 238          * linker tricks when optimizations are disabled.
 239          */
 240         __asm__ __volatile__("ud2");
 241         return 0;
 242 }
 243
 244 #define _uatomic_add_return(addr, v)                                        \
 245         ((__typeof__(*(addr))) __uatomic_add_return((addr),                 \
 246                                                 caa_cast_long_keep_sign(v), \
 247                                                 sizeof(*(addr))))
 248
 249 /* uatomic_and */
 250
 251 static inline __attribute__((always_inline))
 252 void __uatomic_and(void *addr, unsigned long val, int len)
 253 {
 254         switch (len) {
 255         case 1:
 256         {
 257                 __asm__ __volatile__(
 258                 "lock; andb %1, %0"
 259                         : "=m"(*__hp(1, addr))
 260                         : "iq" ((unsigned char)val)
 261                         : "memory");
 262                 return;
 263         }
 264         case 2:
 265         {
 266                 __asm__ __volatile__(
 267                 "lock; andw %1, %0"
 268                         : "=m"(*__hp(2, addr))
 269                         : "ir" ((unsigned short)val)
 270                         : "memory");
 271                 return;
 272         }
 273         case 4:
 274         {
 275                 __asm__ __volatile__(
 276                 "lock; andl %1, %0"
 277                         : "=m"(*__hp(4, addr))
 278                         : "ir" ((unsigned int)val)
 279                         : "memory");
 280                 return;
 281         }
 282 #if (CAA_BITS_PER_LONG == 64)
 283         case 8:
 284         {
 285                 __asm__ __volatile__(
 286                 "lock; andq %1, %0"
 287                         : "=m"(*__hp(8, addr))
 288                         : "er" ((unsigned long)val)
 289                         : "memory");
 290                 return;
 291         }
 292 #endif
 293         }
 294         /*
 295          * generate an illegal instruction. Cannot catch this with
 296          * linker tricks when optimizations are disabled.
 297          */
 298         __asm__ __volatile__("ud2");
 299         return;
 300 }
 301
 302 #define _uatomic_and(addr, v)                                              \
 303         (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 304
 305 /* uatomic_or */
 306
 307 static inline __attribute__((always_inline))
 308 void __uatomic_or(void *addr, unsigned long val, int len)
 309 {
 310         switch (len) {
 311         case 1:
 312         {
 313                 __asm__ __volatile__(
 314                 "lock; orb %1, %0"
 315                         : "=m"(*__hp(1, addr))
 316                         : "iq" ((unsigned char)val)
 317                         : "memory");
 318                 return;
 319         }
 320         case 2:
 321         {
 322                 __asm__ __volatile__(
 323                 "lock; orw %1, %0"
 324                         : "=m"(*__hp(2, addr))
 325                         : "ir" ((unsigned short)val)
 326                         : "memory");
 327                 return;
 328         }
 329         case 4:
 330         {
 331                 __asm__ __volatile__(
 332                 "lock; orl %1, %0"
 333                         : "=m"(*__hp(4, addr))
 334                         : "ir" ((unsigned int)val)
 335                         : "memory");
 336                 return;
 337         }
 338 #if (CAA_BITS_PER_LONG == 64)
 339         case 8:
 340         {
 341                 __asm__ __volatile__(
 342                 "lock; orq %1, %0"
 343                         : "=m"(*__hp(8, addr))
 344                         : "er" ((unsigned long)val)
 345                         : "memory");
 346                 return;
 347         }
 348 #endif
 349         }
 350         /*
 351          * generate an illegal instruction. Cannot catch this with
 352          * linker tricks when optimizations are disabled.
 353          */
 354         __asm__ __volatile__("ud2");
 355         return;
 356 }
 357
 358 #define _uatomic_or(addr, v)                                               \
 359         (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 360
 361 /* uatomic_add */
 362
 363 static inline __attribute__((always_inline))
 364 void __uatomic_add(void *addr, unsigned long val, int len)
 365 {
 366         switch (len) {
 367         case 1:
 368         {
 369                 __asm__ __volatile__(
 370                 "lock; addb %1, %0"
 371                         : "=m"(*__hp(1, addr))
 372                         : "iq" ((unsigned char)val)
 373                         : "memory");
 374                 return;
 375         }
 376         case 2:
 377         {
 378                 __asm__ __volatile__(
 379                 "lock; addw %1, %0"
 380                         : "=m"(*__hp(2, addr))
 381                         : "ir" ((unsigned short)val)
 382                         : "memory");
 383                 return;
 384         }
 385         case 4:
 386         {
 387                 __asm__ __volatile__(
 388                 "lock; addl %1, %0"
 389                         : "=m"(*__hp(4, addr))
 390                         : "ir" ((unsigned int)val)
 391                         : "memory");
 392                 return;
 393         }
 394 #if (CAA_BITS_PER_LONG == 64)
 395         case 8:
 396         {
 397                 __asm__ __volatile__(
 398                 "lock; addq %1, %0"
 399                         : "=m"(*__hp(8, addr))
 400                         : "er" ((unsigned long)val)
 401                         : "memory");
 402                 return;
 403         }
 404 #endif
 405         }
 406         /*
 407          * generate an illegal instruction. Cannot catch this with
 408          * linker tricks when optimizations are disabled.
 409          */
 410         __asm__ __volatile__("ud2");
 411         return;
 412 }
 413
 414 #define _uatomic_add(addr, v)                                              \
 415         (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 416
 417
 418 /* uatomic_inc */
 419
 420 static inline __attribute__((always_inline))
 421 void __uatomic_inc(void *addr, int len)
 422 {
 423         switch (len) {
 424         case 1:
 425         {
 426                 __asm__ __volatile__(
 427                 "lock; incb %0"
 428                         : "=m"(*__hp(1, addr))
 429                         :
 430                         : "memory");
 431                 return;
 432         }
 433         case 2:
 434         {
 435                 __asm__ __volatile__(
 436                 "lock; incw %0"
 437                         : "=m"(*__hp(2, addr))
 438                         :
 439                         : "memory");
 440                 return;
 441         }
 442         case 4:
 443         {
 444                 __asm__ __volatile__(
 445                 "lock; incl %0"
 446                         : "=m"(*__hp(4, addr))
 447                         :
 448                         : "memory");
 449                 return;
 450         }
 451 #if (CAA_BITS_PER_LONG == 64)
 452         case 8:
 453         {
 454                 __asm__ __volatile__(
 455                 "lock; incq %0"
 456                         : "=m"(*__hp(8, addr))
 457                         :
 458                         : "memory");
 459                 return;
 460         }
 461 #endif
 462         }
 463         /* generate an illegal instruction. Cannot catch this with linker tricks
 464          * when optimizations are disabled. */
 465         __asm__ __volatile__("ud2");
 466         return;
 467 }
 468
 469 #define _uatomic_inc(addr)      (__uatomic_inc((addr), sizeof(*(addr))))
 470
 471 /* uatomic_dec */
 472
 473 static inline __attribute__((always_inline))
 474 void __uatomic_dec(void *addr, int len)
 475 {
 476         switch (len) {
 477         case 1:
 478         {
 479                 __asm__ __volatile__(
 480                 "lock; decb %0"
 481                         : "=m"(*__hp(1, addr))
 482                         :
 483                         : "memory");
 484                 return;
 485         }
 486         case 2:
 487         {
 488                 __asm__ __volatile__(
 489                 "lock; decw %0"
 490                         : "=m"(*__hp(2, addr))
 491                         :
 492                         : "memory");
 493                 return;
 494         }
 495         case 4:
 496         {
 497                 __asm__ __volatile__(
 498                 "lock; decl %0"
 499                         : "=m"(*__hp(4, addr))
 500                         :
 501                         : "memory");
 502                 return;
 503         }
 504 #if (CAA_BITS_PER_LONG == 64)
 505         case 8:
 506         {
 507                 __asm__ __volatile__(
 508                 "lock; decq %0"
 509                         : "=m"(*__hp(8, addr))
 510                         :
 511                         : "memory");
 512                 return;
 513         }
 514 #endif
 515         }
 516         /*
 517          * generate an illegal instruction. Cannot catch this with
 518          * linker tricks when optimizations are disabled.
 519          */
 520         __asm__ __volatile__("ud2");
 521         return;
 522 }
 523
 524 #define _uatomic_dec(addr)      (__uatomic_dec((addr), sizeof(*(addr))))
 525
 526 #if ((CAA_BITS_PER_LONG != 64) && defined(URCU_ARCH_I386))
 527
 528 /* For backwards compat */
 529 #define CONFIG_RCU_COMPAT_ARCH 1
 530
 531 extern int __rcu_cas_avail;
 532 extern int __rcu_cas_init(void);
 533
 534 #define UATOMIC_COMPAT(insn)                                                    \
 535         ((caa_likely(__rcu_cas_avail > 0))                                              \
 536         ? (_uatomic_##insn)                                                     \
 537                 : ((caa_unlikely(__rcu_cas_avail < 0)                           \
 538                         ? ((__rcu_cas_init() > 0)                               \
 539                                 ? (_uatomic_##insn)                             \
 540                                 : (compat_uatomic_##insn))                      \
 541                         : (compat_uatomic_##insn))))
 542
 543 /*
 544  * We leave the return value so we don't break the ABI, but remove the
 545  * return value from the API.
 546  */
 547 extern unsigned long _compat_uatomic_set(void *addr,
 548                                          unsigned long _new, int len);
 549 #define compat_uatomic_set(addr, _new)                                         \
 550         ((void) _compat_uatomic_set((addr),                                    \
 551                                 caa_cast_long_keep_sign(_new),                 \
 552                                 sizeof(*(addr))))
 553
 554
 555 extern unsigned long _compat_uatomic_xchg(void *addr,
 556                                           unsigned long _new, int len);
 557 #define compat_uatomic_xchg(addr, _new)                                        \
 558         ((__typeof__(*(addr))) _compat_uatomic_xchg((addr),                    \
 559                                                 caa_cast_long_keep_sign(_new), \
 560                                                 sizeof(*(addr))))
 561
 562 extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
 563                                              unsigned long _new, int len);
 564 #define compat_uatomic_cmpxchg(addr, old, _new)                                \
 565         ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),                 \
 566                                                 caa_cast_long_keep_sign(old),  \
 567                                                 caa_cast_long_keep_sign(_new), \
 568                                                 sizeof(*(addr))))
 569
 570 extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
 571 #define compat_uatomic_and(addr, v)                                    \
 572         (_compat_uatomic_and((addr),                                   \
 573                         caa_cast_long_keep_sign(v),                    \
 574                         sizeof(*(addr))))
 575
 576 extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
 577 #define compat_uatomic_or(addr, v)                                     \
 578         (_compat_uatomic_or((addr),                                    \
 579                           caa_cast_long_keep_sign(v),                  \
 580                           sizeof(*(addr))))
 581
 582 extern unsigned long _compat_uatomic_add_return(void *addr,
 583                                                 unsigned long _new, int len);
 584 #define compat_uatomic_add_return(addr, v)                                  \
 585         ((__typeof__(*(addr))) _compat_uatomic_add_return((addr),           \
 586                                                 caa_cast_long_keep_sign(v), \
 587                                                 sizeof(*(addr))))
 588
 589 #define compat_uatomic_add(addr, v)                                            \
 590                 ((void)compat_uatomic_add_return((addr), (v)))
 591 #define compat_uatomic_inc(addr)                                               \
 592                 (compat_uatomic_add((addr), 1))
 593 #define compat_uatomic_dec(addr)                                               \
 594                 (compat_uatomic_add((addr), -1))
 595
 596 #else
 597 #define UATOMIC_COMPAT(insn)    (_uatomic_##insn)
 598 #endif
 599
 600 /* Read is atomic even in compat mode */
 601 #define uatomic_set(addr, v)                    \
 602                 UATOMIC_COMPAT(set(addr, v))
 603
 604 #define uatomic_cmpxchg(addr, old, _new)        \
 605                 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
 606 #define uatomic_xchg(addr, v)                   \
 607                 UATOMIC_COMPAT(xchg(addr, v))
 608
 609 #define uatomic_and(addr, v)            \
 610                 UATOMIC_COMPAT(and(addr, v))
 611 #define cmm_smp_mb__before_uatomic_and()        cmm_barrier()
 612 #define cmm_smp_mb__after_uatomic_and()         cmm_barrier()
 613
 614 #define uatomic_or(addr, v)             \
 615                 UATOMIC_COMPAT(or(addr, v))
 616 #define cmm_smp_mb__before_uatomic_or()         cmm_barrier()
 617 #define cmm_smp_mb__after_uatomic_or()          cmm_barrier()
 618
 619 #define uatomic_add_return(addr, v)             \
 620                 UATOMIC_COMPAT(add_return(addr, v))
 621
 622 #define uatomic_add(addr, v)    UATOMIC_COMPAT(add(addr, v))
 623 #define cmm_smp_mb__before_uatomic_add()        cmm_barrier()
 624 #define cmm_smp_mb__after_uatomic_add()         cmm_barrier()
 625
 626 #define uatomic_inc(addr)       UATOMIC_COMPAT(inc(addr))
 627 #define cmm_smp_mb__before_uatomic_inc()        cmm_barrier()
 628 #define cmm_smp_mb__after_uatomic_inc()         cmm_barrier()
 629
 630 #define uatomic_dec(addr)       UATOMIC_COMPAT(dec(addr))
 631 #define cmm_smp_mb__before_uatomic_dec()        cmm_barrier()
 632 #define cmm_smp_mb__after_uatomic_dec()         cmm_barrier()
 633
 634 #ifdef __cplusplus
 635 }
 636 #endif
 637
 638 #include <urcu/uatomic/generic.h>
 639
 640 #endif /* _URCU_ARCH_UATOMIC_X86_H */