include/urcu/uatomic/x86.h

   1 #ifndef _URCU_ARCH_UATOMIC_X86_H
   2 #define _URCU_ARCH_UATOMIC_X86_H
   3
   4 /*
   5  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
   6  * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
   7  * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
   8  * Copyright (c) 2009      Mathieu Desnoyers
   9  *
  10  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  11  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  12  *
  13  * Permission is hereby granted to use or copy this program
  14  * for any purpose,  provided the above notices are retained on all copies.
  15  * Permission to modify the code and to distribute modified code is granted,
  16  * provided the above notices are retained, and a notice that the code was
  17  * modified is included with the above copyright notice.
  18  *
  19  * Code inspired from libuatomic_ops-1.2, inherited in part from the
  20  * Boehm-Demers-Weiser conservative garbage collector.
  21  */
  22
  23 #include <urcu/config.h>
  24 #include <urcu/compiler.h>
  25 #include <urcu/system.h>
  26
  27 #define UATOMIC_HAS_ATOMIC_BYTE
  28 #define UATOMIC_HAS_ATOMIC_SHORT
  29
  30 #ifdef __cplusplus
  31 extern "C" {
  32 #endif
  33
  34 /*
  35  * Derived from AO_compare_and_swap() and AO_test_and_set_full().
  36  */
  37
  38 struct __uatomic_dummy {
  39         unsigned long v[10];
  40 };
  41 #define __hp(x) ((struct __uatomic_dummy *)(x))
  42
  43 #define _uatomic_set(addr, v)   ((void) CMM_STORE_SHARED(*(addr), (v)))
  44
  45 /* cmpxchg */
  46
  47 static inline __attribute__((always_inline))
  48 unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
  49                               unsigned long _new, int len)
  50 {
  51         switch (len) {
  52         case 1:
  53         {
  54                 unsigned char result = old;
  55
  56                 __asm__ __volatile__(
  57                 "lock; cmpxchgb %2, %1"
  58                         : "+a"(result), "+m"(*__hp(addr))
  59                         : "q"((unsigned char)_new)
  60                         : "memory");
  61                 return result;
  62         }
  63         case 2:
  64         {
  65                 unsigned short result = old;
  66
  67                 __asm__ __volatile__(
  68                 "lock; cmpxchgw %2, %1"
  69                         : "+a"(result), "+m"(*__hp(addr))
  70                         : "r"((unsigned short)_new)
  71                         : "memory");
  72                 return result;
  73         }
  74         case 4:
  75         {
  76                 unsigned int result = old;
  77
  78                 __asm__ __volatile__(
  79                 "lock; cmpxchgl %2, %1"
  80                         : "+a"(result), "+m"(*__hp(addr))
  81                         : "r"((unsigned int)_new)
  82                         : "memory");
  83                 return result;
  84         }
  85 #if (CAA_BITS_PER_LONG == 64)
  86         case 8:
  87         {
  88                 unsigned long result = old;
  89
  90                 __asm__ __volatile__(
  91                 "lock; cmpxchgq %2, %1"
  92                         : "+a"(result), "+m"(*__hp(addr))
  93                         : "r"((unsigned long)_new)
  94                         : "memory");
  95                 return result;
  96         }
  97 #endif
  98         }
  99         /*
 100          * generate an illegal instruction. Cannot catch this with
 101          * linker tricks when optimizations are disabled.
 102          */
 103         __asm__ __volatile__("ud2");
 104         return 0;
 105 }
 106
 107 #define _uatomic_cmpxchg(addr, old, _new)                                     \
 108         ((__typeof__(*(addr))) __uatomic_cmpxchg((addr),                      \
 109                                                 caa_cast_long_keep_sign(old), \
 110                                                 caa_cast_long_keep_sign(_new),\
 111                                                 sizeof(*(addr))))
 112
 113 /* xchg */
 114
 115 static inline __attribute__((always_inline))
 116 unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
 117 {
 118         /* Note: the "xchg" instruction does not need a "lock" prefix. */
 119         switch (len) {
 120         case 1:
 121         {
 122                 unsigned char result;
 123                 __asm__ __volatile__(
 124                 "xchgb %0, %1"
 125                         : "=q"(result), "+m"(*__hp(addr))
 126                         : "0" ((unsigned char)val)
 127                         : "memory");
 128                 return result;
 129         }
 130         case 2:
 131         {
 132                 unsigned short result;
 133                 __asm__ __volatile__(
 134                 "xchgw %0, %1"
 135                         : "=r"(result), "+m"(*__hp(addr))
 136                         : "0" ((unsigned short)val)
 137                         : "memory");
 138                 return result;
 139         }
 140         case 4:
 141         {
 142                 unsigned int result;
 143                 __asm__ __volatile__(
 144                 "xchgl %0, %1"
 145                         : "=r"(result), "+m"(*__hp(addr))
 146                         : "0" ((unsigned int)val)
 147                         : "memory");
 148                 return result;
 149         }
 150 #if (CAA_BITS_PER_LONG == 64)
 151         case 8:
 152         {
 153                 unsigned long result;
 154                 __asm__ __volatile__(
 155                 "xchgq %0, %1"
 156                         : "=r"(result), "+m"(*__hp(addr))
 157                         : "0" ((unsigned long)val)
 158                         : "memory");
 159                 return result;
 160         }
 161 #endif
 162         }
 163         /*
 164          * generate an illegal instruction. Cannot catch this with
 165          * linker tricks when optimizations are disabled.
 166          */
 167         __asm__ __volatile__("ud2");
 168         return 0;
 169 }
 170
 171 #define _uatomic_xchg(addr, v)                                                \
 172         ((__typeof__(*(addr))) __uatomic_exchange((addr),                     \
 173                                                 caa_cast_long_keep_sign(v),   \
 174                                                 sizeof(*(addr))))
 175
 176 /* uatomic_add_return */
 177
 178 static inline __attribute__((always_inline))
 179 unsigned long __uatomic_add_return(void *addr, unsigned long val,
 180                                  int len)
 181 {
 182         switch (len) {
 183         case 1:
 184         {
 185                 unsigned char result = val;
 186
 187                 __asm__ __volatile__(
 188                 "lock; xaddb %1, %0"
 189                         : "+m"(*__hp(addr)), "+q" (result)
 190                         :
 191                         : "memory");
 192                 return result + (unsigned char)val;
 193         }
 194         case 2:
 195         {
 196                 unsigned short result = val;
 197
 198                 __asm__ __volatile__(
 199                 "lock; xaddw %1, %0"
 200                         : "+m"(*__hp(addr)), "+r" (result)
 201                         :
 202                         : "memory");
 203                 return result + (unsigned short)val;
 204         }
 205         case 4:
 206         {
 207                 unsigned int result = val;
 208
 209                 __asm__ __volatile__(
 210                 "lock; xaddl %1, %0"
 211                         : "+m"(*__hp(addr)), "+r" (result)
 212                         :
 213                         : "memory");
 214                 return result + (unsigned int)val;
 215         }
 216 #if (CAA_BITS_PER_LONG == 64)
 217         case 8:
 218         {
 219                 unsigned long result = val;
 220
 221                 __asm__ __volatile__(
 222                 "lock; xaddq %1, %0"
 223                         : "+m"(*__hp(addr)), "+r" (result)
 224                         :
 225                         : "memory");
 226                 return result + (unsigned long)val;
 227         }
 228 #endif
 229         }
 230         /*
 231          * generate an illegal instruction. Cannot catch this with
 232          * linker tricks when optimizations are disabled.
 233          */
 234         __asm__ __volatile__("ud2");
 235         return 0;
 236 }
 237
 238 #define _uatomic_add_return(addr, v)                                        \
 239         ((__typeof__(*(addr))) __uatomic_add_return((addr),                 \
 240                                                 caa_cast_long_keep_sign(v), \
 241                                                 sizeof(*(addr))))
 242
 243 /* uatomic_and */
 244
 245 static inline __attribute__((always_inline))
 246 void __uatomic_and(void *addr, unsigned long val, int len)
 247 {
 248         switch (len) {
 249         case 1:
 250         {
 251                 __asm__ __volatile__(
 252                 "lock; andb %1, %0"
 253                         : "=m"(*__hp(addr))
 254                         : "iq" ((unsigned char)val)
 255                         : "memory");
 256                 return;
 257         }
 258         case 2:
 259         {
 260                 __asm__ __volatile__(
 261                 "lock; andw %1, %0"
 262                         : "=m"(*__hp(addr))
 263                         : "ir" ((unsigned short)val)
 264                         : "memory");
 265                 return;
 266         }
 267         case 4:
 268         {
 269                 __asm__ __volatile__(
 270                 "lock; andl %1, %0"
 271                         : "=m"(*__hp(addr))
 272                         : "ir" ((unsigned int)val)
 273                         : "memory");
 274                 return;
 275         }
 276 #if (CAA_BITS_PER_LONG == 64)
 277         case 8:
 278         {
 279                 __asm__ __volatile__(
 280                 "lock; andq %1, %0"
 281                         : "=m"(*__hp(addr))
 282                         : "er" ((unsigned long)val)
 283                         : "memory");
 284                 return;
 285         }
 286 #endif
 287         }
 288         /*
 289          * generate an illegal instruction. Cannot catch this with
 290          * linker tricks when optimizations are disabled.
 291          */
 292         __asm__ __volatile__("ud2");
 293         return;
 294 }
 295
 296 #define _uatomic_and(addr, v)                                              \
 297         (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 298
 299 /* uatomic_or */
 300
 301 static inline __attribute__((always_inline))
 302 void __uatomic_or(void *addr, unsigned long val, int len)
 303 {
 304         switch (len) {
 305         case 1:
 306         {
 307                 __asm__ __volatile__(
 308                 "lock; orb %1, %0"
 309                         : "=m"(*__hp(addr))
 310                         : "iq" ((unsigned char)val)
 311                         : "memory");
 312                 return;
 313         }
 314         case 2:
 315         {
 316                 __asm__ __volatile__(
 317                 "lock; orw %1, %0"
 318                         : "=m"(*__hp(addr))
 319                         : "ir" ((unsigned short)val)
 320                         : "memory");
 321                 return;
 322         }
 323         case 4:
 324         {
 325                 __asm__ __volatile__(
 326                 "lock; orl %1, %0"
 327                         : "=m"(*__hp(addr))
 328                         : "ir" ((unsigned int)val)
 329                         : "memory");
 330                 return;
 331         }
 332 #if (CAA_BITS_PER_LONG == 64)
 333         case 8:
 334         {
 335                 __asm__ __volatile__(
 336                 "lock; orq %1, %0"
 337                         : "=m"(*__hp(addr))
 338                         : "er" ((unsigned long)val)
 339                         : "memory");
 340                 return;
 341         }
 342 #endif
 343         }
 344         /*
 345          * generate an illegal instruction. Cannot catch this with
 346          * linker tricks when optimizations are disabled.
 347          */
 348         __asm__ __volatile__("ud2");
 349         return;
 350 }
 351
 352 #define _uatomic_or(addr, v)                                               \
 353         (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 354
 355 /* uatomic_add */
 356
 357 static inline __attribute__((always_inline))
 358 void __uatomic_add(void *addr, unsigned long val, int len)
 359 {
 360         switch (len) {
 361         case 1:
 362         {
 363                 __asm__ __volatile__(
 364                 "lock; addb %1, %0"
 365                         : "=m"(*__hp(addr))
 366                         : "iq" ((unsigned char)val)
 367                         : "memory");
 368                 return;
 369         }
 370         case 2:
 371         {
 372                 __asm__ __volatile__(
 373                 "lock; addw %1, %0"
 374                         : "=m"(*__hp(addr))
 375                         : "ir" ((unsigned short)val)
 376                         : "memory");
 377                 return;
 378         }
 379         case 4:
 380         {
 381                 __asm__ __volatile__(
 382                 "lock; addl %1, %0"
 383                         : "=m"(*__hp(addr))
 384                         : "ir" ((unsigned int)val)
 385                         : "memory");
 386                 return;
 387         }
 388 #if (CAA_BITS_PER_LONG == 64)
 389         case 8:
 390         {
 391                 __asm__ __volatile__(
 392                 "lock; addq %1, %0"
 393                         : "=m"(*__hp(addr))
 394                         : "er" ((unsigned long)val)
 395                         : "memory");
 396                 return;
 397         }
 398 #endif
 399         }
 400         /*
 401          * generate an illegal instruction. Cannot catch this with
 402          * linker tricks when optimizations are disabled.
 403          */
 404         __asm__ __volatile__("ud2");
 405         return;
 406 }
 407
 408 #define _uatomic_add(addr, v)                                              \
 409         (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
 410
 411
 412 /* uatomic_inc */
 413
 414 static inline __attribute__((always_inline))
 415 void __uatomic_inc(void *addr, int len)
 416 {
 417         switch (len) {
 418         case 1:
 419         {
 420                 __asm__ __volatile__(
 421                 "lock; incb %0"
 422                         : "=m"(*__hp(addr))
 423                         :
 424                         : "memory");
 425                 return;
 426         }
 427         case 2:
 428         {
 429                 __asm__ __volatile__(
 430                 "lock; incw %0"
 431                         : "=m"(*__hp(addr))
 432                         :
 433                         : "memory");
 434                 return;
 435         }
 436         case 4:
 437         {
 438                 __asm__ __volatile__(
 439                 "lock; incl %0"
 440                         : "=m"(*__hp(addr))
 441                         :
 442                         : "memory");
 443                 return;
 444         }
 445 #if (CAA_BITS_PER_LONG == 64)
 446         case 8:
 447         {
 448                 __asm__ __volatile__(
 449                 "lock; incq %0"
 450                         : "=m"(*__hp(addr))
 451                         :
 452                         : "memory");
 453                 return;
 454         }
 455 #endif
 456         }
 457         /* generate an illegal instruction. Cannot catch this with linker tricks
 458          * when optimizations are disabled. */
 459         __asm__ __volatile__("ud2");
 460         return;
 461 }
 462
 463 #define _uatomic_inc(addr)      (__uatomic_inc((addr), sizeof(*(addr))))
 464
 465 /* uatomic_dec */
 466
 467 static inline __attribute__((always_inline))
 468 void __uatomic_dec(void *addr, int len)
 469 {
 470         switch (len) {
 471         case 1:
 472         {
 473                 __asm__ __volatile__(
 474                 "lock; decb %0"
 475                         : "=m"(*__hp(addr))
 476                         :
 477                         : "memory");
 478                 return;
 479         }
 480         case 2:
 481         {
 482                 __asm__ __volatile__(
 483                 "lock; decw %0"
 484                         : "=m"(*__hp(addr))
 485                         :
 486                         : "memory");
 487                 return;
 488         }
 489         case 4:
 490         {
 491                 __asm__ __volatile__(
 492                 "lock; decl %0"
 493                         : "=m"(*__hp(addr))
 494                         :
 495                         : "memory");
 496                 return;
 497         }
 498 #if (CAA_BITS_PER_LONG == 64)
 499         case 8:
 500         {
 501                 __asm__ __volatile__(
 502                 "lock; decq %0"
 503                         : "=m"(*__hp(addr))
 504                         :
 505                         : "memory");
 506                 return;
 507         }
 508 #endif
 509         }
 510         /*
 511          * generate an illegal instruction. Cannot catch this with
 512          * linker tricks when optimizations are disabled.
 513          */
 514         __asm__ __volatile__("ud2");
 515         return;
 516 }
 517
 518 #define _uatomic_dec(addr)      (__uatomic_dec((addr), sizeof(*(addr))))
 519
 520 #if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
 521 extern int __rcu_cas_avail;
 522 extern int __rcu_cas_init(void);
 523
 524 #define UATOMIC_COMPAT(insn)                                                    \
 525         ((caa_likely(__rcu_cas_avail > 0))                                              \
 526         ? (_uatomic_##insn)                                                     \
 527                 : ((caa_unlikely(__rcu_cas_avail < 0)                           \
 528                         ? ((__rcu_cas_init() > 0)                               \
 529                                 ? (_uatomic_##insn)                             \
 530                                 : (compat_uatomic_##insn))                      \
 531                         : (compat_uatomic_##insn))))
 532
 533 /*
 534  * We leave the return value so we don't break the ABI, but remove the
 535  * return value from the API.
 536  */
 537 extern unsigned long _compat_uatomic_set(void *addr,
 538                                          unsigned long _new, int len);
 539 #define compat_uatomic_set(addr, _new)                                         \
 540         ((void) _compat_uatomic_set((addr),                                    \
 541                                 caa_cast_long_keep_sign(_new),                 \
 542                                 sizeof(*(addr))))
 543
 544
 545 extern unsigned long _compat_uatomic_xchg(void *addr,
 546                                           unsigned long _new, int len);
 547 #define compat_uatomic_xchg(addr, _new)                                        \
 548         ((__typeof__(*(addr))) _compat_uatomic_xchg((addr),                    \
 549                                                 caa_cast_long_keep_sign(_new), \
 550                                                 sizeof(*(addr))))
 551
 552 extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
 553                                              unsigned long _new, int len);
 554 #define compat_uatomic_cmpxchg(addr, old, _new)                                \
 555         ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),                 \
 556                                                 caa_cast_long_keep_sign(old),  \
 557                                                 caa_cast_long_keep_sign(_new), \
 558                                                 sizeof(*(addr))))
 559
 560 extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
 561 #define compat_uatomic_and(addr, v)                                    \
 562         (_compat_uatomic_and((addr),                                   \
 563                         caa_cast_long_keep_sign(v),                    \
 564                         sizeof(*(addr))))
 565
 566 extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
 567 #define compat_uatomic_or(addr, v)                                     \
 568         (_compat_uatomic_or((addr),                                    \
 569                           caa_cast_long_keep_sign(v),                  \
 570                           sizeof(*(addr))))
 571
 572 extern unsigned long _compat_uatomic_add_return(void *addr,
 573                                                 unsigned long _new, int len);
 574 #define compat_uatomic_add_return(addr, v)                                  \
 575         ((__typeof__(*(addr))) _compat_uatomic_add_return((addr),           \
 576                                                 caa_cast_long_keep_sign(v), \
 577                                                 sizeof(*(addr))))
 578
 579 #define compat_uatomic_add(addr, v)                                            \
 580                 ((void)compat_uatomic_add_return((addr), (v)))
 581 #define compat_uatomic_inc(addr)                                               \
 582                 (compat_uatomic_add((addr), 1))
 583 #define compat_uatomic_dec(addr)                                               \
 584                 (compat_uatomic_add((addr), -1))
 585
 586 #else
 587 #define UATOMIC_COMPAT(insn)    (_uatomic_##insn)
 588 #endif
 589
 590 /* Read is atomic even in compat mode */
 591 #define uatomic_set(addr, v)                    \
 592                 UATOMIC_COMPAT(set(addr, v))
 593
 594 #define uatomic_cmpxchg(addr, old, _new)        \
 595                 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
 596 #define uatomic_xchg(addr, v)                   \
 597                 UATOMIC_COMPAT(xchg(addr, v))
 598
 599 #define uatomic_and(addr, v)            \
 600                 UATOMIC_COMPAT(and(addr, v))
 601 #define cmm_smp_mb__before_uatomic_and()        cmm_barrier()
 602 #define cmm_smp_mb__after_uatomic_and()         cmm_barrier()
 603
 604 #define uatomic_or(addr, v)             \
 605                 UATOMIC_COMPAT(or(addr, v))
 606 #define cmm_smp_mb__before_uatomic_or()         cmm_barrier()
 607 #define cmm_smp_mb__after_uatomic_or()          cmm_barrier()
 608
 609 #define uatomic_add_return(addr, v)             \
 610                 UATOMIC_COMPAT(add_return(addr, v))
 611
 612 #define uatomic_add(addr, v)    UATOMIC_COMPAT(add(addr, v))
 613 #define cmm_smp_mb__before_uatomic_add()        cmm_barrier()
 614 #define cmm_smp_mb__after_uatomic_add()         cmm_barrier()
 615
 616 #define uatomic_inc(addr)       UATOMIC_COMPAT(inc(addr))
 617 #define cmm_smp_mb__before_uatomic_inc()        cmm_barrier()
 618 #define cmm_smp_mb__after_uatomic_inc()         cmm_barrier()
 619
 620 #define uatomic_dec(addr)       UATOMIC_COMPAT(dec(addr))
 621 #define cmm_smp_mb__before_uatomic_dec()        cmm_barrier()
 622 #define cmm_smp_mb__after_uatomic_dec()         cmm_barrier()
 623
 624 #ifdef __cplusplus
 625 }
 626 #endif
 627
 628 #include <urcu/uatomic/generic.h>
 629
 630 #endif /* _URCU_ARCH_UATOMIC_X86_H */