include/urcu/uatomic/generic.h

   1 // SPDX-FileCopyrightText: 1991-1994 by Xerox Corporation.  All rights reserved.
   2 // SPDX-FileCopyrightText: 1996-1999 by Silicon Graphics.  All rights reserved.
   3 // SPDX-FileCopyrightText: 1999-2004 Hewlett-Packard Development Company, L.P.
   4 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5 // SPDX-FileCopyrightText: 2010 Paolo Bonzini
   6 //
   7 // SPDX-License-Identifier: LicenseRef-Boehm-GC
   8
   9 #ifndef _URCU_UATOMIC_GENERIC_H
  10 #define _URCU_UATOMIC_GENERIC_H
  11
  12 /*
  13  * Code inspired from libuatomic_ops-1.2, inherited in part from the
  14  * Boehm-Demers-Weiser conservative garbage collector.
  15  */
  16
  17 #include <stdint.h>
  18 #include <urcu/compiler.h>
  19 #include <urcu/system.h>
  20
  21 #ifdef __cplusplus
  22 extern "C" {
  23 #endif
  24
  25 #ifndef uatomic_set
  26 #define uatomic_set(addr, v)    ((void) CMM_STORE_SHARED(*(addr), (v)))
  27 #endif
  28
  29 #ifndef uatomic_read
  30 #define uatomic_read(addr)      CMM_LOAD_SHARED(*(addr))
  31 #endif
  32
  33 #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR
  34 #ifdef ILLEGAL_INSTR
  35 static inline __attribute__((always_inline))
  36 void _uatomic_link_error(void)
  37 {
  38         /*
  39          * generate an illegal instruction. Cannot catch this with
  40          * linker tricks when optimizations are disabled.
  41          */
  42         __asm__ __volatile__(ILLEGAL_INSTR);
  43 }
  44 #else
  45 static inline __attribute__((always_inline, __noreturn__))
  46 void _uatomic_link_error(void)
  47 {
  48         __builtin_trap();
  49 }
  50 #endif
  51
  52 #else /* #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */
  53 extern void _uatomic_link_error(void);
  54 #endif /* #else #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */
  55
  56 /* cmpxchg */
  57
  58 #ifndef uatomic_cmpxchg
  59 static inline __attribute__((always_inline))
  60 unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
  61                               unsigned long _new, int len)
  62 {
  63         switch (len) {
  64 #ifdef UATOMIC_HAS_ATOMIC_BYTE
  65         case 1:
  66                 return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
  67                                 _new);
  68 #endif
  69 #ifdef UATOMIC_HAS_ATOMIC_SHORT
  70         case 2:
  71                 return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
  72                                 _new);
  73 #endif
  74         case 4:
  75                 return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
  76                                 _new);
  77 #if (CAA_BITS_PER_LONG == 64)
  78         case 8:
  79                 return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
  80                                 _new);
  81 #endif
  82         }
  83         _uatomic_link_error();
  84         return 0;
  85 }
  86
  87
  88 #define uatomic_cmpxchg(addr, old, _new)                                      \
  89         ((__typeof__(*(addr))) _uatomic_cmpxchg((addr),                       \
  90                                                 caa_cast_long_keep_sign(old), \
  91                                                 caa_cast_long_keep_sign(_new),\
  92                                                 sizeof(*(addr))))
  93
  94
  95 /* uatomic_and */
  96
  97 #ifndef uatomic_and
  98 static inline __attribute__((always_inline))
  99 void _uatomic_and(void *addr, unsigned long val,
 100                   int len)
 101 {
 102         switch (len) {
 103 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 104         case 1:
 105                 __sync_and_and_fetch_1((uint8_t *) addr, val);
 106                 return;
 107 #endif
 108 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 109         case 2:
 110                 __sync_and_and_fetch_2((uint16_t *) addr, val);
 111                 return;
 112 #endif
 113         case 4:
 114                 __sync_and_and_fetch_4((uint32_t *) addr, val);
 115                 return;
 116 #if (CAA_BITS_PER_LONG == 64)
 117         case 8:
 118                 __sync_and_and_fetch_8((uint64_t *) addr, val);
 119                 return;
 120 #endif
 121         }
 122         _uatomic_link_error();
 123 }
 124
 125 #define uatomic_and(addr, v)                    \
 126         (_uatomic_and((addr),                   \
 127                 caa_cast_long_keep_sign(v),     \
 128                 sizeof(*(addr))))
 129 #define cmm_smp_mb__before_uatomic_and()        cmm_barrier()
 130 #define cmm_smp_mb__after_uatomic_and()         cmm_barrier()
 131
 132 #endif
 133
 134 /* uatomic_or */
 135
 136 #ifndef uatomic_or
 137 static inline __attribute__((always_inline))
 138 void _uatomic_or(void *addr, unsigned long val,
 139                  int len)
 140 {
 141         switch (len) {
 142 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 143         case 1:
 144                 __sync_or_and_fetch_1((uint8_t *) addr, val);
 145                 return;
 146 #endif
 147 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 148         case 2:
 149                 __sync_or_and_fetch_2((uint16_t *) addr, val);
 150                 return;
 151 #endif
 152         case 4:
 153                 __sync_or_and_fetch_4((uint32_t *) addr, val);
 154                 return;
 155 #if (CAA_BITS_PER_LONG == 64)
 156         case 8:
 157                 __sync_or_and_fetch_8((uint64_t *) addr, val);
 158                 return;
 159 #endif
 160         }
 161         _uatomic_link_error();
 162         return;
 163 }
 164
 165 #define uatomic_or(addr, v)                     \
 166         (_uatomic_or((addr),                    \
 167                 caa_cast_long_keep_sign(v),     \
 168                 sizeof(*(addr))))
 169 #define cmm_smp_mb__before_uatomic_or()         cmm_barrier()
 170 #define cmm_smp_mb__after_uatomic_or()          cmm_barrier()
 171
 172 #endif
 173
 174
 175 /* uatomic_add_return */
 176
 177 #ifndef uatomic_add_return
 178 static inline __attribute__((always_inline))
 179 unsigned long _uatomic_add_return(void *addr, unsigned long val,
 180                                  int len)
 181 {
 182         switch (len) {
 183 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 184         case 1:
 185                 return __sync_add_and_fetch_1((uint8_t *) addr, val);
 186 #endif
 187 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 188         case 2:
 189                 return __sync_add_and_fetch_2((uint16_t *) addr, val);
 190 #endif
 191         case 4:
 192                 return __sync_add_and_fetch_4((uint32_t *) addr, val);
 193 #if (CAA_BITS_PER_LONG == 64)
 194         case 8:
 195                 return __sync_add_and_fetch_8((uint64_t *) addr, val);
 196 #endif
 197         }
 198         _uatomic_link_error();
 199         return 0;
 200 }
 201
 202
 203 #define uatomic_add_return(addr, v)                                         \
 204         ((__typeof__(*(addr))) _uatomic_add_return((addr),                  \
 205                                                 caa_cast_long_keep_sign(v), \
 206                                                 sizeof(*(addr))))
 207 #endif /* #ifndef uatomic_add_return */
 208
 209 #ifndef uatomic_xchg
 210 /* xchg */
 211
 212 static inline __attribute__((always_inline))
 213 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
 214 {
 215         switch (len) {
 216 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 217         case 1:
 218         {
 219                 uint8_t old;
 220
 221                 do {
 222                         old = uatomic_read((uint8_t *) addr);
 223                 } while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
 224                                 old, val));
 225
 226                 return old;
 227         }
 228 #endif
 229 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 230         case 2:
 231         {
 232                 uint16_t old;
 233
 234                 do {
 235                         old = uatomic_read((uint16_t *) addr);
 236                 } while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
 237                                 old, val));
 238
 239                 return old;
 240         }
 241 #endif
 242         case 4:
 243         {
 244                 uint32_t old;
 245
 246                 do {
 247                         old = uatomic_read((uint32_t *) addr);
 248                 } while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
 249                                 old, val));
 250
 251                 return old;
 252         }
 253 #if (CAA_BITS_PER_LONG == 64)
 254         case 8:
 255         {
 256                 uint64_t old;
 257
 258                 do {
 259                         old = uatomic_read((uint64_t *) addr);
 260                 } while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
 261                                 old, val));
 262
 263                 return old;
 264         }
 265 #endif
 266         }
 267         _uatomic_link_error();
 268         return 0;
 269 }
 270
 271 #define uatomic_xchg(addr, v)                                               \
 272         ((__typeof__(*(addr))) _uatomic_exchange((addr),                    \
 273                                                 caa_cast_long_keep_sign(v), \
 274                                                 sizeof(*(addr))))
 275 #endif /* #ifndef uatomic_xchg */
 276
 277 #else /* #ifndef uatomic_cmpxchg */
 278
 279 #ifndef uatomic_and
 280 /* uatomic_and */
 281
 282 static inline __attribute__((always_inline))
 283 void _uatomic_and(void *addr, unsigned long val, int len)
 284 {
 285         switch (len) {
 286 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 287         case 1:
 288         {
 289                 uint8_t old, oldt;
 290
 291                 oldt = uatomic_read((uint8_t *) addr);
 292                 do {
 293                         old = oldt;
 294                         oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
 295                 } while (oldt != old);
 296
 297                 return;
 298         }
 299 #endif
 300 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 301         case 2:
 302         {
 303                 uint16_t old, oldt;
 304
 305                 oldt = uatomic_read((uint16_t *) addr);
 306                 do {
 307                         old = oldt;
 308                         oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
 309                 } while (oldt != old);
 310         }
 311 #endif
 312         case 4:
 313         {
 314                 uint32_t old, oldt;
 315
 316                 oldt = uatomic_read((uint32_t *) addr);
 317                 do {
 318                         old = oldt;
 319                         oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
 320                 } while (oldt != old);
 321
 322                 return;
 323         }
 324 #if (CAA_BITS_PER_LONG == 64)
 325         case 8:
 326         {
 327                 uint64_t old, oldt;
 328
 329                 oldt = uatomic_read((uint64_t *) addr);
 330                 do {
 331                         old = oldt;
 332                         oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
 333                 } while (oldt != old);
 334
 335                 return;
 336         }
 337 #endif
 338         }
 339         _uatomic_link_error();
 340 }
 341
 342 #define uatomic_and(addr, v)                    \
 343         (_uatomic_and((addr),                   \
 344                 caa_cast_long_keep_sign(v),     \
 345                 sizeof(*(addr))))
 346 #define cmm_smp_mb__before_uatomic_and()        cmm_barrier()
 347 #define cmm_smp_mb__after_uatomic_and()         cmm_barrier()
 348
 349 #endif /* #ifndef uatomic_and */
 350
 351 #ifndef uatomic_or
 352 /* uatomic_or */
 353
 354 static inline __attribute__((always_inline))
 355 void _uatomic_or(void *addr, unsigned long val, int len)
 356 {
 357         switch (len) {
 358 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 359         case 1:
 360         {
 361                 uint8_t old, oldt;
 362
 363                 oldt = uatomic_read((uint8_t *) addr);
 364                 do {
 365                         old = oldt;
 366                         oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
 367                 } while (oldt != old);
 368
 369                 return;
 370         }
 371 #endif
 372 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 373         case 2:
 374         {
 375                 uint16_t old, oldt;
 376
 377                 oldt = uatomic_read((uint16_t *) addr);
 378                 do {
 379                         old = oldt;
 380                         oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
 381                 } while (oldt != old);
 382
 383                 return;
 384         }
 385 #endif
 386         case 4:
 387         {
 388                 uint32_t old, oldt;
 389
 390                 oldt = uatomic_read((uint32_t *) addr);
 391                 do {
 392                         old = oldt;
 393                         oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
 394                 } while (oldt != old);
 395
 396                 return;
 397         }
 398 #if (CAA_BITS_PER_LONG == 64)
 399         case 8:
 400         {
 401                 uint64_t old, oldt;
 402
 403                 oldt = uatomic_read((uint64_t *) addr);
 404                 do {
 405                         old = oldt;
 406                         oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
 407                 } while (oldt != old);
 408
 409                 return;
 410         }
 411 #endif
 412         }
 413         _uatomic_link_error();
 414 }
 415
 416 #define uatomic_or(addr, v)                     \
 417         (_uatomic_or((addr),                    \
 418                 caa_cast_long_keep_sign(v),     \
 419                 sizeof(*(addr))))
 420 #define cmm_smp_mb__before_uatomic_or()         cmm_barrier()
 421 #define cmm_smp_mb__after_uatomic_or()          cmm_barrier()
 422
 423 #endif /* #ifndef uatomic_or */
 424
 425 #ifndef uatomic_add_return
 426 /* uatomic_add_return */
 427
 428 static inline __attribute__((always_inline))
 429 unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
 430 {
 431         switch (len) {
 432 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 433         case 1:
 434         {
 435                 uint8_t old, oldt;
 436
 437                 oldt = uatomic_read((uint8_t *) addr);
 438                 do {
 439                         old = oldt;
 440                         oldt = uatomic_cmpxchg((uint8_t *) addr,
 441                                                old, old + val);
 442                 } while (oldt != old);
 443
 444                 return old + val;
 445         }
 446 #endif
 447 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 448         case 2:
 449         {
 450                 uint16_t old, oldt;
 451
 452                 oldt = uatomic_read((uint16_t *) addr);
 453                 do {
 454                         old = oldt;
 455                         oldt = uatomic_cmpxchg((uint16_t *) addr,
 456                                                old, old + val);
 457                 } while (oldt != old);
 458
 459                 return old + val;
 460         }
 461 #endif
 462         case 4:
 463         {
 464                 uint32_t old, oldt;
 465
 466                 oldt = uatomic_read((uint32_t *) addr);
 467                 do {
 468                         old = oldt;
 469                         oldt = uatomic_cmpxchg((uint32_t *) addr,
 470                                                old, old + val);
 471                 } while (oldt != old);
 472
 473                 return old + val;
 474         }
 475 #if (CAA_BITS_PER_LONG == 64)
 476         case 8:
 477         {
 478                 uint64_t old, oldt;
 479
 480                 oldt = uatomic_read((uint64_t *) addr);
 481                 do {
 482                         old = oldt;
 483                         oldt = uatomic_cmpxchg((uint64_t *) addr,
 484                                                old, old + val);
 485                 } while (oldt != old);
 486
 487                 return old + val;
 488         }
 489 #endif
 490         }
 491         _uatomic_link_error();
 492         return 0;
 493 }
 494
 495 #define uatomic_add_return(addr, v)                                         \
 496         ((__typeof__(*(addr))) _uatomic_add_return((addr),                  \
 497                                                 caa_cast_long_keep_sign(v), \
 498                                                 sizeof(*(addr))))
 499 #endif /* #ifndef uatomic_add_return */
 500
 501 #ifndef uatomic_xchg
 502 /* xchg */
 503
 504 static inline __attribute__((always_inline))
 505 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
 506 {
 507         switch (len) {
 508 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 509         case 1:
 510         {
 511                 uint8_t old, oldt;
 512
 513                 oldt = uatomic_read((uint8_t *) addr);
 514                 do {
 515                         old = oldt;
 516                         oldt = uatomic_cmpxchg((uint8_t *) addr,
 517                                                old, val);
 518                 } while (oldt != old);
 519
 520                 return old;
 521         }
 522 #endif
 523 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 524         case 2:
 525         {
 526                 uint16_t old, oldt;
 527
 528                 oldt = uatomic_read((uint16_t *) addr);
 529                 do {
 530                         old = oldt;
 531                         oldt = uatomic_cmpxchg((uint16_t *) addr,
 532                                                old, val);
 533                 } while (oldt != old);
 534
 535                 return old;
 536         }
 537 #endif
 538         case 4:
 539         {
 540                 uint32_t old, oldt;
 541
 542                 oldt = uatomic_read((uint32_t *) addr);
 543                 do {
 544                         old = oldt;
 545                         oldt = uatomic_cmpxchg((uint32_t *) addr,
 546                                                old, val);
 547                 } while (oldt != old);
 548
 549                 return old;
 550         }
 551 #if (CAA_BITS_PER_LONG == 64)
 552         case 8:
 553         {
 554                 uint64_t old, oldt;
 555
 556                 oldt = uatomic_read((uint64_t *) addr);
 557                 do {
 558                         old = oldt;
 559                         oldt = uatomic_cmpxchg((uint64_t *) addr,
 560                                                old, val);
 561                 } while (oldt != old);
 562
 563                 return old;
 564         }
 565 #endif
 566         }
 567         _uatomic_link_error();
 568         return 0;
 569 }
 570
 571 #define uatomic_xchg(addr, v)                                               \
 572         ((__typeof__(*(addr))) _uatomic_exchange((addr),                    \
 573                                                 caa_cast_long_keep_sign(v), \
 574                                                 sizeof(*(addr))))
 575 #endif /* #ifndef uatomic_xchg */
 576
 577 #endif /* #else #ifndef uatomic_cmpxchg */
 578
 579 /* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
 580
 581 #ifndef uatomic_add
 582 #define uatomic_add(addr, v)            (void)uatomic_add_return((addr), (v))
 583 #define cmm_smp_mb__before_uatomic_add()        cmm_barrier()
 584 #define cmm_smp_mb__after_uatomic_add()         cmm_barrier()
 585 #endif
 586
 587 #define uatomic_sub_return(addr, v)     \
 588         uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
 589 #define uatomic_sub(addr, v)            \
 590         uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
 591 #define cmm_smp_mb__before_uatomic_sub()        cmm_smp_mb__before_uatomic_add()
 592 #define cmm_smp_mb__after_uatomic_sub()         cmm_smp_mb__after_uatomic_add()
 593
 594 #ifndef uatomic_inc
 595 #define uatomic_inc(addr)               uatomic_add((addr), 1)
 596 #define cmm_smp_mb__before_uatomic_inc()        cmm_smp_mb__before_uatomic_add()
 597 #define cmm_smp_mb__after_uatomic_inc()         cmm_smp_mb__after_uatomic_add()
 598 #endif
 599
 600 #ifndef uatomic_dec
 601 #define uatomic_dec(addr)               uatomic_add((addr), -1)
 602 #define cmm_smp_mb__before_uatomic_dec()        cmm_smp_mb__before_uatomic_add()
 603 #define cmm_smp_mb__after_uatomic_dec()         cmm_smp_mb__after_uatomic_add()
 604 #endif
 605
 606 #ifdef __cplusplus
 607 }
 608 #endif
 609
 610 #endif /* _URCU_UATOMIC_GENERIC_H */