src/urcu.c

   1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
   3 //
   4 // SPDX-License-Identifier: LGPL-2.1-or-later
   5
   6 /*
   7  * Userspace RCU library
   8  *
   9  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  10  */
  11
  12 #define URCU_NO_COMPAT_IDENTIFIERS
  13 #define _BSD_SOURCE
  14 #define _LGPL_SOURCE
  15 #define _DEFAULT_SOURCE
  16 #include <stdio.h>
  17 #include <pthread.h>
  18 #include <signal.h>
  19 #include <stdlib.h>
  20 #include <stdint.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdbool.h>
  24 #include <poll.h>
  25
  26 #include <urcu/config.h>
  27 #include <urcu/annotate.h>
  28 #include <urcu/assert.h>
  29 #include <urcu/arch.h>
  30 #include <urcu/wfcqueue.h>
  31 #include <urcu/map/urcu.h>
  32 #include <urcu/static/urcu.h>
  33 #include <urcu/pointer.h>
  34 #include <urcu/tls-compat.h>
  35
  36 #include "urcu-die.h"
  37 #include "urcu-wait.h"
  38 #include "urcu-utils.h"
  39
  40 #define URCU_API_MAP
  41 /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
  42 #undef _LGPL_SOURCE
  43 #include <urcu/urcu.h>
  44 #define _LGPL_SOURCE
  45
  46 /*
  47  * If a reader is really non-cooperative and refuses to commit its
  48  * rcu_active_readers count to memory (there is no barrier in the reader
  49  * per-se), kick it after 10 loops waiting for it.
  50  */
  51 #define KICK_READER_LOOPS       10
  52
  53 /*
  54  * Active attempts to check for reader Q.S. before calling futex().
  55  */
  56 #define RCU_QS_ACTIVE_ATTEMPTS 100
  57
  58 /* If the headers do not support membarrier system call, fall back on RCU_MB */
  59 #ifdef __NR_membarrier
  60 # define membarrier(...)                syscall(__NR_membarrier, __VA_ARGS__)
  61 #else
  62 # define membarrier(...)                -ENOSYS
  63 #endif
  64
  65 enum membarrier_cmd {
  66         MEMBARRIER_CMD_QUERY                            = 0,
  67         MEMBARRIER_CMD_SHARED                           = (1 << 0),
  68         /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
  69         /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
  70         MEMBARRIER_CMD_PRIVATE_EXPEDITED                = (1 << 3),
  71         MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED       = (1 << 4),
  72 };
  73
  74 #ifdef RCU_MEMBARRIER
  75 static int init_done;
  76 static int urcu_memb_has_sys_membarrier_private_expedited;
  77
  78 #ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER
  79 /*
  80  * Explicitly initialize to zero because we can't alias a non-static
  81  * uninitialized variable.
  82  */
  83 int urcu_memb_has_sys_membarrier = 0;
  84 #endif
  85
  86 void __attribute__((constructor)) rcu_init(void);
  87 #endif
  88
  89 #ifdef RCU_MB
  90 void rcu_init(void)
  91 {
  92 }
  93 #endif
  94
  95 #ifdef RCU_SIGNAL
  96 static int init_done;
  97
  98 void __attribute__((constructor)) rcu_init(void);
  99
 100 static DEFINE_URCU_TLS(int, rcu_signal_was_blocked);
 101 #endif
 102
 103 void __attribute__((destructor)) rcu_exit(void);
 104 static void urcu_call_rcu_exit(void);
 105
 106 /*
 107  * rcu_gp_lock ensures mutual exclusion between threads calling
 108  * synchronize_rcu().
 109  */
 110 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
 111 /*
 112  * rcu_registry_lock ensures mutual exclusion between threads
 113  * registering and unregistering themselves to/from the registry, and
 114  * with threads reading that registry from synchronize_rcu(). However,
 115  * this lock is not held all the way through the completion of awaiting
 116  * for the grace period. It is sporadically released between iterations
 117  * on the registry.
 118  * rcu_registry_lock may nest inside rcu_gp_lock.
 119  */
 120 static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
 121 struct urcu_gp rcu_gp = { .ctr = URCU_GP_COUNT };
 122
 123 /*
 124  * Written to only by each individual reader. Read by both the reader and the
 125  * writers.
 126  */
 127 DEFINE_URCU_TLS(struct urcu_reader, rcu_reader);
 128
 129 static CDS_LIST_HEAD(registry);
 130
 131 /*
 132  * Queue keeping threads awaiting to wait for a grace period. Contains
 133  * struct gp_waiters_thread objects.
 134  */
 135 static DEFINE_URCU_WAIT_QUEUE(gp_waiters);
 136
 137 static void mutex_lock(pthread_mutex_t *mutex)
 138 {
 139         int ret;
 140
 141 #ifndef DISTRUST_SIGNALS_EXTREME
 142         ret = pthread_mutex_lock(mutex);
 143         if (ret)
 144                 urcu_die(ret);
 145 #else /* #ifndef DISTRUST_SIGNALS_EXTREME */
 146         while ((ret = pthread_mutex_trylock(mutex)) != 0) {
 147                 if (ret != EBUSY && ret != EINTR)
 148                         urcu_die(ret);
 149                 if (CMM_LOAD_SHARED(URCU_TLS(rcu_reader).need_mb)) {
 150                         cmm_smp_mb();
 151                         _CMM_STORE_SHARED(URCU_TLS(rcu_reader).need_mb, 0);
 152                         cmm_smp_mb();
 153                 }
 154                 (void) poll(NULL, 0, 10);
 155         }
 156 #endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
 157 }
 158
 159 static void mutex_unlock(pthread_mutex_t *mutex)
 160 {
 161         int ret;
 162
 163         ret = pthread_mutex_unlock(mutex);
 164         if (ret)
 165                 urcu_die(ret);
 166 }
 167
 168 #ifdef RCU_MEMBARRIER
 169 static void smp_mb_master(void)
 170 {
 171         if (caa_likely(urcu_memb_has_sys_membarrier)) {
 172                 if (membarrier(urcu_memb_has_sys_membarrier_private_expedited ?
 173                                 MEMBARRIER_CMD_PRIVATE_EXPEDITED :
 174                                 MEMBARRIER_CMD_SHARED, 0))
 175                         urcu_die(errno);
 176         } else {
 177                 cmm_smp_mb();
 178         }
 179 }
 180 #endif
 181
 182 #ifdef RCU_MB
 183 static void smp_mb_master(void)
 184 {
 185         cmm_smp_mb();
 186 }
 187 #endif
 188
 189 #ifdef RCU_SIGNAL
 190 static void force_mb_all_readers(void)
 191 {
 192         struct urcu_reader *index;
 193
 194         /*
 195          * Ask for each threads to execute a cmm_smp_mb() so we can consider the
 196          * compiler barriers around rcu read lock as real memory barriers.
 197          */
 198         if (cds_list_empty(&registry))
 199                 return;
 200         /*
 201          * pthread_kill has a cmm_smp_mb(). But beware, we assume it performs
 202          * a cache flush on architectures with non-coherent cache. Let's play
 203          * safe and don't assume anything : we use cmm_smp_mc() to make sure the
 204          * cache flush is enforced.
 205          */
 206         cds_list_for_each_entry(index, &registry, node) {
 207                 CMM_STORE_SHARED(index->need_mb, 1);
 208                 pthread_kill(index->tid, SIGRCU);
 209         }
 210         /*
 211          * Wait for sighandler (and thus mb()) to execute on every thread.
 212          *
 213          * Note that the pthread_kill() will never be executed on systems
 214          * that correctly deliver signals in a timely manner.  However, it
 215          * is not uncommon for kernels to have bugs that can result in
 216          * lost or unduly delayed signals.
 217          *
 218          * If you are seeing the below pthread_kill() executing much at
 219          * all, we suggest testing the underlying kernel and filing the
 220          * relevant bug report.  For Linux kernels, we recommend getting
 221          * the Linux Test Project (LTP).
 222          */
 223         cds_list_for_each_entry(index, &registry, node) {
 224                 while (CMM_LOAD_SHARED(index->need_mb)) {
 225                         pthread_kill(index->tid, SIGRCU);
 226                         (void) poll(NULL, 0, 1);
 227                 }
 228         }
 229         cmm_smp_mb();   /* read ->need_mb before ending the barrier */
 230 }
 231
 232 static void smp_mb_master(void)
 233 {
 234         force_mb_all_readers();
 235 }
 236 #endif /* #ifdef RCU_SIGNAL */
 237
 238 /*
 239  * synchronize_rcu() waiting. Single thread.
 240  * Always called with rcu_registry lock held. Releases this lock and
 241  * grabs it again. Holds the lock when it returns.
 242  */
 243 static void wait_gp(void)
 244 {
 245         /*
 246          * Read reader_gp before read futex. smp_mb_master() needs to
 247          * be called with the rcu registry lock held in RCU_SIGNAL
 248          * flavor.
 249          */
 250         smp_mb_master();
 251         /* Temporarily unlock the registry lock. */
 252         mutex_unlock(&rcu_registry_lock);
 253         while (uatomic_read(&rcu_gp.futex) == -1) {
 254                 if (!futex_async(&rcu_gp.futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
 255                         /*
 256                          * Prior queued wakeups queued by unrelated code
 257                          * using the same address can cause futex wait to
 258                          * return 0 even through the futex value is still
 259                          * -1 (spurious wakeups). Check the value again
 260                          * in user-space to validate whether it really
 261                          * differs from -1.
 262                          */
 263                         continue;
 264                 }
 265                 switch (errno) {
 266                 case EAGAIN:
 267                         /* Value already changed. */
 268                         goto end;
 269                 case EINTR:
 270                         /* Retry if interrupted by signal. */
 271                         break;  /* Get out of switch. Check again. */
 272                 default:
 273                         /* Unexpected error. */
 274                         urcu_die(errno);
 275                 }
 276         }
 277 end:
 278         /*
 279          * Re-lock the registry lock before the next loop.
 280          */
 281         mutex_lock(&rcu_registry_lock);
 282 }
 283
 284 /*
 285  * Always called with rcu_registry lock held. Releases this lock between
 286  * iterations and grabs it again. Holds the lock when it returns.
 287  */
 288 static void wait_for_readers(struct cds_list_head *input_readers,
 289                         struct cds_list_head *cur_snap_readers,
 290                         struct cds_list_head *qsreaders,
 291                         cmm_annotate_t *group)
 292 {
 293         unsigned int wait_loops = 0;
 294         struct urcu_reader *index, *tmp;
 295 #ifdef HAS_INCOHERENT_CACHES
 296         unsigned int wait_gp_loops = 0;
 297 #endif /* HAS_INCOHERENT_CACHES */
 298
 299         /*
 300          * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 301          * indicate quiescence (not nested), or observe the current
 302          * rcu_gp.ctr value.
 303          */
 304         for (;;) {
 305                 if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
 306                         wait_loops++;
 307                 if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 308                         uatomic_dec(&rcu_gp.futex);
 309                         /* Write futex before read reader_gp */
 310                         smp_mb_master();
 311                 }
 312
 313                 cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
 314                         switch (urcu_common_reader_state(&rcu_gp, &index->ctr, group)) {
 315                         case URCU_READER_ACTIVE_CURRENT:
 316                                 if (cur_snap_readers) {
 317                                         cds_list_move(&index->node,
 318                                                 cur_snap_readers);
 319                                         break;
 320                                 }
 321                                 /* Fall-through */
 322                         case URCU_READER_INACTIVE:
 323                                 cds_list_move(&index->node, qsreaders);
 324                                 break;
 325                         case URCU_READER_ACTIVE_OLD:
 326                                 /*
 327                                  * Old snapshot. Leaving node in
 328                                  * input_readers will make us busy-loop
 329                                  * until the snapshot becomes current or
 330                                  * the reader becomes inactive.
 331                                  */
 332                                 break;
 333                         }
 334                 }
 335
 336 #ifndef HAS_INCOHERENT_CACHES
 337                 if (cds_list_empty(input_readers)) {
 338                         if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 339                                 /* Read reader_gp before write futex */
 340                                 smp_mb_master();
 341                                 uatomic_set(&rcu_gp.futex, 0);
 342                         }
 343                         break;
 344                 } else {
 345                         if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 346                                 /* wait_gp unlocks/locks registry lock. */
 347                                 wait_gp();
 348                         } else {
 349                                 /* Temporarily unlock the registry lock. */
 350                                 mutex_unlock(&rcu_registry_lock);
 351                                 caa_cpu_relax();
 352                                 /*
 353                                  * Re-lock the registry lock before the
 354                                  * next loop.
 355                                  */
 356                                 mutex_lock(&rcu_registry_lock);
 357                         }
 358                 }
 359 #else /* #ifndef HAS_INCOHERENT_CACHES */
 360                 /*
 361                  * BUSY-LOOP. Force the reader thread to commit its
 362                  * URCU_TLS(rcu_reader).ctr update to memory if we wait
 363                  * for too long.
 364                  */
 365                 if (cds_list_empty(input_readers)) {
 366                         if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 367                                 /* Read reader_gp before write futex */
 368                                 smp_mb_master();
 369                                 uatomic_set(&rcu_gp.futex, 0);
 370                         }
 371                         break;
 372                 } else {
 373                         if (wait_gp_loops == KICK_READER_LOOPS) {
 374                                 smp_mb_master();
 375                                 wait_gp_loops = 0;
 376                         }
 377                         if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 378                                 /* wait_gp unlocks/locks registry lock. */
 379                                 wait_gp();
 380                                 wait_gp_loops++;
 381                         } else {
 382                                 /* Temporarily unlock the registry lock. */
 383                                 mutex_unlock(&rcu_registry_lock);
 384                                 caa_cpu_relax();
 385                                 /*
 386                                  * Re-lock the registry lock before the
 387                                  * next loop.
 388                                  */
 389                                 mutex_lock(&rcu_registry_lock);
 390                         }
 391                 }
 392 #endif /* #else #ifndef HAS_INCOHERENT_CACHES */
 393         }
 394 }
 395
 396 void synchronize_rcu(void)
 397 {
 398         cmm_annotate_define(acquire_group);
 399         cmm_annotate_define(release_group);
 400         CDS_LIST_HEAD(cur_snap_readers);
 401         CDS_LIST_HEAD(qsreaders);
 402         DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
 403         struct urcu_waiters waiters;
 404
 405         /*
 406          * Add ourself to gp_waiters queue of threads awaiting to wait
 407          * for a grace period. Proceed to perform the grace period only
 408          * if we are the first thread added into the queue.
 409          * The implicit memory barrier before urcu_wait_add()
 410          * orders prior memory accesses of threads put into the wait
 411          * queue before their insertion into the wait queue.
 412          */
 413         if (urcu_wait_add(&gp_waiters, &wait) != 0) {
 414                 /*
 415                  * Not first in queue: will be awakened by another thread.
 416                  * Implies a memory barrier after grace period.
 417                  */
 418                 urcu_adaptative_busy_wait(&wait);
 419                 return;
 420         }
 421         /* We won't need to wake ourself up */
 422         urcu_wait_set_state(&wait, URCU_WAIT_RUNNING);
 423
 424         mutex_lock(&rcu_gp_lock);
 425
 426         /*
 427          * Move all waiters into our local queue.
 428          */
 429         urcu_move_waiters(&waiters, &gp_waiters);
 430
 431         mutex_lock(&rcu_registry_lock);
 432
 433         if (cds_list_empty(&registry))
 434                 goto out;
 435
 436         /*
 437          * All threads should read qparity before accessing data structure
 438          * where new ptr points to. Must be done within rcu_registry_lock
 439          * because it iterates on reader threads.
 440          */
 441         /* Write new ptr before changing the qparity */
 442         smp_mb_master();
 443         cmm_annotate_group_mb_release(&release_group);
 444
 445         /*
 446          * Wait for readers to observe original parity or be quiescent.
 447          * wait_for_readers() can release and grab again rcu_registry_lock
 448          * internally.
 449          */
 450         wait_for_readers(&registry, &cur_snap_readers, &qsreaders, &acquire_group);
 451
 452         /*
 453          * Must finish waiting for quiescent state for original parity before
 454          * committing next rcu_gp.ctr update to memory. Failure to do so could
 455          * result in the writer waiting forever while new readers are always
 456          * accessing data (no progress).  Enforce compiler-order of load
 457          * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
 458          */
 459         cmm_barrier();
 460
 461         /*
 462          * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
 463          * model easier to understand. It does not have a big performance impact
 464          * anyway, given this is the write-side.
 465          */
 466         cmm_smp_mb();
 467
 468         /* Switch parity: 0 -> 1, 1 -> 0 */
 469         cmm_annotate_group_mem_release(&release_group, &rcu_gp.ctr);
 470         uatomic_store(&rcu_gp.ctr, rcu_gp.ctr ^ URCU_GP_CTR_PHASE, CMM_RELAXED);
 471
 472         /*
 473          * Must commit rcu_gp.ctr update to memory before waiting for quiescent
 474          * state. Failure to do so could result in the writer waiting forever
 475          * while new readers are always accessing data (no progress). Enforce
 476          * compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
 477          */
 478         cmm_barrier();
 479
 480         /*
 481          *
 482          * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
 483          * model easier to understand. It does not have a big performance impact
 484          * anyway, given this is the write-side.
 485          */
 486         cmm_smp_mb();
 487
 488         /*
 489          * Wait for readers to observe new parity or be quiescent.
 490          * wait_for_readers() can release and grab again rcu_registry_lock
 491          * internally.
 492          */
 493         wait_for_readers(&cur_snap_readers, NULL, &qsreaders, &acquire_group);
 494
 495         /*
 496          * Put quiescent reader list back into registry.
 497          */
 498         cds_list_splice(&qsreaders, &registry);
 499
 500         /*
 501          * Finish waiting for reader threads before letting the old ptr
 502          * being freed. Must be done within rcu_registry_lock because it
 503          * iterates on reader threads.
 504          */
 505         smp_mb_master();
 506         cmm_annotate_group_mb_acquire(&acquire_group);
 507 out:
 508         mutex_unlock(&rcu_registry_lock);
 509         mutex_unlock(&rcu_gp_lock);
 510
 511         /*
 512          * Wakeup waiters only after we have completed the grace period
 513          * and have ensured the memory barriers at the end of the grace
 514          * period have been issued.
 515          */
 516         urcu_wake_all_waiters(&waiters);
 517 }
 518
 519 /*
 520  * library wrappers to be used by non-LGPL compatible source code.
 521  */
 522
 523 void rcu_read_lock(void)
 524 {
 525         _rcu_read_lock();
 526 }
 527
 528 void rcu_read_unlock(void)
 529 {
 530         _rcu_read_unlock();
 531 }
 532
 533 int rcu_read_ongoing(void)
 534 {
 535         return _rcu_read_ongoing();
 536 }
 537
 538 #ifdef RCU_SIGNAL
 539 /*
 540  * Make sure the signal used by the urcu-signal flavor is unblocked
 541  * while the thread is registered.
 542  */
 543 static
 544 void urcu_signal_unblock(void)
 545 {
 546         sigset_t mask, oldmask;
 547         int ret;
 548
 549         ret = sigemptyset(&mask);
 550         urcu_posix_assert(!ret);
 551         ret = sigaddset(&mask, SIGRCU);
 552         urcu_posix_assert(!ret);
 553         ret = pthread_sigmask(SIG_UNBLOCK, &mask, &oldmask);
 554         urcu_posix_assert(!ret);
 555         URCU_TLS(rcu_signal_was_blocked) = sigismember(&oldmask, SIGRCU);
 556 }
 557
 558 static
 559 void urcu_signal_restore(void)
 560 {
 561         sigset_t mask;
 562         int ret;
 563
 564         if (!URCU_TLS(rcu_signal_was_blocked))
 565                 return;
 566         ret = sigemptyset(&mask);
 567         urcu_posix_assert(!ret);
 568         ret = sigaddset(&mask, SIGRCU);
 569         urcu_posix_assert(!ret);
 570         ret = pthread_sigmask(SIG_BLOCK, &mask, NULL);
 571         urcu_posix_assert(!ret);
 572 }
 573 #else
 574 static
 575 void urcu_signal_unblock(void) { }
 576 static
 577 void urcu_signal_restore(void) { }
 578 #endif
 579
 580 void rcu_register_thread(void)
 581 {
 582         urcu_signal_unblock();
 583
 584         URCU_TLS(rcu_reader).tid = pthread_self();
 585         urcu_posix_assert(URCU_TLS(rcu_reader).need_mb == 0);
 586         urcu_posix_assert(!(URCU_TLS(rcu_reader).ctr & URCU_GP_CTR_NEST_MASK));
 587
 588         mutex_lock(&rcu_registry_lock);
 589         urcu_posix_assert(!URCU_TLS(rcu_reader).registered);
 590         URCU_TLS(rcu_reader).registered = 1;
 591         rcu_init();     /* In case gcc does not support constructor attribute */
 592         cds_list_add(&URCU_TLS(rcu_reader).node, &registry);
 593         mutex_unlock(&rcu_registry_lock);
 594 }
 595
 596 void rcu_unregister_thread(void)
 597 {
 598         mutex_lock(&rcu_registry_lock);
 599         urcu_posix_assert(URCU_TLS(rcu_reader).registered);
 600         URCU_TLS(rcu_reader).registered = 0;
 601         cds_list_del(&URCU_TLS(rcu_reader).node);
 602         mutex_unlock(&rcu_registry_lock);
 603
 604         urcu_signal_restore();
 605 }
 606
 607 #ifdef RCU_MEMBARRIER
 608
 609 #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
 610 static
 611 void rcu_sys_membarrier_status(bool available)
 612 {
 613         if (!available)
 614                 abort();
 615 }
 616 #else
 617 static
 618 void rcu_sys_membarrier_status(bool available)
 619 {
 620         if (!available)
 621                 return;
 622         urcu_memb_has_sys_membarrier = 1;
 623 }
 624 #endif
 625
 626 static
 627 void rcu_sys_membarrier_init(void)
 628 {
 629         bool available = false;
 630         int mask;
 631
 632         mask = membarrier(MEMBARRIER_CMD_QUERY, 0);
 633         if (mask >= 0) {
 634                 if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
 635                         if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0))
 636                                 urcu_die(errno);
 637                         urcu_memb_has_sys_membarrier_private_expedited = 1;
 638                         available = true;
 639                 } else if (mask & MEMBARRIER_CMD_SHARED) {
 640                         available = true;
 641                 }
 642         }
 643         rcu_sys_membarrier_status(available);
 644 }
 645
 646 void rcu_init(void)
 647 {
 648         if (init_done)
 649                 return;
 650         init_done = 1;
 651         rcu_sys_membarrier_init();
 652 }
 653 #endif
 654
 655 #ifdef RCU_SIGNAL
 656 static void sigrcu_handler(int signo __attribute__((unused)),
 657                 siginfo_t *siginfo __attribute__((unused)),
 658                 void *context __attribute__((unused)))
 659 {
 660         /*
 661          * Executing this cmm_smp_mb() is the only purpose of this signal handler.
 662          * It punctually promotes cmm_barrier() into cmm_smp_mb() on every thread it is
 663          * executed on.
 664          */
 665         cmm_smp_mb();
 666         _CMM_STORE_SHARED(URCU_TLS(rcu_reader).need_mb, 0);
 667         cmm_smp_mb();
 668 }
 669
 670 /*
 671  * rcu_init constructor. Called when the library is linked, but also when
 672  * reader threads are calling rcu_register_thread().
 673  * Should only be called by a single thread at a given time. This is ensured by
 674  * holing the rcu_registry_lock from rcu_register_thread() or by running
 675  * at library load time, which should not be executed by multiple
 676  * threads nor concurrently with rcu_register_thread() anyway.
 677  */
 678 void rcu_init(void)
 679 {
 680         struct sigaction act;
 681         int ret;
 682
 683         if (init_done)
 684                 return;
 685         init_done = 1;
 686
 687         act.sa_sigaction = sigrcu_handler;
 688         act.sa_flags = SA_SIGINFO | SA_RESTART;
 689         sigemptyset(&act.sa_mask);
 690         ret = sigaction(SIGRCU, &act, NULL);
 691         if (ret)
 692                 urcu_die(errno);
 693 }
 694
 695 /*
 696  * Don't unregister the SIGRCU signal handler anymore, because
 697  * call_rcu threads could still be using it shortly before the
 698  * application exits.
 699  * Assertion disabled because call_rcu threads are now rcu
 700  * readers, and left running at exit.
 701  * urcu_posix_assert(cds_list_empty(&registry));
 702  */
 703
 704 #endif /* #ifdef RCU_SIGNAL */
 705
 706 void rcu_exit(void)
 707 {
 708         urcu_call_rcu_exit();
 709 }
 710
 711 DEFINE_RCU_FLAVOR(rcu_flavor);
 712
 713 #include "urcu-call-rcu-impl.h"
 714 #include "urcu-defer-impl.h"
 715 #include "urcu-poll-impl.h"