src/urcu.c

   1 /*
   2  * urcu.c
   3  *
   4  * Userspace RCU library
   5  *
   6  * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   7  * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
   8  *
   9  * This library is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public
  11  * License as published by the Free Software Foundation; either
  12  * version 2.1 of the License, or (at your option) any later version.
  13  *
  14  * This library is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with this library; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  22  *
  23  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  24  */
  25
  26 #define URCU_NO_COMPAT_IDENTIFIERS
  27 #define _BSD_SOURCE
  28 #define _LGPL_SOURCE
  29 #define _DEFAULT_SOURCE
  30 #include <stdio.h>
  31 #include <pthread.h>
  32 #include <signal.h>
  33 #include <assert.h>
  34 #include <stdlib.h>
  35 #include <stdint.h>
  36 #include <string.h>
  37 #include <errno.h>
  38 #include <stdbool.h>
  39 #include <poll.h>
  40
  41 #include <urcu/config.h>
  42 #include <urcu/arch.h>
  43 #include <urcu/wfcqueue.h>
  44 #include <urcu/map/urcu.h>
  45 #include <urcu/static/urcu.h>
  46 #include <urcu/pointer.h>
  47 #include <urcu/tls-compat.h>
  48
  49 #include "urcu-die.h"
  50 #include "urcu-wait.h"
  51 #include "urcu-utils.h"
  52
  53 #define URCU_API_MAP
  54 /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
  55 #undef _LGPL_SOURCE
  56 #include <urcu/urcu.h>
  57 #define _LGPL_SOURCE
  58
  59 /*
  60  * If a reader is really non-cooperative and refuses to commit its
  61  * rcu_active_readers count to memory (there is no barrier in the reader
  62  * per-se), kick it after 10 loops waiting for it.
  63  */
  64 #define KICK_READER_LOOPS       10
  65
  66 /*
  67  * Active attempts to check for reader Q.S. before calling futex().
  68  */
  69 #define RCU_QS_ACTIVE_ATTEMPTS 100
  70
  71 /* If the headers do not support membarrier system call, fall back on RCU_MB */
  72 #ifdef __NR_membarrier
  73 # define membarrier(...)                syscall(__NR_membarrier, __VA_ARGS__)
  74 #else
  75 # define membarrier(...)                -ENOSYS
  76 #endif
  77
  78 enum membarrier_cmd {
  79         MEMBARRIER_CMD_QUERY                            = 0,
  80         MEMBARRIER_CMD_SHARED                           = (1 << 0),
  81         /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
  82         /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
  83         MEMBARRIER_CMD_PRIVATE_EXPEDITED                = (1 << 3),
  84         MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED       = (1 << 4),
  85 };
  86
  87 #ifdef RCU_MEMBARRIER
  88 static int init_done;
  89 static int urcu_memb_has_sys_membarrier_private_expedited;
  90
  91 #ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER
  92 /*
  93  * Explicitly initialize to zero because we can't alias a non-static
  94  * uninitialized variable.
  95  */
  96 int urcu_memb_has_sys_membarrier = 0;
  97 URCU_ATTR_ALIAS("urcu_memb_has_sys_membarrier")
  98 extern int rcu_has_sys_membarrier_memb;
  99 #endif
 100
 101 void __attribute__((constructor)) rcu_init(void);
 102 #endif
 103
 104 #ifdef RCU_MB
 105 void rcu_init(void)
 106 {
 107 }
 108 URCU_ATTR_ALIAS(urcu_stringify(rcu_init))
 109 void alias_rcu_init(void);
 110 #endif
 111
 112 #ifdef RCU_SIGNAL
 113 static int init_done;
 114
 115 void __attribute__((constructor)) rcu_init(void);
 116 #endif
 117
 118 void __attribute__((destructor)) rcu_exit(void);
 119 static void urcu_call_rcu_exit(void);
 120
 121 /*
 122  * rcu_gp_lock ensures mutual exclusion between threads calling
 123  * synchronize_rcu().
 124  */
 125 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
 126 /*
 127  * rcu_registry_lock ensures mutual exclusion between threads
 128  * registering and unregistering themselves to/from the registry, and
 129  * with threads reading that registry from synchronize_rcu(). However,
 130  * this lock is not held all the way through the completion of awaiting
 131  * for the grace period. It is sporadically released between iterations
 132  * on the registry.
 133  * rcu_registry_lock may nest inside rcu_gp_lock.
 134  */
 135 static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
 136 struct urcu_gp rcu_gp = { .ctr = URCU_GP_COUNT };
 137 URCU_ATTR_ALIAS(urcu_stringify(rcu_gp))
 138 extern struct urcu_gp alias_rcu_gp;
 139
 140 /*
 141  * Written to only by each individual reader. Read by both the reader and the
 142  * writers.
 143  */
 144 DEFINE_URCU_TLS(struct urcu_reader, rcu_reader);
 145 DEFINE_URCU_TLS_ALIAS(struct urcu_reader, rcu_reader, alias_rcu_reader);
 146
 147 static CDS_LIST_HEAD(registry);
 148
 149 /*
 150  * Queue keeping threads awaiting to wait for a grace period. Contains
 151  * struct gp_waiters_thread objects.
 152  */
 153 static DEFINE_URCU_WAIT_QUEUE(gp_waiters);
 154
 155 static void mutex_lock(pthread_mutex_t *mutex)
 156 {
 157         int ret;
 158
 159 #ifndef DISTRUST_SIGNALS_EXTREME
 160         ret = pthread_mutex_lock(mutex);
 161         if (ret)
 162                 urcu_die(ret);
 163 #else /* #ifndef DISTRUST_SIGNALS_EXTREME */
 164         while ((ret = pthread_mutex_trylock(mutex)) != 0) {
 165                 if (ret != EBUSY && ret != EINTR)
 166                         urcu_die(ret);
 167                 if (CMM_LOAD_SHARED(URCU_TLS(rcu_reader).need_mb)) {
 168                         cmm_smp_mb();
 169                         _CMM_STORE_SHARED(URCU_TLS(rcu_reader).need_mb, 0);
 170                         cmm_smp_mb();
 171                 }
 172                 (void) poll(NULL, 0, 10);
 173         }
 174 #endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
 175 }
 176
 177 static void mutex_unlock(pthread_mutex_t *mutex)
 178 {
 179         int ret;
 180
 181         ret = pthread_mutex_unlock(mutex);
 182         if (ret)
 183                 urcu_die(ret);
 184 }
 185
 186 #ifdef RCU_MEMBARRIER
 187 static void smp_mb_master(void)
 188 {
 189         if (caa_likely(urcu_memb_has_sys_membarrier)) {
 190                 if (membarrier(urcu_memb_has_sys_membarrier_private_expedited ?
 191                                 MEMBARRIER_CMD_PRIVATE_EXPEDITED :
 192                                 MEMBARRIER_CMD_SHARED, 0))
 193                         urcu_die(errno);
 194         } else {
 195                 cmm_smp_mb();
 196         }
 197 }
 198 #endif
 199
 200 #ifdef RCU_MB
 201 static void smp_mb_master(void)
 202 {
 203         cmm_smp_mb();
 204 }
 205 #endif
 206
 207 #ifdef RCU_SIGNAL
 208 static void force_mb_all_readers(void)
 209 {
 210         struct urcu_reader *index;
 211
 212         /*
 213          * Ask for each threads to execute a cmm_smp_mb() so we can consider the
 214          * compiler barriers around rcu read lock as real memory barriers.
 215          */
 216         if (cds_list_empty(&registry))
 217                 return;
 218         /*
 219          * pthread_kill has a cmm_smp_mb(). But beware, we assume it performs
 220          * a cache flush on architectures with non-coherent cache. Let's play
 221          * safe and don't assume anything : we use cmm_smp_mc() to make sure the
 222          * cache flush is enforced.
 223          */
 224         cds_list_for_each_entry(index, &registry, node) {
 225                 CMM_STORE_SHARED(index->need_mb, 1);
 226                 pthread_kill(index->tid, SIGRCU);
 227         }
 228         /*
 229          * Wait for sighandler (and thus mb()) to execute on every thread.
 230          *
 231          * Note that the pthread_kill() will never be executed on systems
 232          * that correctly deliver signals in a timely manner.  However, it
 233          * is not uncommon for kernels to have bugs that can result in
 234          * lost or unduly delayed signals.
 235          *
 236          * If you are seeing the below pthread_kill() executing much at
 237          * all, we suggest testing the underlying kernel and filing the
 238          * relevant bug report.  For Linux kernels, we recommend getting
 239          * the Linux Test Project (LTP).
 240          */
 241         cds_list_for_each_entry(index, &registry, node) {
 242                 while (CMM_LOAD_SHARED(index->need_mb)) {
 243                         pthread_kill(index->tid, SIGRCU);
 244                         (void) poll(NULL, 0, 1);
 245                 }
 246         }
 247         cmm_smp_mb();   /* read ->need_mb before ending the barrier */
 248 }
 249
 250 static void smp_mb_master(void)
 251 {
 252         force_mb_all_readers();
 253 }
 254 #endif /* #ifdef RCU_SIGNAL */
 255
 256 /*
 257  * synchronize_rcu() waiting. Single thread.
 258  * Always called with rcu_registry lock held. Releases this lock and
 259  * grabs it again. Holds the lock when it returns.
 260  */
 261 static void wait_gp(void)
 262 {
 263         /*
 264          * Read reader_gp before read futex. smp_mb_master() needs to
 265          * be called with the rcu registry lock held in RCU_SIGNAL
 266          * flavor.
 267          */
 268         smp_mb_master();
 269         /* Temporarily unlock the registry lock. */
 270         mutex_unlock(&rcu_registry_lock);
 271         while (uatomic_read(&rcu_gp.futex) == -1) {
 272                 if (!futex_async(&rcu_gp.futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
 273                         /*
 274                          * Prior queued wakeups queued by unrelated code
 275                          * using the same address can cause futex wait to
 276                          * return 0 even through the futex value is still
 277                          * -1 (spurious wakeups). Check the value again
 278                          * in user-space to validate whether it really
 279                          * differs from -1.
 280                          */
 281                         continue;
 282                 }
 283                 switch (errno) {
 284                 case EAGAIN:
 285                         /* Value already changed. */
 286                         goto end;
 287                 case EINTR:
 288                         /* Retry if interrupted by signal. */
 289                         break;  /* Get out of switch. Check again. */
 290                 default:
 291                         /* Unexpected error. */
 292                         urcu_die(errno);
 293                 }
 294         }
 295 end:
 296         /*
 297          * Re-lock the registry lock before the next loop.
 298          */
 299         mutex_lock(&rcu_registry_lock);
 300 }
 301
 302 /*
 303  * Always called with rcu_registry lock held. Releases this lock between
 304  * iterations and grabs it again. Holds the lock when it returns.
 305  */
 306 static void wait_for_readers(struct cds_list_head *input_readers,
 307                         struct cds_list_head *cur_snap_readers,
 308                         struct cds_list_head *qsreaders)
 309 {
 310         unsigned int wait_loops = 0;
 311         struct urcu_reader *index, *tmp;
 312 #ifdef HAS_INCOHERENT_CACHES
 313         unsigned int wait_gp_loops = 0;
 314 #endif /* HAS_INCOHERENT_CACHES */
 315
 316         /*
 317          * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 318          * indicate quiescence (not nested), or observe the current
 319          * rcu_gp.ctr value.
 320          */
 321         for (;;) {
 322                 if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
 323                         wait_loops++;
 324                 if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 325                         uatomic_dec(&rcu_gp.futex);
 326                         /* Write futex before read reader_gp */
 327                         smp_mb_master();
 328                 }
 329
 330                 cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
 331                         switch (urcu_common_reader_state(&rcu_gp, &index->ctr)) {
 332                         case URCU_READER_ACTIVE_CURRENT:
 333                                 if (cur_snap_readers) {
 334                                         cds_list_move(&index->node,
 335                                                 cur_snap_readers);
 336                                         break;
 337                                 }
 338                                 /* Fall-through */
 339                         case URCU_READER_INACTIVE:
 340                                 cds_list_move(&index->node, qsreaders);
 341                                 break;
 342                         case URCU_READER_ACTIVE_OLD:
 343                                 /*
 344                                  * Old snapshot. Leaving node in
 345                                  * input_readers will make us busy-loop
 346                                  * until the snapshot becomes current or
 347                                  * the reader becomes inactive.
 348                                  */
 349                                 break;
 350                         }
 351                 }
 352
 353 #ifndef HAS_INCOHERENT_CACHES
 354                 if (cds_list_empty(input_readers)) {
 355                         if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 356                                 /* Read reader_gp before write futex */
 357                                 smp_mb_master();
 358                                 uatomic_set(&rcu_gp.futex, 0);
 359                         }
 360                         break;
 361                 } else {
 362                         if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 363                                 /* wait_gp unlocks/locks registry lock. */
 364                                 wait_gp();
 365                         } else {
 366                                 /* Temporarily unlock the registry lock. */
 367                                 mutex_unlock(&rcu_registry_lock);
 368                                 caa_cpu_relax();
 369                                 /*
 370                                  * Re-lock the registry lock before the
 371                                  * next loop.
 372                                  */
 373                                 mutex_lock(&rcu_registry_lock);
 374                         }
 375                 }
 376 #else /* #ifndef HAS_INCOHERENT_CACHES */
 377                 /*
 378                  * BUSY-LOOP. Force the reader thread to commit its
 379                  * URCU_TLS(rcu_reader).ctr update to memory if we wait
 380                  * for too long.
 381                  */
 382                 if (cds_list_empty(input_readers)) {
 383                         if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 384                                 /* Read reader_gp before write futex */
 385                                 smp_mb_master();
 386                                 uatomic_set(&rcu_gp.futex, 0);
 387                         }
 388                         break;
 389                 } else {
 390                         if (wait_gp_loops == KICK_READER_LOOPS) {
 391                                 smp_mb_master();
 392                                 wait_gp_loops = 0;
 393                         }
 394                         if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 395                                 /* wait_gp unlocks/locks registry lock. */
 396                                 wait_gp();
 397                                 wait_gp_loops++;
 398                         } else {
 399                                 /* Temporarily unlock the registry lock. */
 400                                 mutex_unlock(&rcu_registry_lock);
 401                                 caa_cpu_relax();
 402                                 /*
 403                                  * Re-lock the registry lock before the
 404                                  * next loop.
 405                                  */
 406                                 mutex_lock(&rcu_registry_lock);
 407                         }
 408                 }
 409 #endif /* #else #ifndef HAS_INCOHERENT_CACHES */
 410         }
 411 }
 412
 413 void synchronize_rcu(void)
 414 {
 415         CDS_LIST_HEAD(cur_snap_readers);
 416         CDS_LIST_HEAD(qsreaders);
 417         DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
 418         struct urcu_waiters waiters;
 419
 420         /*
 421          * Add ourself to gp_waiters queue of threads awaiting to wait
 422          * for a grace period. Proceed to perform the grace period only
 423          * if we are the first thread added into the queue.
 424          * The implicit memory barrier before urcu_wait_add()
 425          * orders prior memory accesses of threads put into the wait
 426          * queue before their insertion into the wait queue.
 427          */
 428         if (urcu_wait_add(&gp_waiters, &wait) != 0) {
 429                 /* Not first in queue: will be awakened by another thread. */
 430                 urcu_adaptative_busy_wait(&wait);
 431                 /* Order following memory accesses after grace period. */
 432                 cmm_smp_mb();
 433                 return;
 434         }
 435         /* We won't need to wake ourself up */
 436         urcu_wait_set_state(&wait, URCU_WAIT_RUNNING);
 437
 438         mutex_lock(&rcu_gp_lock);
 439
 440         /*
 441          * Move all waiters into our local queue.
 442          */
 443         urcu_move_waiters(&waiters, &gp_waiters);
 444
 445         mutex_lock(&rcu_registry_lock);
 446
 447         if (cds_list_empty(&registry))
 448                 goto out;
 449
 450         /*
 451          * All threads should read qparity before accessing data structure
 452          * where new ptr points to. Must be done within rcu_registry_lock
 453          * because it iterates on reader threads.
 454          */
 455         /* Write new ptr before changing the qparity */
 456         smp_mb_master();
 457
 458         /*
 459          * Wait for readers to observe original parity or be quiescent.
 460          * wait_for_readers() can release and grab again rcu_registry_lock
 461          * interally.
 462          */
 463         wait_for_readers(&registry, &cur_snap_readers, &qsreaders);
 464
 465         /*
 466          * Must finish waiting for quiescent state for original parity before
 467          * committing next rcu_gp.ctr update to memory. Failure to do so could
 468          * result in the writer waiting forever while new readers are always
 469          * accessing data (no progress).  Enforce compiler-order of load
 470          * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
 471          */
 472         cmm_barrier();
 473
 474         /*
 475          * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
 476          * model easier to understand. It does not have a big performance impact
 477          * anyway, given this is the write-side.
 478          */
 479         cmm_smp_mb();
 480
 481         /* Switch parity: 0 -> 1, 1 -> 0 */
 482         CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ URCU_GP_CTR_PHASE);
 483
 484         /*
 485          * Must commit rcu_gp.ctr update to memory before waiting for quiescent
 486          * state. Failure to do so could result in the writer waiting forever
 487          * while new readers are always accessing data (no progress). Enforce
 488          * compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
 489          */
 490         cmm_barrier();
 491
 492         /*
 493          *
 494          * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
 495          * model easier to understand. It does not have a big performance impact
 496          * anyway, given this is the write-side.
 497          */
 498         cmm_smp_mb();
 499
 500         /*
 501          * Wait for readers to observe new parity or be quiescent.
 502          * wait_for_readers() can release and grab again rcu_registry_lock
 503          * interally.
 504          */
 505         wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
 506
 507         /*
 508          * Put quiescent reader list back into registry.
 509          */
 510         cds_list_splice(&qsreaders, &registry);
 511
 512         /*
 513          * Finish waiting for reader threads before letting the old ptr
 514          * being freed. Must be done within rcu_registry_lock because it
 515          * iterates on reader threads.
 516          */
 517         smp_mb_master();
 518 out:
 519         mutex_unlock(&rcu_registry_lock);
 520         mutex_unlock(&rcu_gp_lock);
 521
 522         /*
 523          * Wakeup waiters only after we have completed the grace period
 524          * and have ensured the memory barriers at the end of the grace
 525          * period have been issued.
 526          */
 527         urcu_wake_all_waiters(&waiters);
 528 }
 529 URCU_ATTR_ALIAS(urcu_stringify(synchronize_rcu))
 530 void alias_synchronize_rcu();
 531
 532 /*
 533  * library wrappers to be used by non-LGPL compatible source code.
 534  */
 535
 536 void rcu_read_lock(void)
 537 {
 538         _rcu_read_lock();
 539 }
 540 URCU_ATTR_ALIAS(urcu_stringify(rcu_read_lock))
 541 void alias_rcu_read_lock();
 542
 543 void rcu_read_unlock(void)
 544 {
 545         _rcu_read_unlock();
 546 }
 547 URCU_ATTR_ALIAS(urcu_stringify(rcu_read_unlock))
 548 void alias_rcu_read_unlock();
 549
 550 int rcu_read_ongoing(void)
 551 {
 552         return _rcu_read_ongoing();
 553 }
 554 URCU_ATTR_ALIAS(urcu_stringify(rcu_read_ongoing))
 555 void alias_rcu_read_ongoing();
 556
 557 void rcu_register_thread(void)
 558 {
 559         URCU_TLS(rcu_reader).tid = pthread_self();
 560         assert(URCU_TLS(rcu_reader).need_mb == 0);
 561         assert(!(URCU_TLS(rcu_reader).ctr & URCU_GP_CTR_NEST_MASK));
 562
 563         mutex_lock(&rcu_registry_lock);
 564         assert(!URCU_TLS(rcu_reader).registered);
 565         URCU_TLS(rcu_reader).registered = 1;
 566         rcu_init();     /* In case gcc does not support constructor attribute */
 567         cds_list_add(&URCU_TLS(rcu_reader).node, &registry);
 568         mutex_unlock(&rcu_registry_lock);
 569 }
 570 URCU_ATTR_ALIAS(urcu_stringify(rcu_register_thread))
 571 void alias_rcu_register_thread();
 572
 573 void rcu_unregister_thread(void)
 574 {
 575         mutex_lock(&rcu_registry_lock);
 576         assert(URCU_TLS(rcu_reader).registered);
 577         URCU_TLS(rcu_reader).registered = 0;
 578         cds_list_del(&URCU_TLS(rcu_reader).node);
 579         mutex_unlock(&rcu_registry_lock);
 580 }
 581 URCU_ATTR_ALIAS(urcu_stringify(rcu_unregister_thread))
 582 void alias_rcu_unregister_thread();
 583
 584 #ifdef RCU_MEMBARRIER
 585
 586 #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
 587 static
 588 void rcu_sys_membarrier_status(bool available)
 589 {
 590         if (!available)
 591                 abort();
 592 }
 593 #else
 594 static
 595 void rcu_sys_membarrier_status(bool available)
 596 {
 597         if (!available)
 598                 return;
 599         urcu_memb_has_sys_membarrier = 1;
 600 }
 601 #endif
 602
 603 static
 604 void rcu_sys_membarrier_init(void)
 605 {
 606         bool available = false;
 607         int mask;
 608
 609         mask = membarrier(MEMBARRIER_CMD_QUERY, 0);
 610         if (mask >= 0) {
 611                 if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
 612                         if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0))
 613                                 urcu_die(errno);
 614                         urcu_memb_has_sys_membarrier_private_expedited = 1;
 615                         available = true;
 616                 } else if (mask & MEMBARRIER_CMD_SHARED) {
 617                         available = true;
 618                 }
 619         }
 620         rcu_sys_membarrier_status(available);
 621 }
 622
 623 void rcu_init(void)
 624 {
 625         if (init_done)
 626                 return;
 627         init_done = 1;
 628         rcu_sys_membarrier_init();
 629 }
 630 URCU_ATTR_ALIAS(urcu_stringify(rcu_init))
 631 void alias_rcu_init(void);
 632 #endif
 633
 634 #ifdef RCU_SIGNAL
 635 static void sigrcu_handler(int signo __attribute__((unused)),
 636                 siginfo_t *siginfo __attribute__((unused)),
 637                 void *context __attribute__((unused)))
 638 {
 639         /*
 640          * Executing this cmm_smp_mb() is the only purpose of this signal handler.
 641          * It punctually promotes cmm_barrier() into cmm_smp_mb() on every thread it is
 642          * executed on.
 643          */
 644         cmm_smp_mb();
 645         _CMM_STORE_SHARED(URCU_TLS(rcu_reader).need_mb, 0);
 646         cmm_smp_mb();
 647 }
 648
 649 /*
 650  * rcu_init constructor. Called when the library is linked, but also when
 651  * reader threads are calling rcu_register_thread().
 652  * Should only be called by a single thread at a given time. This is ensured by
 653  * holing the rcu_registry_lock from rcu_register_thread() or by running
 654  * at library load time, which should not be executed by multiple
 655  * threads nor concurrently with rcu_register_thread() anyway.
 656  */
 657 void rcu_init(void)
 658 {
 659         struct sigaction act;
 660         int ret;
 661
 662         if (init_done)
 663                 return;
 664         init_done = 1;
 665
 666         act.sa_sigaction = sigrcu_handler;
 667         act.sa_flags = SA_SIGINFO | SA_RESTART;
 668         sigemptyset(&act.sa_mask);
 669         ret = sigaction(SIGRCU, &act, NULL);
 670         if (ret)
 671                 urcu_die(errno);
 672 }
 673 URCU_ATTR_ALIAS(urcu_stringify(rcu_init))
 674 void alias_rcu_init(void);
 675
 676 /*
 677  * Don't unregister the SIGRCU signal handler anymore, because
 678  * call_rcu threads could still be using it shortly before the
 679  * application exits.
 680  * Assertion disabled because call_rcu threads are now rcu
 681  * readers, and left running at exit.
 682  * assert(cds_list_empty(&registry));
 683  */
 684
 685 #endif /* #ifdef RCU_SIGNAL */
 686
 687 void rcu_exit(void)
 688 {
 689         urcu_call_rcu_exit();
 690 }
 691 URCU_ATTR_ALIAS(urcu_stringify(rcu_exit))
 692 void alias_rcu_exit(void);
 693
 694 DEFINE_RCU_FLAVOR(rcu_flavor);
 695 DEFINE_RCU_FLAVOR_ALIAS(rcu_flavor, alias_rcu_flavor);
 696
 697 #include "urcu-call-rcu-impl.h"
 698 #include "urcu-defer-impl.h"