src/urcu-bp.c

   1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
   3 //
   4 // SPDX-License-Identifier: LGPL-2.1-or-later
   5
   6 /*
   7  * Userspace RCU library, "bulletproof" version.
   8  *
   9  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  10  */
  11
  12 #define URCU_NO_COMPAT_IDENTIFIERS
  13 #define _LGPL_SOURCE
  14 #include <stdio.h>
  15 #include <pthread.h>
  16 #include <signal.h>
  17 #include <stdlib.h>
  18 #include <string.h>
  19 #include <errno.h>
  20 #include <poll.h>
  21 #include <unistd.h>
  22 #include <stdbool.h>
  23 #include <sys/mman.h>
  24
  25 #include <urcu/annotate.h>
  26 #include <urcu/assert.h>
  27 #include <urcu/config.h>
  28 #include <urcu/arch.h>
  29 #include <urcu/wfcqueue.h>
  30 #include <urcu/map/urcu-bp.h>
  31 #include <urcu/static/urcu-bp.h>
  32 #include <urcu/pointer.h>
  33 #include <urcu/tls-compat.h>
  34
  35 #include "urcu-die.h"
  36 #include "urcu-utils.h"
  37
  38 #define URCU_API_MAP
  39 /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
  40 #undef _LGPL_SOURCE
  41 #include <urcu/urcu-bp.h>
  42 #define _LGPL_SOURCE
  43
  44 #ifndef MAP_ANONYMOUS
  45 #define MAP_ANONYMOUS MAP_ANON
  46 #endif
  47
  48 #ifdef __linux__
  49 static
  50 void *mremap_wrapper(void *old_address, size_t old_size,
  51                 size_t new_size, int flags)
  52 {
  53         return mremap(old_address, old_size, new_size, flags);
  54 }
  55 #else
  56
  57 #define MREMAP_MAYMOVE  1
  58 #define MREMAP_FIXED    2
  59
  60 /*
  61  * mremap wrapper for non-Linux systems not allowing MAYMOVE.
  62  * This is not generic.
  63 */
  64 static
  65 void *mremap_wrapper(void *old_address __attribute__((unused)),
  66                 size_t old_size __attribute__((unused)),
  67                 size_t new_size __attribute__((unused)),
  68                 int flags)
  69 {
  70         urcu_posix_assert(!(flags & MREMAP_MAYMOVE));
  71
  72         return MAP_FAILED;
  73 }
  74 #endif
  75
  76 /* Sleep delay in ms */
  77 #define RCU_SLEEP_DELAY_MS      10
  78 #define INIT_NR_THREADS         8
  79 #define ARENA_INIT_ALLOC                \
  80         sizeof(struct registry_chunk)   \
  81         + INIT_NR_THREADS * sizeof(struct urcu_bp_reader)
  82
  83 /*
  84  * Active attempts to check for reader Q.S. before calling sleep().
  85  */
  86 #define RCU_QS_ACTIVE_ATTEMPTS 100
  87
  88 static
  89 int urcu_bp_refcount;
  90
  91 /* If the headers do not support membarrier system call, fall back smp_mb. */
  92 #ifdef __NR_membarrier
  93 # define membarrier(...)                syscall(__NR_membarrier, __VA_ARGS__)
  94 #else
  95 # define membarrier(...)                -ENOSYS
  96 #endif
  97
  98 enum membarrier_cmd {
  99         MEMBARRIER_CMD_QUERY                            = 0,
 100         MEMBARRIER_CMD_SHARED                           = (1 << 0),
 101         /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
 102         /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
 103         MEMBARRIER_CMD_PRIVATE_EXPEDITED                = (1 << 3),
 104         MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED       = (1 << 4),
 105 };
 106
 107 static
 108 void __attribute__((constructor)) _urcu_bp_init(void);
 109 static
 110 void urcu_bp_exit(void);
 111 static
 112 void __attribute__((destructor)) urcu_bp_exit_destructor(void);
 113 static void urcu_call_rcu_exit(void);
 114
 115 #ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER
 116 int urcu_bp_has_sys_membarrier;
 117 #endif
 118
 119 /*
 120  * rcu_gp_lock ensures mutual exclusion between threads calling
 121  * synchronize_rcu().
 122  */
 123 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
 124 /*
 125  * rcu_registry_lock ensures mutual exclusion between threads
 126  * registering and unregistering themselves to/from the registry, and
 127  * with threads reading that registry from synchronize_rcu(). However,
 128  * this lock is not held all the way through the completion of awaiting
 129  * for the grace period. It is sporadically released between iterations
 130  * on the registry.
 131  * rcu_registry_lock may nest inside rcu_gp_lock.
 132  */
 133 static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
 134
 135 static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
 136 static int initialized;
 137
 138 static pthread_key_t urcu_bp_key;
 139
 140 struct urcu_bp_gp urcu_bp_gp = { .ctr = URCU_BP_GP_COUNT };
 141
 142 /*
 143  * Pointer to registry elements. Written to only by each individual reader. Read
 144  * by both the reader and the writers.
 145  */
 146 DEFINE_URCU_TLS(struct urcu_bp_reader *, urcu_bp_reader);
 147
 148 static CDS_LIST_HEAD(registry);
 149
 150 struct registry_chunk {
 151         size_t data_len;                /* data length */
 152         size_t used;                    /* amount of data used */
 153         struct cds_list_head node;      /* chunk_list node */
 154         char data[];
 155 };
 156
 157 struct registry_arena {
 158         struct cds_list_head chunk_list;
 159 };
 160
 161 static struct registry_arena registry_arena = {
 162         .chunk_list = CDS_LIST_HEAD_INIT(registry_arena.chunk_list),
 163 };
 164
 165 /* Saved fork signal mask, protected by rcu_gp_lock */
 166 static sigset_t saved_fork_signal_mask;
 167
 168 static void mutex_lock(pthread_mutex_t *mutex)
 169 {
 170         int ret;
 171
 172 #ifndef DISTRUST_SIGNALS_EXTREME
 173         ret = pthread_mutex_lock(mutex);
 174         if (ret)
 175                 urcu_die(ret);
 176 #else /* #ifndef DISTRUST_SIGNALS_EXTREME */
 177         while ((ret = pthread_mutex_trylock(mutex)) != 0) {
 178                 if (ret != EBUSY && ret != EINTR)
 179                         urcu_die(ret);
 180                 poll(NULL,0,10);
 181         }
 182 #endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
 183 }
 184
 185 static void mutex_unlock(pthread_mutex_t *mutex)
 186 {
 187         int ret;
 188
 189         ret = pthread_mutex_unlock(mutex);
 190         if (ret)
 191                 urcu_die(ret);
 192 }
 193
 194 static void smp_mb_master(void)
 195 {
 196         if (caa_likely(urcu_bp_has_sys_membarrier)) {
 197                 if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0))
 198                         urcu_die(errno);
 199         } else {
 200                 cmm_smp_mb();
 201         }
 202 }
 203
 204 /*
 205  * Always called with rcu_registry lock held. Releases this lock between
 206  * iterations and grabs it again. Holds the lock when it returns.
 207  */
 208 static void wait_for_readers(struct cds_list_head *input_readers,
 209                         struct cds_list_head *cur_snap_readers,
 210                         struct cds_list_head *qsreaders,
 211                         cmm_annotate_t *group)
 212 {
 213         unsigned int wait_loops = 0;
 214         struct urcu_bp_reader *index, *tmp;
 215
 216         /*
 217          * Wait for each thread URCU_TLS(urcu_bp_reader).ctr to either
 218          * indicate quiescence (not nested), or observe the current
 219          * rcu_gp.ctr value.
 220          */
 221         for (;;) {
 222                 if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
 223                         wait_loops++;
 224
 225                 cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
 226                         switch (urcu_bp_reader_state(&index->ctr, group)) {
 227                         case URCU_BP_READER_ACTIVE_CURRENT:
 228                                 if (cur_snap_readers) {
 229                                         cds_list_move(&index->node,
 230                                                 cur_snap_readers);
 231                                         break;
 232                                 }
 233                                 /* Fall-through */
 234                         case URCU_BP_READER_INACTIVE:
 235                                 cds_list_move(&index->node, qsreaders);
 236                                 break;
 237                         case URCU_BP_READER_ACTIVE_OLD:
 238                                 /*
 239                                  * Old snapshot. Leaving node in
 240                                  * input_readers will make us busy-loop
 241                                  * until the snapshot becomes current or
 242                                  * the reader becomes inactive.
 243                                  */
 244                                 break;
 245                         }
 246                 }
 247
 248                 if (cds_list_empty(input_readers)) {
 249                         break;
 250                 } else {
 251                         /* Temporarily unlock the registry lock. */
 252                         mutex_unlock(&rcu_registry_lock);
 253                         if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS)
 254                                 (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS);
 255                         else
 256                                 caa_cpu_relax();
 257                         /* Re-lock the registry lock before the next loop. */
 258                         mutex_lock(&rcu_registry_lock);
 259                 }
 260         }
 261 }
 262
 263 void urcu_bp_synchronize_rcu(void)
 264 {
 265         cmm_annotate_define(acquire_group);
 266         cmm_annotate_define(release_group);
 267         CDS_LIST_HEAD(cur_snap_readers);
 268         CDS_LIST_HEAD(qsreaders);
 269         sigset_t newmask, oldmask;
 270         int ret;
 271
 272         ret = sigfillset(&newmask);
 273         urcu_posix_assert(!ret);
 274         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 275         urcu_posix_assert(!ret);
 276
 277         mutex_lock(&rcu_gp_lock);
 278
 279         mutex_lock(&rcu_registry_lock);
 280
 281         if (cds_list_empty(&registry))
 282                 goto out;
 283
 284         /* All threads should read qparity before accessing data structure
 285          * where new ptr points to. */
 286         /* Write new ptr before changing the qparity */
 287         smp_mb_master();
 288         cmm_annotate_group_mb_release(&release_group);
 289
 290         /*
 291          * Wait for readers to observe original parity or be quiescent.
 292          * wait_for_readers() can release and grab again rcu_registry_lock
 293          * internally.
 294          */
 295         wait_for_readers(&registry, &cur_snap_readers, &qsreaders, &acquire_group);
 296
 297         /*
 298          * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
 299          * model easier to understand. It does not have a big performance impact
 300          * anyway, given this is the write-side.
 301          */
 302         cmm_smp_mb();
 303
 304         /* Switch parity: 0 -> 1, 1 -> 0 */
 305         cmm_annotate_group_mem_release(&release_group, &rcu_gp.ctr);
 306         uatomic_store(&rcu_gp.ctr, rcu_gp.ctr ^ URCU_BP_GP_CTR_PHASE, CMM_RELAXED);
 307
 308         /*
 309          * Must commit qparity update to memory before waiting for other parity
 310          * quiescent state. Failure to do so could result in the writer waiting
 311          * forever while new readers are always accessing data (no progress).
 312          * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
 313          */
 314
 315         /*
 316          * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
 317          * model easier to understand. It does not have a big performance impact
 318          * anyway, given this is the write-side.
 319          */
 320         cmm_smp_mb();
 321
 322         /*
 323          * Wait for readers to observe new parity or be quiescent.
 324          * wait_for_readers() can release and grab again rcu_registry_lock
 325          * internally.
 326          */
 327         wait_for_readers(&cur_snap_readers, NULL, &qsreaders, &acquire_group);
 328
 329         /*
 330          * Put quiescent reader list back into registry.
 331          */
 332         cds_list_splice(&qsreaders, &registry);
 333
 334         /*
 335          * Finish waiting for reader threads before letting the old ptr being
 336          * freed.
 337          */
 338         smp_mb_master();
 339         cmm_annotate_group_mb_acquire(&acquire_group);
 340 out:
 341         mutex_unlock(&rcu_registry_lock);
 342         mutex_unlock(&rcu_gp_lock);
 343         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 344         urcu_posix_assert(!ret);
 345 }
 346
 347 /*
 348  * library wrappers to be used by non-LGPL compatible source code.
 349  */
 350
 351 void urcu_bp_read_lock(void)
 352 {
 353         _urcu_bp_read_lock();
 354 }
 355
 356 void urcu_bp_read_unlock(void)
 357 {
 358         _urcu_bp_read_unlock();
 359 }
 360
 361 int urcu_bp_read_ongoing(void)
 362 {
 363         return _urcu_bp_read_ongoing();
 364 }
 365
 366 /*
 367  * Only grow for now. If empty, allocate a ARENA_INIT_ALLOC sized chunk.
 368  * Else, try expanding the last chunk. If this fails, allocate a new
 369  * chunk twice as big as the last chunk.
 370  * Memory used by chunks _never_ moves. A chunk could theoretically be
 371  * freed when all "used" slots are released, but we don't do it at this
 372  * point.
 373  */
 374 static
 375 void expand_arena(struct registry_arena *arena)
 376 {
 377         struct registry_chunk *new_chunk, *last_chunk;
 378         size_t old_chunk_len, new_chunk_len;
 379
 380         /* No chunk. */
 381         if (cds_list_empty(&arena->chunk_list)) {
 382                 urcu_posix_assert(ARENA_INIT_ALLOC >=
 383                         sizeof(struct registry_chunk)
 384                         + sizeof(struct rcu_reader));
 385                 new_chunk_len = ARENA_INIT_ALLOC;
 386                 new_chunk = (struct registry_chunk *) mmap(NULL,
 387                         new_chunk_len,
 388                         PROT_READ | PROT_WRITE,
 389                         MAP_ANONYMOUS | MAP_PRIVATE,
 390                         -1, 0);
 391                 if (new_chunk == MAP_FAILED)
 392                         abort();
 393                 memset(new_chunk, 0, new_chunk_len);
 394                 new_chunk->data_len =
 395                         new_chunk_len - sizeof(struct registry_chunk);
 396                 cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
 397                 return;         /* We're done. */
 398         }
 399
 400         /* Try expanding last chunk. */
 401         last_chunk = cds_list_entry(arena->chunk_list.prev,
 402                 struct registry_chunk, node);
 403         old_chunk_len =
 404                 last_chunk->data_len + sizeof(struct registry_chunk);
 405         new_chunk_len = old_chunk_len << 1;
 406
 407         /* Don't allow memory mapping to move, just expand. */
 408         new_chunk = mremap_wrapper(last_chunk, old_chunk_len,
 409                 new_chunk_len, 0);
 410         if (new_chunk != MAP_FAILED) {
 411                 /* Should not have moved. */
 412                 urcu_posix_assert(new_chunk == last_chunk);
 413                 memset((char *) last_chunk + old_chunk_len, 0,
 414                         new_chunk_len - old_chunk_len);
 415                 last_chunk->data_len =
 416                         new_chunk_len - sizeof(struct registry_chunk);
 417                 return;         /* We're done. */
 418         }
 419
 420         /* Remap did not succeed, we need to add a new chunk. */
 421         new_chunk = (struct registry_chunk *) mmap(NULL,
 422                 new_chunk_len,
 423                 PROT_READ | PROT_WRITE,
 424                 MAP_ANONYMOUS | MAP_PRIVATE,
 425                 -1, 0);
 426         if (new_chunk == MAP_FAILED)
 427                 abort();
 428         memset(new_chunk, 0, new_chunk_len);
 429         new_chunk->data_len =
 430                 new_chunk_len - sizeof(struct registry_chunk);
 431         cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
 432 }
 433
 434 static
 435 struct rcu_reader *arena_alloc(struct registry_arena *arena)
 436 {
 437         struct registry_chunk *chunk;
 438         struct rcu_reader *rcu_reader_reg;
 439         int expand_done = 0;    /* Only allow to expand once per alloc */
 440         size_t len = sizeof(struct rcu_reader);
 441
 442 retry:
 443         cds_list_for_each_entry(chunk, &arena->chunk_list, node) {
 444                 if (chunk->data_len - chunk->used < len)
 445                         continue;
 446                 /* Find spot */
 447                 for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
 448                                 rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
 449                                 rcu_reader_reg++) {
 450                         if (!rcu_reader_reg->alloc) {
 451                                 rcu_reader_reg->alloc = 1;
 452                                 chunk->used += len;
 453                                 return rcu_reader_reg;
 454                         }
 455                 }
 456         }
 457
 458         if (!expand_done) {
 459                 expand_arena(arena);
 460                 expand_done = 1;
 461                 goto retry;
 462         }
 463
 464         return NULL;
 465 }
 466
 467 /* Called with signals off and mutex locked */
 468 static
 469 void add_thread(void)
 470 {
 471         struct rcu_reader *rcu_reader_reg;
 472         int ret;
 473
 474         rcu_reader_reg = arena_alloc(&registry_arena);
 475         if (!rcu_reader_reg)
 476                 abort();
 477         ret = pthread_setspecific(urcu_bp_key, rcu_reader_reg);
 478         if (ret)
 479                 abort();
 480
 481         /* Add to registry */
 482         rcu_reader_reg->tid = pthread_self();
 483         urcu_posix_assert(rcu_reader_reg->ctr == 0);
 484         cds_list_add(&rcu_reader_reg->node, &registry);
 485         /*
 486          * Reader threads are pointing to the reader registry. This is
 487          * why its memory should never be relocated.
 488          */
 489         URCU_TLS(urcu_bp_reader) = rcu_reader_reg;
 490 }
 491
 492 /* Called with mutex locked */
 493 static
 494 void cleanup_thread(struct registry_chunk *chunk,
 495                 struct rcu_reader *rcu_reader_reg)
 496 {
 497         rcu_reader_reg->ctr = 0;
 498         cds_list_del(&rcu_reader_reg->node);
 499         rcu_reader_reg->tid = 0;
 500         rcu_reader_reg->alloc = 0;
 501         chunk->used -= sizeof(struct rcu_reader);
 502 }
 503
 504 static
 505 struct registry_chunk *find_chunk(struct rcu_reader *rcu_reader_reg)
 506 {
 507         struct registry_chunk *chunk;
 508
 509         cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
 510                 if (rcu_reader_reg < (struct rcu_reader *) &chunk->data[0])
 511                         continue;
 512                 if (rcu_reader_reg >= (struct rcu_reader *) &chunk->data[chunk->data_len])
 513                         continue;
 514                 return chunk;
 515         }
 516         return NULL;
 517 }
 518
 519 /* Called with signals off and mutex locked */
 520 static
 521 void remove_thread(struct rcu_reader *rcu_reader_reg)
 522 {
 523         cleanup_thread(find_chunk(rcu_reader_reg), rcu_reader_reg);
 524         URCU_TLS(urcu_bp_reader) = NULL;
 525 }
 526
 527 /* Disable signals, take mutex, add to registry */
 528 void urcu_bp_register(void)
 529 {
 530         sigset_t newmask, oldmask;
 531         int ret;
 532
 533         ret = sigfillset(&newmask);
 534         if (ret)
 535                 abort();
 536         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 537         if (ret)
 538                 abort();
 539
 540         /*
 541          * Check if a signal concurrently registered our thread since
 542          * the check in rcu_read_lock().
 543          */
 544         if (URCU_TLS(urcu_bp_reader))
 545                 goto end;
 546
 547         /*
 548          * Take care of early registration before urcu_bp constructor.
 549          */
 550         _urcu_bp_init();
 551
 552         mutex_lock(&rcu_registry_lock);
 553         add_thread();
 554         mutex_unlock(&rcu_registry_lock);
 555 end:
 556         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 557         if (ret)
 558                 abort();
 559 }
 560
 561 void urcu_bp_register_thread(void)
 562 {
 563         if (caa_unlikely(!URCU_TLS(urcu_bp_reader)))
 564                 urcu_bp_register(); /* If not yet registered. */
 565 }
 566
 567 /* Disable signals, take mutex, remove from registry */
 568 static
 569 void urcu_bp_unregister(struct rcu_reader *rcu_reader_reg)
 570 {
 571         sigset_t newmask, oldmask;
 572         int ret;
 573
 574         ret = sigfillset(&newmask);
 575         if (ret)
 576                 abort();
 577         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 578         if (ret)
 579                 abort();
 580
 581         mutex_lock(&rcu_registry_lock);
 582         remove_thread(rcu_reader_reg);
 583         mutex_unlock(&rcu_registry_lock);
 584         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 585         if (ret)
 586                 abort();
 587         urcu_bp_exit();
 588 }
 589
 590 /*
 591  * Remove thread from the registry when it exits, and flag it as
 592  * destroyed so garbage collection can take care of it.
 593  */
 594 static
 595 void urcu_bp_thread_exit_notifier(void *rcu_key)
 596 {
 597         urcu_bp_unregister(rcu_key);
 598 }
 599
 600 #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
 601 static
 602 void urcu_bp_sys_membarrier_status(bool available)
 603 {
 604         if (!available)
 605                 abort();
 606 }
 607 #else
 608 static
 609 void urcu_bp_sys_membarrier_status(bool available)
 610 {
 611         if (!available)
 612                 return;
 613         urcu_bp_has_sys_membarrier = 1;
 614 }
 615 #endif
 616
 617 static
 618 void urcu_bp_sys_membarrier_init(void)
 619 {
 620         bool available = false;
 621         int mask;
 622
 623         mask = membarrier(MEMBARRIER_CMD_QUERY, 0);
 624         if (mask >= 0) {
 625                 if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
 626                         if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0))
 627                                 urcu_die(errno);
 628                         available = true;
 629                 }
 630         }
 631         urcu_bp_sys_membarrier_status(available);
 632 }
 633
 634 static
 635 void _urcu_bp_init(void)
 636 {
 637         mutex_lock(&init_lock);
 638         if (!urcu_bp_refcount++) {
 639                 int ret;
 640
 641                 ret = pthread_key_create(&urcu_bp_key,
 642                                 urcu_bp_thread_exit_notifier);
 643                 if (ret)
 644                         abort();
 645                 urcu_bp_sys_membarrier_init();
 646                 initialized = 1;
 647         }
 648         mutex_unlock(&init_lock);
 649 }
 650
 651 static
 652 void urcu_bp_exit(void)
 653 {
 654         mutex_lock(&init_lock);
 655         if (!--urcu_bp_refcount) {
 656                 struct registry_chunk *chunk, *tmp;
 657                 int ret;
 658
 659                 cds_list_for_each_entry_safe(chunk, tmp,
 660                                 &registry_arena.chunk_list, node) {
 661                         munmap((void *) chunk, chunk->data_len
 662                                         + sizeof(struct registry_chunk));
 663                 }
 664                 CDS_INIT_LIST_HEAD(&registry_arena.chunk_list);
 665                 ret = pthread_key_delete(urcu_bp_key);
 666                 if (ret)
 667                         abort();
 668         }
 669         mutex_unlock(&init_lock);
 670 }
 671
 672 static
 673 void urcu_bp_exit_destructor(void)
 674 {
 675         urcu_call_rcu_exit();
 676         urcu_bp_exit();
 677 }
 678
 679 /*
 680  * Holding the rcu_gp_lock and rcu_registry_lock across fork will make
 681  * sure we fork() don't race with a concurrent thread executing with
 682  * any of those locks held. This ensures that the registry and data
 683  * protected by rcu_gp_lock are in a coherent state in the child.
 684  */
 685 void urcu_bp_before_fork(void)
 686 {
 687         sigset_t newmask, oldmask;
 688         int ret;
 689
 690         ret = sigfillset(&newmask);
 691         urcu_posix_assert(!ret);
 692         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 693         urcu_posix_assert(!ret);
 694         mutex_lock(&rcu_gp_lock);
 695         mutex_lock(&rcu_registry_lock);
 696         saved_fork_signal_mask = oldmask;
 697 }
 698
 699 void urcu_bp_after_fork_parent(void)
 700 {
 701         sigset_t oldmask;
 702         int ret;
 703
 704         oldmask = saved_fork_signal_mask;
 705         mutex_unlock(&rcu_registry_lock);
 706         mutex_unlock(&rcu_gp_lock);
 707         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 708         urcu_posix_assert(!ret);
 709 }
 710
 711 /*
 712  * Prune all entries from registry except our own thread. Fits the Linux
 713  * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held.
 714  */
 715 static
 716 void urcu_bp_prune_registry(void)
 717 {
 718         struct registry_chunk *chunk;
 719         struct urcu_bp_reader *rcu_reader_reg;
 720
 721         cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
 722                 for (rcu_reader_reg = (struct urcu_bp_reader *) &chunk->data[0];
 723                                 rcu_reader_reg < (struct urcu_bp_reader *) &chunk->data[chunk->data_len];
 724                                 rcu_reader_reg++) {
 725                         if (!rcu_reader_reg->alloc)
 726                                 continue;
 727                         if (rcu_reader_reg->tid == pthread_self())
 728                                 continue;
 729                         cleanup_thread(chunk, rcu_reader_reg);
 730                 }
 731         }
 732 }
 733
 734 void urcu_bp_after_fork_child(void)
 735 {
 736         sigset_t oldmask;
 737         int ret;
 738
 739         urcu_bp_prune_registry();
 740         oldmask = saved_fork_signal_mask;
 741         mutex_unlock(&rcu_registry_lock);
 742         mutex_unlock(&rcu_gp_lock);
 743         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 744         urcu_posix_assert(!ret);
 745 }
 746
 747 void *urcu_bp_dereference_sym(void *p)
 748 {
 749         return _rcu_dereference(p);
 750 }
 751
 752 void *urcu_bp_set_pointer_sym(void **p, void *v)
 753 {
 754         cmm_wmb();
 755         uatomic_set(p, v);
 756         return v;
 757 }
 758
 759 void *urcu_bp_xchg_pointer_sym(void **p, void *v)
 760 {
 761         cmm_wmb();
 762         return uatomic_xchg(p, v);
 763 }
 764
 765 void *urcu_bp_cmpxchg_pointer_sym(void **p, void *old, void *_new)
 766 {
 767         cmm_wmb();
 768         return uatomic_cmpxchg(p, old, _new);
 769 }
 770
 771 DEFINE_RCU_FLAVOR(rcu_flavor);
 772
 773 #include "urcu-call-rcu-impl.h"
 774 #include "urcu-defer-impl.h"
 775 #include "urcu-poll-impl.h"