Fix: handle sys_futex() FUTEX_WAIT interrupted by signal
[urcu.git] / urcu-bp.c
CommitLineData
fdee2e6d
MD
1/*
2 * urcu-bp.c
3 *
4 * Userspace RCU library, "bulletproof" version.
5 *
6982d6d7 6 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
fdee2e6d
MD
7 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
8 *
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
13 *
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
18 *
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with this library; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 *
23 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
24 */
25
0617bf4c 26#define _GNU_SOURCE
71c811bf 27#define _LGPL_SOURCE
fdee2e6d
MD
28#include <stdio.h>
29#include <pthread.h>
30#include <signal.h>
31#include <assert.h>
32#include <stdlib.h>
33#include <string.h>
34#include <errno.h>
35#include <poll.h>
36#include <unistd.h>
37#include <sys/mman.h>
38
d73fb81f 39#include "urcu/wfcqueue.h"
57760d44 40#include "urcu/map/urcu-bp.h"
af7c2dbe 41#include "urcu/static/urcu-bp.h"
618b2595 42#include "urcu-pointer.h"
bd252a04 43#include "urcu/tls-compat.h"
71c811bf 44
4a6d7378
MD
45#include "urcu-die.h"
46
fdee2e6d 47/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
71c811bf 48#undef _LGPL_SOURCE
fdee2e6d 49#include "urcu-bp.h"
71c811bf 50#define _LGPL_SOURCE
fdee2e6d 51
4c1ae2ea
MD
52#ifndef MAP_ANONYMOUS
53#define MAP_ANONYMOUS MAP_ANON
54#endif
55
c7eaf61c
MD
56#ifdef __linux__
57static
58void *mremap_wrapper(void *old_address, size_t old_size,
59 size_t new_size, int flags)
60{
61 return mremap(old_address, old_size, new_size, flags);
62}
63#else
45a4872f
MD
64
65#define MREMAP_MAYMOVE 1
66#define MREMAP_FIXED 2
67
68/*
95b94246 69 * mremap wrapper for non-Linux systems not allowing MAYMOVE.
45a4872f
MD
70 * This is not generic.
71*/
c7eaf61c
MD
72static
73void *mremap_wrapper(void *old_address, size_t old_size,
74 size_t new_size, int flags)
45a4872f 75{
95b94246
MD
76 assert(!(flags & MREMAP_MAYMOVE));
77
78 return MAP_FAILED;
45a4872f
MD
79}
80#endif
81
9340c38d
MD
82/* Sleep delay in ms */
83#define RCU_SLEEP_DELAY_MS 10
95b94246
MD
84#define INIT_NR_THREADS 8
85#define ARENA_INIT_ALLOC \
86 sizeof(struct registry_chunk) \
87 + INIT_NR_THREADS * sizeof(struct rcu_reader)
fdee2e6d 88
b7b6a8f5
PB
89/*
90 * Active attempts to check for reader Q.S. before calling sleep().
91 */
92#define RCU_QS_ACTIVE_ATTEMPTS 100
93
76d6a951
MD
94static
95int rcu_bp_refcount;
96
c1be8fb9
MD
97static
98void __attribute__((constructor)) rcu_bp_init(void);
99static
02be5561 100void __attribute__((destructor)) rcu_bp_exit(void);
fdee2e6d 101
731ccb96
MD
102/*
103 * rcu_gp_lock ensures mutual exclusion between threads calling
104 * synchronize_rcu().
105 */
6abb4bd5 106static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
731ccb96
MD
107/*
108 * rcu_registry_lock ensures mutual exclusion between threads
109 * registering and unregistering themselves to/from the registry, and
110 * with threads reading that registry from synchronize_rcu(). However,
111 * this lock is not held all the way through the completion of awaiting
112 * for the grace period. It is sporadically released between iterations
113 * on the registry.
114 * rcu_registry_lock may nest inside rcu_gp_lock.
115 */
116static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
fdee2e6d 117
c1be8fb9
MD
118static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
119static int initialized;
120
121static pthread_key_t urcu_bp_key;
122
c13c2e55 123struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
fdee2e6d
MD
124
125/*
126 * Pointer to registry elements. Written to only by each individual reader. Read
127 * by both the reader and the writers.
128 */
bd252a04 129DEFINE_URCU_TLS(struct rcu_reader *, rcu_reader);
fdee2e6d 130
16aa9ee8 131static CDS_LIST_HEAD(registry);
fdee2e6d 132
95b94246
MD
133struct registry_chunk {
134 size_t data_len; /* data length */
c1be8fb9 135 size_t used; /* amount of data used */
95b94246
MD
136 struct cds_list_head node; /* chunk_list node */
137 char data[];
138};
139
fdee2e6d 140struct registry_arena {
95b94246 141 struct cds_list_head chunk_list;
fdee2e6d
MD
142};
143
95b94246
MD
144static struct registry_arena registry_arena = {
145 .chunk_list = CDS_LIST_HEAD_INIT(registry_arena.chunk_list),
146};
fdee2e6d 147
4cf1675f
MD
148/* Saved fork signal mask, protected by rcu_gp_lock */
149static sigset_t saved_fork_signal_mask;
150
6abb4bd5 151static void mutex_lock(pthread_mutex_t *mutex)
fdee2e6d
MD
152{
153 int ret;
154
155#ifndef DISTRUST_SIGNALS_EXTREME
6abb4bd5 156 ret = pthread_mutex_lock(mutex);
4a6d7378
MD
157 if (ret)
158 urcu_die(ret);
fdee2e6d 159#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
6abb4bd5 160 while ((ret = pthread_mutex_trylock(mutex)) != 0) {
4a6d7378
MD
161 if (ret != EBUSY && ret != EINTR)
162 urcu_die(ret);
fdee2e6d
MD
163 poll(NULL,0,10);
164 }
165#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
166}
167
6abb4bd5 168static void mutex_unlock(pthread_mutex_t *mutex)
fdee2e6d
MD
169{
170 int ret;
171
6abb4bd5 172 ret = pthread_mutex_unlock(mutex);
4a6d7378
MD
173 if (ret)
174 urcu_die(ret);
fdee2e6d
MD
175}
176
731ccb96
MD
177/*
178 * Always called with rcu_registry lock held. Releases this lock between
179 * iterations and grabs it again. Holds the lock when it returns.
180 */
52c75091
MD
181static void wait_for_readers(struct cds_list_head *input_readers,
182 struct cds_list_head *cur_snap_readers,
183 struct cds_list_head *qsreaders)
fdee2e6d 184{
9340c38d 185 unsigned int wait_loops = 0;
02be5561 186 struct rcu_reader *index, *tmp;
fdee2e6d 187
fdee2e6d 188 /*
dd61d077
MD
189 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
190 * indicate quiescence (not nested), or observe the current
c13c2e55 191 * rcu_gp.ctr value.
fdee2e6d
MD
192 */
193 for (;;) {
9340c38d
MD
194 if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
195 wait_loops++;
196
52c75091
MD
197 cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
198 switch (rcu_reader_state(&index->ctr)) {
199 case RCU_READER_ACTIVE_CURRENT:
200 if (cur_snap_readers) {
201 cds_list_move(&index->node,
202 cur_snap_readers);
203 break;
204 }
205 /* Fall-through */
206 case RCU_READER_INACTIVE:
207 cds_list_move(&index->node, qsreaders);
208 break;
209 case RCU_READER_ACTIVE_OLD:
210 /*
211 * Old snapshot. Leaving node in
212 * input_readers will make us busy-loop
213 * until the snapshot becomes current or
214 * the reader becomes inactive.
215 */
216 break;
217 }
fdee2e6d
MD
218 }
219
52c75091 220 if (cds_list_empty(input_readers)) {
fdee2e6d
MD
221 break;
222 } else {
731ccb96
MD
223 /* Temporarily unlock the registry lock. */
224 mutex_unlock(&rcu_registry_lock);
9340c38d
MD
225 if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS)
226 (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS);
fdee2e6d 227 else
06f22bdb 228 caa_cpu_relax();
731ccb96
MD
229 /* Re-lock the registry lock before the next loop. */
230 mutex_lock(&rcu_registry_lock);
fdee2e6d
MD
231 }
232 }
fdee2e6d
MD
233}
234
235void synchronize_rcu(void)
236{
52c75091
MD
237 CDS_LIST_HEAD(cur_snap_readers);
238 CDS_LIST_HEAD(qsreaders);
fdee2e6d
MD
239 sigset_t newmask, oldmask;
240 int ret;
241
6ed4b2e6 242 ret = sigfillset(&newmask);
fdee2e6d 243 assert(!ret);
6ed4b2e6 244 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
fdee2e6d
MD
245 assert(!ret);
246
6abb4bd5 247 mutex_lock(&rcu_gp_lock);
fdee2e6d 248
731ccb96
MD
249 mutex_lock(&rcu_registry_lock);
250
16aa9ee8 251 if (cds_list_empty(&registry))
2dfb8b5e 252 goto out;
fdee2e6d
MD
253
254 /* All threads should read qparity before accessing data structure
2dfb8b5e 255 * where new ptr points to. */
fdee2e6d 256 /* Write new ptr before changing the qparity */
5481ddb3 257 cmm_smp_mb();
fdee2e6d 258
fdee2e6d 259 /*
dd61d077 260 * Wait for readers to observe original parity or be quiescent.
731ccb96
MD
261 * wait_for_readers() can release and grab again rcu_registry_lock
262 * interally.
dd61d077 263 */
52c75091 264 wait_for_readers(&registry, &cur_snap_readers, &qsreaders);
dd61d077
MD
265
266 /*
267 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
268 * model easier to understand. It does not have a big performance impact
269 * anyway, given this is the write-side.
270 */
271 cmm_smp_mb();
272
273 /* Switch parity: 0 -> 1, 1 -> 0 */
c13c2e55 274 CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
dd61d077
MD
275
276 /*
277 * Must commit qparity update to memory before waiting for other parity
278 * quiescent state. Failure to do so could result in the writer waiting
279 * forever while new readers are always accessing data (no progress).
280 * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
fdee2e6d 281 */
fdee2e6d
MD
282
283 /*
5481ddb3 284 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
fdee2e6d
MD
285 * model easier to understand. It does not have a big performance impact
286 * anyway, given this is the write-side.
287 */
5481ddb3 288 cmm_smp_mb();
fdee2e6d 289
fdee2e6d 290 /*
dd61d077 291 * Wait for readers to observe new parity or be quiescent.
731ccb96
MD
292 * wait_for_readers() can release and grab again rcu_registry_lock
293 * interally.
fdee2e6d 294 */
52c75091
MD
295 wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
296
297 /*
298 * Put quiescent reader list back into registry.
299 */
300 cds_list_splice(&qsreaders, &registry);
fdee2e6d
MD
301
302 /*
2dfb8b5e
MD
303 * Finish waiting for reader threads before letting the old ptr being
304 * freed.
fdee2e6d 305 */
5481ddb3 306 cmm_smp_mb();
2dfb8b5e 307out:
731ccb96 308 mutex_unlock(&rcu_registry_lock);
6abb4bd5 309 mutex_unlock(&rcu_gp_lock);
fdee2e6d
MD
310 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
311 assert(!ret);
312}
313
314/*
315 * library wrappers to be used by non-LGPL compatible source code.
316 */
317
318void rcu_read_lock(void)
319{
320 _rcu_read_lock();
321}
322
323void rcu_read_unlock(void)
324{
325 _rcu_read_unlock();
326}
327
882f3357
MD
328int rcu_read_ongoing(void)
329{
330 return _rcu_read_ongoing();
331}
332
fdee2e6d 333/*
95b94246
MD
334 * Only grow for now. If empty, allocate a ARENA_INIT_ALLOC sized chunk.
335 * Else, try expanding the last chunk. If this fails, allocate a new
336 * chunk twice as big as the last chunk.
337 * Memory used by chunks _never_ moves. A chunk could theoretically be
338 * freed when all "used" slots are released, but we don't do it at this
339 * point.
fdee2e6d 340 */
95b94246
MD
341static
342void expand_arena(struct registry_arena *arena)
fdee2e6d 343{
95b94246
MD
344 struct registry_chunk *new_chunk, *last_chunk;
345 size_t old_chunk_len, new_chunk_len;
346
347 /* No chunk. */
348 if (cds_list_empty(&arena->chunk_list)) {
349 assert(ARENA_INIT_ALLOC >=
350 sizeof(struct registry_chunk)
351 + sizeof(struct rcu_reader));
352 new_chunk_len = ARENA_INIT_ALLOC;
353 new_chunk = mmap(NULL, new_chunk_len,
9d8612b7
MD
354 PROT_READ | PROT_WRITE,
355 MAP_ANONYMOUS | MAP_PRIVATE,
356 -1, 0);
95b94246
MD
357 if (new_chunk == MAP_FAILED)
358 abort();
359 bzero(new_chunk, new_chunk_len);
360 new_chunk->data_len =
361 new_chunk_len - sizeof(struct registry_chunk);
362 cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
363 return; /* We're done. */
364 }
9d8612b7 365
95b94246
MD
366 /* Try expanding last chunk. */
367 last_chunk = cds_list_entry(arena->chunk_list.prev,
368 struct registry_chunk, node);
369 old_chunk_len =
370 last_chunk->data_len + sizeof(struct registry_chunk);
371 new_chunk_len = old_chunk_len << 1;
372
373 /* Don't allow memory mapping to move, just expand. */
374 new_chunk = mremap_wrapper(last_chunk, old_chunk_len,
375 new_chunk_len, 0);
376 if (new_chunk != MAP_FAILED) {
377 /* Should not have moved. */
378 assert(new_chunk == last_chunk);
379 bzero((char *) last_chunk + old_chunk_len,
380 new_chunk_len - old_chunk_len);
381 last_chunk->data_len =
382 new_chunk_len - sizeof(struct registry_chunk);
383 return; /* We're done. */
384 }
0617bf4c 385
95b94246
MD
386 /* Remap did not succeed, we need to add a new chunk. */
387 new_chunk = mmap(NULL, new_chunk_len,
388 PROT_READ | PROT_WRITE,
389 MAP_ANONYMOUS | MAP_PRIVATE,
390 -1, 0);
391 if (new_chunk == MAP_FAILED)
392 abort();
393 bzero(new_chunk, new_chunk_len);
394 new_chunk->data_len =
395 new_chunk_len - sizeof(struct registry_chunk);
396 cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
397}
fdee2e6d 398
95b94246
MD
399static
400struct rcu_reader *arena_alloc(struct registry_arena *arena)
401{
402 struct registry_chunk *chunk;
403 struct rcu_reader *rcu_reader_reg;
404 int expand_done = 0; /* Only allow to expand once per alloc */
405 size_t len = sizeof(struct rcu_reader);
406
407retry:
408 cds_list_for_each_entry(chunk, &arena->chunk_list, node) {
409 if (chunk->data_len - chunk->used < len)
410 continue;
411 /* Find spot */
412 for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
413 rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
414 rcu_reader_reg++) {
415 if (!rcu_reader_reg->alloc) {
416 rcu_reader_reg->alloc = 1;
417 chunk->used += len;
418 return rcu_reader_reg;
419 }
420 }
421 }
422
423 if (!expand_done) {
424 expand_arena(arena);
425 expand_done = 1;
426 goto retry;
427 }
428
429 return NULL;
fdee2e6d
MD
430}
431
432/* Called with signals off and mutex locked */
95b94246
MD
433static
434void add_thread(void)
fdee2e6d 435{
02be5561 436 struct rcu_reader *rcu_reader_reg;
c1be8fb9 437 int ret;
fdee2e6d 438
95b94246
MD
439 rcu_reader_reg = arena_alloc(&registry_arena);
440 if (!rcu_reader_reg)
441 abort();
c1be8fb9
MD
442 ret = pthread_setspecific(urcu_bp_key, rcu_reader_reg);
443 if (ret)
444 abort();
fdee2e6d
MD
445
446 /* Add to registry */
02be5561
MD
447 rcu_reader_reg->tid = pthread_self();
448 assert(rcu_reader_reg->ctr == 0);
16aa9ee8 449 cds_list_add(&rcu_reader_reg->node, &registry);
95b94246
MD
450 /*
451 * Reader threads are pointing to the reader registry. This is
452 * why its memory should never be relocated.
453 */
bd252a04 454 URCU_TLS(rcu_reader) = rcu_reader_reg;
fdee2e6d
MD
455}
456
c1be8fb9
MD
457/* Called with mutex locked */
458static
459void cleanup_thread(struct registry_chunk *chunk,
460 struct rcu_reader *rcu_reader_reg)
461{
462 rcu_reader_reg->ctr = 0;
463 cds_list_del(&rcu_reader_reg->node);
464 rcu_reader_reg->tid = 0;
465 rcu_reader_reg->alloc = 0;
466 chunk->used -= sizeof(struct rcu_reader);
467}
468
469static
470struct registry_chunk *find_chunk(struct rcu_reader *rcu_reader_reg)
fdee2e6d 471{
95b94246 472 struct registry_chunk *chunk;
fdee2e6d 473
95b94246 474 cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
c1be8fb9
MD
475 if (rcu_reader_reg < (struct rcu_reader *) &chunk->data[0])
476 continue;
477 if (rcu_reader_reg >= (struct rcu_reader *) &chunk->data[chunk->data_len])
478 continue;
479 return chunk;
480 }
481 return NULL;
482}
95b94246 483
c1be8fb9
MD
484/* Called with signals off and mutex locked */
485static
76d6a951 486void remove_thread(struct rcu_reader *rcu_reader_reg)
c1be8fb9 487{
c1be8fb9
MD
488 cleanup_thread(find_chunk(rcu_reader_reg), rcu_reader_reg);
489 URCU_TLS(rcu_reader) = NULL;
fdee2e6d
MD
490}
491
492/* Disable signals, take mutex, add to registry */
493void rcu_bp_register(void)
494{
495 sigset_t newmask, oldmask;
496 int ret;
497
6ed4b2e6 498 ret = sigfillset(&newmask);
c1be8fb9
MD
499 if (ret)
500 abort();
6ed4b2e6 501 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
c1be8fb9
MD
502 if (ret)
503 abort();
fdee2e6d
MD
504
505 /*
506 * Check if a signal concurrently registered our thread since
c1be8fb9
MD
507 * the check in rcu_read_lock().
508 */
bd252a04 509 if (URCU_TLS(rcu_reader))
fdee2e6d
MD
510 goto end;
511
c1be8fb9
MD
512 /*
513 * Take care of early registration before urcu_bp constructor.
514 */
515 rcu_bp_init();
516
731ccb96 517 mutex_lock(&rcu_registry_lock);
fdee2e6d 518 add_thread();
731ccb96 519 mutex_unlock(&rcu_registry_lock);
fdee2e6d
MD
520end:
521 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
c1be8fb9
MD
522 if (ret)
523 abort();
524}
525
526/* Disable signals, take mutex, remove from registry */
527static
76d6a951 528void rcu_bp_unregister(struct rcu_reader *rcu_reader_reg)
c1be8fb9
MD
529{
530 sigset_t newmask, oldmask;
531 int ret;
532
533 ret = sigfillset(&newmask);
534 if (ret)
535 abort();
536 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
537 if (ret)
538 abort();
539
731ccb96 540 mutex_lock(&rcu_registry_lock);
76d6a951 541 remove_thread(rcu_reader_reg);
731ccb96 542 mutex_unlock(&rcu_registry_lock);
c1be8fb9
MD
543 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
544 if (ret)
545 abort();
76d6a951 546 rcu_bp_exit();
c1be8fb9
MD
547}
548
549/*
550 * Remove thread from the registry when it exits, and flag it as
551 * destroyed so garbage collection can take care of it.
552 */
553static
554void urcu_bp_thread_exit_notifier(void *rcu_key)
555{
76d6a951 556 rcu_bp_unregister(rcu_key);
c1be8fb9
MD
557}
558
559static
560void rcu_bp_init(void)
561{
562 mutex_lock(&init_lock);
76d6a951 563 if (!rcu_bp_refcount++) {
c1be8fb9
MD
564 int ret;
565
566 ret = pthread_key_create(&urcu_bp_key,
567 urcu_bp_thread_exit_notifier);
568 if (ret)
569 abort();
570 initialized = 1;
571 }
572 mutex_unlock(&init_lock);
fdee2e6d
MD
573}
574
c1be8fb9 575static
9380711a 576void rcu_bp_exit(void)
fdee2e6d 577{
76d6a951
MD
578 mutex_lock(&init_lock);
579 if (!--rcu_bp_refcount) {
580 struct registry_chunk *chunk, *tmp;
581 int ret;
95b94246 582
76d6a951
MD
583 cds_list_for_each_entry_safe(chunk, tmp,
584 &registry_arena.chunk_list, node) {
585 munmap(chunk, chunk->data_len
586 + sizeof(struct registry_chunk));
587 }
588 ret = pthread_key_delete(urcu_bp_key);
589 if (ret)
590 abort();
95b94246 591 }
76d6a951 592 mutex_unlock(&init_lock);
fdee2e6d 593}
4cf1675f
MD
594
595/*
731ccb96
MD
596 * Holding the rcu_gp_lock and rcu_registry_lock across fork will make
597 * sure we fork() don't race with a concurrent thread executing with
598 * any of those locks held. This ensures that the registry and data
599 * protected by rcu_gp_lock are in a coherent state in the child.
4cf1675f
MD
600 */
601void rcu_bp_before_fork(void)
602{
603 sigset_t newmask, oldmask;
604 int ret;
605
6ed4b2e6 606 ret = sigfillset(&newmask);
4cf1675f 607 assert(!ret);
6ed4b2e6 608 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
4cf1675f
MD
609 assert(!ret);
610 mutex_lock(&rcu_gp_lock);
731ccb96 611 mutex_lock(&rcu_registry_lock);
4cf1675f
MD
612 saved_fork_signal_mask = oldmask;
613}
614
615void rcu_bp_after_fork_parent(void)
616{
617 sigset_t oldmask;
618 int ret;
619
620 oldmask = saved_fork_signal_mask;
731ccb96 621 mutex_unlock(&rcu_registry_lock);
4cf1675f
MD
622 mutex_unlock(&rcu_gp_lock);
623 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
624 assert(!ret);
625}
626
c1be8fb9
MD
627/*
628 * Prune all entries from registry except our own thread. Fits the Linux
731ccb96 629 * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held.
c1be8fb9
MD
630 */
631static
632void urcu_bp_prune_registry(void)
633{
634 struct registry_chunk *chunk;
635 struct rcu_reader *rcu_reader_reg;
636
637 cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
638 for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
639 rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
640 rcu_reader_reg++) {
641 if (!rcu_reader_reg->alloc)
642 continue;
643 if (rcu_reader_reg->tid == pthread_self())
644 continue;
645 cleanup_thread(chunk, rcu_reader_reg);
646 }
647 }
648}
649
4cf1675f
MD
650void rcu_bp_after_fork_child(void)
651{
652 sigset_t oldmask;
653 int ret;
654
c1be8fb9 655 urcu_bp_prune_registry();
4cf1675f 656 oldmask = saved_fork_signal_mask;
731ccb96 657 mutex_unlock(&rcu_registry_lock);
4cf1675f
MD
658 mutex_unlock(&rcu_gp_lock);
659 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
660 assert(!ret);
661}
5e77fc1f 662
9b7981bb
MD
663void *rcu_dereference_sym_bp(void *p)
664{
665 return _rcu_dereference(p);
666}
667
5efd3cd2
MD
668void *rcu_set_pointer_sym_bp(void **p, void *v)
669{
670 cmm_wmb();
424d4ed5
MD
671 uatomic_set(p, v);
672 return v;
5efd3cd2
MD
673}
674
675void *rcu_xchg_pointer_sym_bp(void **p, void *v)
676{
677 cmm_wmb();
678 return uatomic_xchg(p, v);
679}
680
681void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new)
682{
683 cmm_wmb();
684 return uatomic_cmpxchg(p, old, _new);
685}
686
5e6b23a6 687DEFINE_RCU_FLAVOR(rcu_flavor);
541d828d 688
5e77fc1f 689#include "urcu-call-rcu-impl.h"
0376e7b2 690#include "urcu-defer-impl.h"
This page took 0.099207 seconds and 4 git commands to generate.