Bump version to 0.14.0-pre
[urcu.git] / src / urcu-bp.c
CommitLineData
fdee2e6d
MD
1/*
2 * urcu-bp.c
3 *
4 * Userspace RCU library, "bulletproof" version.
5 *
6982d6d7 6 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
fdee2e6d
MD
7 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
8 *
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
13 *
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
18 *
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with this library; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 *
23 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
24 */
25
e37faee1 26#define URCU_NO_COMPAT_IDENTIFIERS
71c811bf 27#define _LGPL_SOURCE
fdee2e6d
MD
28#include <stdio.h>
29#include <pthread.h>
30#include <signal.h>
fdee2e6d
MD
31#include <stdlib.h>
32#include <string.h>
33#include <errno.h>
34#include <poll.h>
35#include <unistd.h>
3745305b 36#include <stdbool.h>
fdee2e6d
MD
37#include <sys/mman.h>
38
01477510 39#include <urcu/assert.h>
375db287 40#include <urcu/config.h>
4477a870
MD
41#include <urcu/arch.h>
42#include <urcu/wfcqueue.h>
43#include <urcu/map/urcu-bp.h>
44#include <urcu/static/urcu-bp.h>
45#include <urcu/pointer.h>
46#include <urcu/tls-compat.h>
71c811bf 47
4a6d7378 48#include "urcu-die.h"
ce28e67a 49#include "urcu-utils.h"
4a6d7378 50
4477a870 51#define URCU_API_MAP
fdee2e6d 52/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
71c811bf 53#undef _LGPL_SOURCE
4477a870 54#include <urcu/urcu-bp.h>
71c811bf 55#define _LGPL_SOURCE
fdee2e6d 56
4c1ae2ea
MD
57#ifndef MAP_ANONYMOUS
58#define MAP_ANONYMOUS MAP_ANON
59#endif
60
c7eaf61c
MD
61#ifdef __linux__
62static
63void *mremap_wrapper(void *old_address, size_t old_size,
64 size_t new_size, int flags)
65{
66 return mremap(old_address, old_size, new_size, flags);
67}
68#else
45a4872f
MD
69
70#define MREMAP_MAYMOVE 1
71#define MREMAP_FIXED 2
72
73/*
95b94246 74 * mremap wrapper for non-Linux systems not allowing MAYMOVE.
45a4872f
MD
75 * This is not generic.
76*/
c7eaf61c 77static
a142df4e
MJ
78void *mremap_wrapper(void *old_address __attribute__((unused)),
79 size_t old_size __attribute__((unused)),
80 size_t new_size __attribute__((unused)),
81 int flags)
45a4872f 82{
01477510 83 urcu_posix_assert(!(flags & MREMAP_MAYMOVE));
95b94246
MD
84
85 return MAP_FAILED;
45a4872f
MD
86}
87#endif
88
9340c38d
MD
89/* Sleep delay in ms */
90#define RCU_SLEEP_DELAY_MS 10
95b94246
MD
91#define INIT_NR_THREADS 8
92#define ARENA_INIT_ALLOC \
93 sizeof(struct registry_chunk) \
4477a870 94 + INIT_NR_THREADS * sizeof(struct urcu_bp_reader)
fdee2e6d 95
b7b6a8f5
PB
96/*
97 * Active attempts to check for reader Q.S. before calling sleep().
98 */
99#define RCU_QS_ACTIVE_ATTEMPTS 100
100
76d6a951 101static
4477a870 102int urcu_bp_refcount;
76d6a951 103
999991c6
MD
104/* If the headers do not support membarrier system call, fall back smp_mb. */
105#ifdef __NR_membarrier
106# define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__)
f541831e
MD
107#else
108# define membarrier(...) -ENOSYS
109#endif
110
111enum membarrier_cmd {
3745305b
MD
112 MEMBARRIER_CMD_QUERY = 0,
113 MEMBARRIER_CMD_SHARED = (1 << 0),
114 /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
115 /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
116 MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3),
117 MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4),
f541831e
MD
118};
119
c1be8fb9 120static
4477a870 121void __attribute__((constructor)) _urcu_bp_init(void);
c1be8fb9 122static
4477a870 123void __attribute__((destructor)) urcu_bp_exit(void);
fdee2e6d 124
d8d9a340 125#ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER
f541831e 126int urcu_bp_has_sys_membarrier;
d8d9a340 127#endif
f541831e 128
731ccb96
MD
129/*
130 * rcu_gp_lock ensures mutual exclusion between threads calling
131 * synchronize_rcu().
132 */
6abb4bd5 133static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
731ccb96
MD
134/*
135 * rcu_registry_lock ensures mutual exclusion between threads
136 * registering and unregistering themselves to/from the registry, and
137 * with threads reading that registry from synchronize_rcu(). However,
138 * this lock is not held all the way through the completion of awaiting
139 * for the grace period. It is sporadically released between iterations
140 * on the registry.
141 * rcu_registry_lock may nest inside rcu_gp_lock.
142 */
143static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
fdee2e6d 144
c1be8fb9
MD
145static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
146static int initialized;
147
148static pthread_key_t urcu_bp_key;
149
4477a870 150struct urcu_bp_gp urcu_bp_gp = { .ctr = URCU_BP_GP_COUNT };
fdee2e6d
MD
151
152/*
153 * Pointer to registry elements. Written to only by each individual reader. Read
154 * by both the reader and the writers.
155 */
4477a870 156DEFINE_URCU_TLS(struct urcu_bp_reader *, urcu_bp_reader);
fdee2e6d 157
16aa9ee8 158static CDS_LIST_HEAD(registry);
fdee2e6d 159
95b94246
MD
160struct registry_chunk {
161 size_t data_len; /* data length */
c1be8fb9 162 size_t used; /* amount of data used */
95b94246
MD
163 struct cds_list_head node; /* chunk_list node */
164 char data[];
165};
166
fdee2e6d 167struct registry_arena {
95b94246 168 struct cds_list_head chunk_list;
fdee2e6d
MD
169};
170
95b94246
MD
171static struct registry_arena registry_arena = {
172 .chunk_list = CDS_LIST_HEAD_INIT(registry_arena.chunk_list),
173};
fdee2e6d 174
4cf1675f
MD
175/* Saved fork signal mask, protected by rcu_gp_lock */
176static sigset_t saved_fork_signal_mask;
177
6abb4bd5 178static void mutex_lock(pthread_mutex_t *mutex)
fdee2e6d
MD
179{
180 int ret;
181
182#ifndef DISTRUST_SIGNALS_EXTREME
6abb4bd5 183 ret = pthread_mutex_lock(mutex);
4a6d7378
MD
184 if (ret)
185 urcu_die(ret);
fdee2e6d 186#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
6abb4bd5 187 while ((ret = pthread_mutex_trylock(mutex)) != 0) {
4a6d7378
MD
188 if (ret != EBUSY && ret != EINTR)
189 urcu_die(ret);
fdee2e6d
MD
190 poll(NULL,0,10);
191 }
192#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
193}
194
6abb4bd5 195static void mutex_unlock(pthread_mutex_t *mutex)
fdee2e6d
MD
196{
197 int ret;
198
6abb4bd5 199 ret = pthread_mutex_unlock(mutex);
4a6d7378
MD
200 if (ret)
201 urcu_die(ret);
fdee2e6d
MD
202}
203
f541831e
MD
204static void smp_mb_master(void)
205{
3745305b
MD
206 if (caa_likely(urcu_bp_has_sys_membarrier)) {
207 if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0))
208 urcu_die(errno);
209 } else {
f541831e 210 cmm_smp_mb();
3745305b 211 }
f541831e
MD
212}
213
731ccb96
MD
214/*
215 * Always called with rcu_registry lock held. Releases this lock between
216 * iterations and grabs it again. Holds the lock when it returns.
217 */
52c75091
MD
218static void wait_for_readers(struct cds_list_head *input_readers,
219 struct cds_list_head *cur_snap_readers,
220 struct cds_list_head *qsreaders)
fdee2e6d 221{
9340c38d 222 unsigned int wait_loops = 0;
4477a870 223 struct urcu_bp_reader *index, *tmp;
fdee2e6d 224
fdee2e6d 225 /*
4477a870 226 * Wait for each thread URCU_TLS(urcu_bp_reader).ctr to either
dd61d077 227 * indicate quiescence (not nested), or observe the current
c13c2e55 228 * rcu_gp.ctr value.
fdee2e6d
MD
229 */
230 for (;;) {
9340c38d
MD
231 if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
232 wait_loops++;
233
52c75091 234 cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
4477a870
MD
235 switch (urcu_bp_reader_state(&index->ctr)) {
236 case URCU_BP_READER_ACTIVE_CURRENT:
52c75091
MD
237 if (cur_snap_readers) {
238 cds_list_move(&index->node,
239 cur_snap_readers);
240 break;
241 }
242 /* Fall-through */
4477a870 243 case URCU_BP_READER_INACTIVE:
52c75091
MD
244 cds_list_move(&index->node, qsreaders);
245 break;
4477a870 246 case URCU_BP_READER_ACTIVE_OLD:
52c75091
MD
247 /*
248 * Old snapshot. Leaving node in
249 * input_readers will make us busy-loop
250 * until the snapshot becomes current or
251 * the reader becomes inactive.
252 */
253 break;
254 }
fdee2e6d
MD
255 }
256
52c75091 257 if (cds_list_empty(input_readers)) {
fdee2e6d
MD
258 break;
259 } else {
731ccb96
MD
260 /* Temporarily unlock the registry lock. */
261 mutex_unlock(&rcu_registry_lock);
9340c38d
MD
262 if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS)
263 (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS);
fdee2e6d 264 else
06f22bdb 265 caa_cpu_relax();
731ccb96
MD
266 /* Re-lock the registry lock before the next loop. */
267 mutex_lock(&rcu_registry_lock);
fdee2e6d
MD
268 }
269 }
fdee2e6d
MD
270}
271
4477a870 272void urcu_bp_synchronize_rcu(void)
fdee2e6d 273{
52c75091
MD
274 CDS_LIST_HEAD(cur_snap_readers);
275 CDS_LIST_HEAD(qsreaders);
fdee2e6d
MD
276 sigset_t newmask, oldmask;
277 int ret;
278
6ed4b2e6 279 ret = sigfillset(&newmask);
01477510 280 urcu_posix_assert(!ret);
6ed4b2e6 281 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
01477510 282 urcu_posix_assert(!ret);
fdee2e6d 283
6abb4bd5 284 mutex_lock(&rcu_gp_lock);
fdee2e6d 285
731ccb96
MD
286 mutex_lock(&rcu_registry_lock);
287
16aa9ee8 288 if (cds_list_empty(&registry))
2dfb8b5e 289 goto out;
fdee2e6d
MD
290
291 /* All threads should read qparity before accessing data structure
2dfb8b5e 292 * where new ptr points to. */
fdee2e6d 293 /* Write new ptr before changing the qparity */
f541831e 294 smp_mb_master();
fdee2e6d 295
fdee2e6d 296 /*
dd61d077 297 * Wait for readers to observe original parity or be quiescent.
731ccb96 298 * wait_for_readers() can release and grab again rcu_registry_lock
f99c6e92 299 * internally.
dd61d077 300 */
52c75091 301 wait_for_readers(&registry, &cur_snap_readers, &qsreaders);
dd61d077
MD
302
303 /*
304 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
305 * model easier to understand. It does not have a big performance impact
306 * anyway, given this is the write-side.
307 */
308 cmm_smp_mb();
309
310 /* Switch parity: 0 -> 1, 1 -> 0 */
4477a870 311 CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ URCU_BP_GP_CTR_PHASE);
dd61d077
MD
312
313 /*
314 * Must commit qparity update to memory before waiting for other parity
315 * quiescent state. Failure to do so could result in the writer waiting
316 * forever while new readers are always accessing data (no progress).
317 * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
fdee2e6d 318 */
fdee2e6d
MD
319
320 /*
5481ddb3 321 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
fdee2e6d
MD
322 * model easier to understand. It does not have a big performance impact
323 * anyway, given this is the write-side.
324 */
5481ddb3 325 cmm_smp_mb();
fdee2e6d 326
fdee2e6d 327 /*
dd61d077 328 * Wait for readers to observe new parity or be quiescent.
731ccb96 329 * wait_for_readers() can release and grab again rcu_registry_lock
f99c6e92 330 * internally.
fdee2e6d 331 */
52c75091
MD
332 wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
333
334 /*
335 * Put quiescent reader list back into registry.
336 */
337 cds_list_splice(&qsreaders, &registry);
fdee2e6d
MD
338
339 /*
2dfb8b5e
MD
340 * Finish waiting for reader threads before letting the old ptr being
341 * freed.
fdee2e6d 342 */
f541831e 343 smp_mb_master();
2dfb8b5e 344out:
731ccb96 345 mutex_unlock(&rcu_registry_lock);
6abb4bd5 346 mutex_unlock(&rcu_gp_lock);
fdee2e6d 347 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
01477510 348 urcu_posix_assert(!ret);
fdee2e6d
MD
349}
350
351/*
352 * library wrappers to be used by non-LGPL compatible source code.
353 */
354
4477a870 355void urcu_bp_read_lock(void)
fdee2e6d 356{
4477a870 357 _urcu_bp_read_lock();
fdee2e6d
MD
358}
359
4477a870 360void urcu_bp_read_unlock(void)
fdee2e6d 361{
4477a870 362 _urcu_bp_read_unlock();
fdee2e6d
MD
363}
364
4477a870 365int urcu_bp_read_ongoing(void)
882f3357 366{
4477a870 367 return _urcu_bp_read_ongoing();
882f3357
MD
368}
369
fdee2e6d 370/*
95b94246
MD
371 * Only grow for now. If empty, allocate a ARENA_INIT_ALLOC sized chunk.
372 * Else, try expanding the last chunk. If this fails, allocate a new
373 * chunk twice as big as the last chunk.
374 * Memory used by chunks _never_ moves. A chunk could theoretically be
375 * freed when all "used" slots are released, but we don't do it at this
376 * point.
fdee2e6d 377 */
95b94246
MD
378static
379void expand_arena(struct registry_arena *arena)
fdee2e6d 380{
95b94246
MD
381 struct registry_chunk *new_chunk, *last_chunk;
382 size_t old_chunk_len, new_chunk_len;
383
384 /* No chunk. */
385 if (cds_list_empty(&arena->chunk_list)) {
01477510 386 urcu_posix_assert(ARENA_INIT_ALLOC >=
95b94246
MD
387 sizeof(struct registry_chunk)
388 + sizeof(struct rcu_reader));
389 new_chunk_len = ARENA_INIT_ALLOC;
5592d049
MJ
390 new_chunk = (struct registry_chunk *) mmap(NULL,
391 new_chunk_len,
9d8612b7
MD
392 PROT_READ | PROT_WRITE,
393 MAP_ANONYMOUS | MAP_PRIVATE,
394 -1, 0);
95b94246
MD
395 if (new_chunk == MAP_FAILED)
396 abort();
d3ac5bb7 397 memset(new_chunk, 0, new_chunk_len);
95b94246
MD
398 new_chunk->data_len =
399 new_chunk_len - sizeof(struct registry_chunk);
400 cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
401 return; /* We're done. */
402 }
9d8612b7 403
95b94246
MD
404 /* Try expanding last chunk. */
405 last_chunk = cds_list_entry(arena->chunk_list.prev,
406 struct registry_chunk, node);
407 old_chunk_len =
408 last_chunk->data_len + sizeof(struct registry_chunk);
409 new_chunk_len = old_chunk_len << 1;
410
411 /* Don't allow memory mapping to move, just expand. */
412 new_chunk = mremap_wrapper(last_chunk, old_chunk_len,
413 new_chunk_len, 0);
414 if (new_chunk != MAP_FAILED) {
415 /* Should not have moved. */
01477510 416 urcu_posix_assert(new_chunk == last_chunk);
d3ac5bb7 417 memset((char *) last_chunk + old_chunk_len, 0,
95b94246
MD
418 new_chunk_len - old_chunk_len);
419 last_chunk->data_len =
420 new_chunk_len - sizeof(struct registry_chunk);
421 return; /* We're done. */
422 }
0617bf4c 423
95b94246 424 /* Remap did not succeed, we need to add a new chunk. */
5592d049
MJ
425 new_chunk = (struct registry_chunk *) mmap(NULL,
426 new_chunk_len,
95b94246
MD
427 PROT_READ | PROT_WRITE,
428 MAP_ANONYMOUS | MAP_PRIVATE,
429 -1, 0);
430 if (new_chunk == MAP_FAILED)
431 abort();
d3ac5bb7 432 memset(new_chunk, 0, new_chunk_len);
95b94246
MD
433 new_chunk->data_len =
434 new_chunk_len - sizeof(struct registry_chunk);
435 cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
436}
fdee2e6d 437
95b94246
MD
438static
439struct rcu_reader *arena_alloc(struct registry_arena *arena)
440{
441 struct registry_chunk *chunk;
442 struct rcu_reader *rcu_reader_reg;
443 int expand_done = 0; /* Only allow to expand once per alloc */
444 size_t len = sizeof(struct rcu_reader);
445
446retry:
447 cds_list_for_each_entry(chunk, &arena->chunk_list, node) {
448 if (chunk->data_len - chunk->used < len)
449 continue;
450 /* Find spot */
451 for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
452 rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
453 rcu_reader_reg++) {
454 if (!rcu_reader_reg->alloc) {
455 rcu_reader_reg->alloc = 1;
456 chunk->used += len;
457 return rcu_reader_reg;
458 }
459 }
460 }
461
462 if (!expand_done) {
463 expand_arena(arena);
464 expand_done = 1;
465 goto retry;
466 }
467
468 return NULL;
fdee2e6d
MD
469}
470
471/* Called with signals off and mutex locked */
95b94246
MD
472static
473void add_thread(void)
fdee2e6d 474{
02be5561 475 struct rcu_reader *rcu_reader_reg;
c1be8fb9 476 int ret;
fdee2e6d 477
95b94246
MD
478 rcu_reader_reg = arena_alloc(&registry_arena);
479 if (!rcu_reader_reg)
480 abort();
c1be8fb9
MD
481 ret = pthread_setspecific(urcu_bp_key, rcu_reader_reg);
482 if (ret)
483 abort();
fdee2e6d
MD
484
485 /* Add to registry */
02be5561 486 rcu_reader_reg->tid = pthread_self();
01477510 487 urcu_posix_assert(rcu_reader_reg->ctr == 0);
16aa9ee8 488 cds_list_add(&rcu_reader_reg->node, &registry);
95b94246
MD
489 /*
490 * Reader threads are pointing to the reader registry. This is
491 * why its memory should never be relocated.
492 */
4477a870 493 URCU_TLS(urcu_bp_reader) = rcu_reader_reg;
fdee2e6d
MD
494}
495
c1be8fb9
MD
496/* Called with mutex locked */
497static
498void cleanup_thread(struct registry_chunk *chunk,
499 struct rcu_reader *rcu_reader_reg)
500{
501 rcu_reader_reg->ctr = 0;
502 cds_list_del(&rcu_reader_reg->node);
503 rcu_reader_reg->tid = 0;
504 rcu_reader_reg->alloc = 0;
505 chunk->used -= sizeof(struct rcu_reader);
506}
507
508static
509struct registry_chunk *find_chunk(struct rcu_reader *rcu_reader_reg)
fdee2e6d 510{
95b94246 511 struct registry_chunk *chunk;
fdee2e6d 512
95b94246 513 cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
c1be8fb9
MD
514 if (rcu_reader_reg < (struct rcu_reader *) &chunk->data[0])
515 continue;
516 if (rcu_reader_reg >= (struct rcu_reader *) &chunk->data[chunk->data_len])
517 continue;
518 return chunk;
519 }
520 return NULL;
521}
95b94246 522
c1be8fb9
MD
523/* Called with signals off and mutex locked */
524static
76d6a951 525void remove_thread(struct rcu_reader *rcu_reader_reg)
c1be8fb9 526{
c1be8fb9 527 cleanup_thread(find_chunk(rcu_reader_reg), rcu_reader_reg);
4477a870 528 URCU_TLS(urcu_bp_reader) = NULL;
fdee2e6d
MD
529}
530
531/* Disable signals, take mutex, add to registry */
4477a870 532void urcu_bp_register(void)
fdee2e6d
MD
533{
534 sigset_t newmask, oldmask;
535 int ret;
536
6ed4b2e6 537 ret = sigfillset(&newmask);
c1be8fb9
MD
538 if (ret)
539 abort();
6ed4b2e6 540 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
c1be8fb9
MD
541 if (ret)
542 abort();
fdee2e6d
MD
543
544 /*
545 * Check if a signal concurrently registered our thread since
c1be8fb9
MD
546 * the check in rcu_read_lock().
547 */
4477a870 548 if (URCU_TLS(urcu_bp_reader))
fdee2e6d
MD
549 goto end;
550
c1be8fb9
MD
551 /*
552 * Take care of early registration before urcu_bp constructor.
553 */
4477a870 554 _urcu_bp_init();
c1be8fb9 555
731ccb96 556 mutex_lock(&rcu_registry_lock);
fdee2e6d 557 add_thread();
731ccb96 558 mutex_unlock(&rcu_registry_lock);
fdee2e6d
MD
559end:
560 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
c1be8fb9
MD
561 if (ret)
562 abort();
563}
564
5b46e39d
MD
565void urcu_bp_register_thread(void)
566{
567 if (caa_unlikely(!URCU_TLS(urcu_bp_reader)))
568 urcu_bp_register(); /* If not yet registered. */
569}
570
c1be8fb9
MD
571/* Disable signals, take mutex, remove from registry */
572static
4477a870 573void urcu_bp_unregister(struct rcu_reader *rcu_reader_reg)
c1be8fb9
MD
574{
575 sigset_t newmask, oldmask;
576 int ret;
577
578 ret = sigfillset(&newmask);
579 if (ret)
580 abort();
581 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
582 if (ret)
583 abort();
584
731ccb96 585 mutex_lock(&rcu_registry_lock);
76d6a951 586 remove_thread(rcu_reader_reg);
731ccb96 587 mutex_unlock(&rcu_registry_lock);
c1be8fb9
MD
588 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
589 if (ret)
590 abort();
4477a870 591 urcu_bp_exit();
c1be8fb9
MD
592}
593
594/*
595 * Remove thread from the registry when it exits, and flag it as
596 * destroyed so garbage collection can take care of it.
597 */
598static
599void urcu_bp_thread_exit_notifier(void *rcu_key)
600{
4477a870 601 urcu_bp_unregister(rcu_key);
c1be8fb9
MD
602}
603
d8d9a340
MD
604#ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
605static
4477a870 606void urcu_bp_sys_membarrier_status(bool available)
d8d9a340
MD
607{
608 if (!available)
609 abort();
610}
611#else
612static
4477a870 613void urcu_bp_sys_membarrier_status(bool available)
d8d9a340 614{
3745305b
MD
615 if (!available)
616 return;
617 urcu_bp_has_sys_membarrier = 1;
d8d9a340
MD
618}
619#endif
620
3745305b 621static
4477a870 622void urcu_bp_sys_membarrier_init(void)
3745305b
MD
623{
624 bool available = false;
625 int mask;
626
627 mask = membarrier(MEMBARRIER_CMD_QUERY, 0);
628 if (mask >= 0) {
629 if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
630 if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0))
631 urcu_die(errno);
632 available = true;
633 }
634 }
4477a870 635 urcu_bp_sys_membarrier_status(available);
3745305b
MD
636}
637
c1be8fb9 638static
4477a870 639void _urcu_bp_init(void)
c1be8fb9
MD
640{
641 mutex_lock(&init_lock);
4477a870 642 if (!urcu_bp_refcount++) {
c1be8fb9
MD
643 int ret;
644
645 ret = pthread_key_create(&urcu_bp_key,
646 urcu_bp_thread_exit_notifier);
647 if (ret)
648 abort();
4477a870 649 urcu_bp_sys_membarrier_init();
c1be8fb9
MD
650 initialized = 1;
651 }
652 mutex_unlock(&init_lock);
fdee2e6d
MD
653}
654
c1be8fb9 655static
4477a870 656void urcu_bp_exit(void)
fdee2e6d 657{
76d6a951 658 mutex_lock(&init_lock);
4477a870 659 if (!--urcu_bp_refcount) {
76d6a951
MD
660 struct registry_chunk *chunk, *tmp;
661 int ret;
95b94246 662
76d6a951
MD
663 cds_list_for_each_entry_safe(chunk, tmp,
664 &registry_arena.chunk_list, node) {
5592d049 665 munmap((void *) chunk, chunk->data_len
76d6a951
MD
666 + sizeof(struct registry_chunk));
667 }
7937ae1c 668 CDS_INIT_LIST_HEAD(&registry_arena.chunk_list);
76d6a951
MD
669 ret = pthread_key_delete(urcu_bp_key);
670 if (ret)
671 abort();
95b94246 672 }
76d6a951 673 mutex_unlock(&init_lock);
fdee2e6d 674}
4cf1675f
MD
675
676/*
731ccb96
MD
677 * Holding the rcu_gp_lock and rcu_registry_lock across fork will make
678 * sure we fork() don't race with a concurrent thread executing with
679 * any of those locks held. This ensures that the registry and data
680 * protected by rcu_gp_lock are in a coherent state in the child.
4cf1675f 681 */
4477a870 682void urcu_bp_before_fork(void)
4cf1675f
MD
683{
684 sigset_t newmask, oldmask;
685 int ret;
686
6ed4b2e6 687 ret = sigfillset(&newmask);
01477510 688 urcu_posix_assert(!ret);
6ed4b2e6 689 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
01477510 690 urcu_posix_assert(!ret);
4cf1675f 691 mutex_lock(&rcu_gp_lock);
731ccb96 692 mutex_lock(&rcu_registry_lock);
4cf1675f
MD
693 saved_fork_signal_mask = oldmask;
694}
695
4477a870 696void urcu_bp_after_fork_parent(void)
4cf1675f
MD
697{
698 sigset_t oldmask;
699 int ret;
700
701 oldmask = saved_fork_signal_mask;
731ccb96 702 mutex_unlock(&rcu_registry_lock);
4cf1675f
MD
703 mutex_unlock(&rcu_gp_lock);
704 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
01477510 705 urcu_posix_assert(!ret);
4cf1675f
MD
706}
707
c1be8fb9
MD
708/*
709 * Prune all entries from registry except our own thread. Fits the Linux
731ccb96 710 * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held.
c1be8fb9
MD
711 */
712static
713void urcu_bp_prune_registry(void)
714{
715 struct registry_chunk *chunk;
4477a870 716 struct urcu_bp_reader *rcu_reader_reg;
c1be8fb9
MD
717
718 cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
4477a870
MD
719 for (rcu_reader_reg = (struct urcu_bp_reader *) &chunk->data[0];
720 rcu_reader_reg < (struct urcu_bp_reader *) &chunk->data[chunk->data_len];
c1be8fb9
MD
721 rcu_reader_reg++) {
722 if (!rcu_reader_reg->alloc)
723 continue;
724 if (rcu_reader_reg->tid == pthread_self())
725 continue;
726 cleanup_thread(chunk, rcu_reader_reg);
727 }
728 }
729}
730
4477a870 731void urcu_bp_after_fork_child(void)
4cf1675f
MD
732{
733 sigset_t oldmask;
734 int ret;
735
c1be8fb9 736 urcu_bp_prune_registry();
4cf1675f 737 oldmask = saved_fork_signal_mask;
731ccb96 738 mutex_unlock(&rcu_registry_lock);
4cf1675f
MD
739 mutex_unlock(&rcu_gp_lock);
740 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
01477510 741 urcu_posix_assert(!ret);
4cf1675f 742}
5e77fc1f 743
4477a870 744void *urcu_bp_dereference_sym(void *p)
9b7981bb
MD
745{
746 return _rcu_dereference(p);
747}
748
4477a870 749void *urcu_bp_set_pointer_sym(void **p, void *v)
5efd3cd2
MD
750{
751 cmm_wmb();
424d4ed5
MD
752 uatomic_set(p, v);
753 return v;
5efd3cd2
MD
754}
755
4477a870 756void *urcu_bp_xchg_pointer_sym(void **p, void *v)
5efd3cd2
MD
757{
758 cmm_wmb();
759 return uatomic_xchg(p, v);
760}
761
4477a870 762void *urcu_bp_cmpxchg_pointer_sym(void **p, void *old, void *_new)
5efd3cd2
MD
763{
764 cmm_wmb();
765 return uatomic_cmpxchg(p, old, _new);
766}
767
5e6b23a6 768DEFINE_RCU_FLAVOR(rcu_flavor);
541d828d 769
5e77fc1f 770#include "urcu-call-rcu-impl.h"
0376e7b2 771#include "urcu-defer-impl.h"
This page took 0.082854 seconds and 4 git commands to generate.