[urcu.git] / urcu-bp.c

/*
 * urcu-bp.c
 *
 * Userspace RCU library, "bulletproof" version.
 *
 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
 */

#define _GNU_SOURCE
#define _LGPL_SOURCE
#include <stdio.h>
#include <pthread.h>
#include <signal.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <poll.h>
#include <unistd.h>
#include <sys/mman.h>

#include "urcu/wfqueue.h"
#include "urcu/map/urcu-bp.h"
#include "urcu/static/urcu-bp.h"

/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#undef _LGPL_SOURCE
#include "urcu-bp.h"
#define _LGPL_SOURCE

#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif

#ifndef __linux__

#define MREMAP_MAYMOVE	1
#define MREMAP_FIXED	2

/*
 * mremap wrapper for non-Linux systems. Maps a RW, anonymous private mapping.
 * This is not generic.
*/
void *mremap(void *old_address, size_t old_size, size_t new_size, int flags)
{
	void *new_address;

	assert(flags & MREMAP_MAYMOVE);
	assert(!(flags & MREMAP_FIXED));
	new_address = mmap(old_address, new_size,
			   PROT_READ | PROT_WRITE,
			   MAP_ANONYMOUS | MAP_PRIVATE,
			   -1, 0);
	if (new_address == MAP_FAILED)
		return MAP_FAILED;
	if (old_address) {
		memcpy(new_address, old_address, old_size);
		munmap(old_address, old_size);
	}
	return new_address;
}
#endif

/* Sleep delay in us */
#define RCU_SLEEP_DELAY		1000
#define ARENA_INIT_ALLOC	16

/*
 * Active attempts to check for reader Q.S. before calling sleep().
 */
#define RCU_QS_ACTIVE_ATTEMPTS 100

void __attribute__((destructor)) rcu_bp_exit(void);

static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;

#ifdef DEBUG_YIELD
unsigned int yield_active;
unsigned int __thread rand_yield;
#endif

/*
 * Global grace period counter.
 * Contains the current RCU_GP_CTR_PHASE.
 * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
 * Written to only by writer with mutex taken. Read by both writer and readers.
 */
long rcu_gp_ctr = RCU_GP_COUNT;

/*
 * Pointer to registry elements. Written to only by each individual reader. Read
 * by both the reader and the writers.
 */
struct rcu_reader __thread *rcu_reader;

static CDS_LIST_HEAD(registry);

struct registry_arena {
	void *p;
	size_t len;
	size_t used;
};

static struct registry_arena registry_arena;

/* Saved fork signal mask, protected by rcu_gp_lock */
static sigset_t saved_fork_signal_mask;

static void rcu_gc_registry(void);

static void mutex_lock(pthread_mutex_t *mutex)
{
	int ret;

#ifndef DISTRUST_SIGNALS_EXTREME
	ret = pthread_mutex_lock(mutex);
	if (ret) {
		perror("Error in pthread mutex lock");
		exit(-1);
	}
#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
	while ((ret = pthread_mutex_trylock(mutex)) != 0) {
		if (ret != EBUSY && ret != EINTR) {
			printf("ret = %d, errno = %d\n", ret, errno);
			perror("Error in pthread mutex lock");
			exit(-1);
		}
		poll(NULL,0,10);
	}
#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
}

static void mutex_unlock(pthread_mutex_t *mutex)
{
	int ret;

	ret = pthread_mutex_unlock(mutex);
	if (ret) {
		perror("Error in pthread mutex unlock");
		exit(-1);
	}
}

void update_counter_and_wait(void)
{
	CDS_LIST_HEAD(qsreaders);
	int wait_loops = 0;
	struct rcu_reader *index, *tmp;

	/* Switch parity: 0 -> 1, 1 -> 0 */
	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);

	/*
	 * Must commit qparity update to memory before waiting for other parity
	 * quiescent state. Failure to do so could result in the writer waiting
	 * forever while new readers are always accessing data (no progress).
	 * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
	 */

	/*
	 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	 * model easier to understand. It does not have a big performance impact
	 * anyway, given this is the write-side.
	 */
	cmm_smp_mb();

	/*
	 * Wait for each thread rcu_reader.ctr count to become 0.
	 */
	for (;;) {
		wait_loops++;
		cds_list_for_each_entry_safe(index, tmp, &registry, node) {
			if (!rcu_old_gp_ongoing(&index->ctr))
				cds_list_move(&index->node, &qsreaders);
		}

		if (cds_list_empty(&registry)) {
			break;
		} else {
			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
				usleep(RCU_SLEEP_DELAY);
			else
				caa_cpu_relax();
		}
	}
	/* put back the reader list in the registry */
	cds_list_splice(&qsreaders, &registry);
}

void synchronize_rcu(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigemptyset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	assert(!ret);

	mutex_lock(&rcu_gp_lock);

	if (cds_list_empty(&registry))
		goto out;

	/* All threads should read qparity before accessing data structure
	 * where new ptr points to. */
	/* Write new ptr before changing the qparity */
	cmm_smp_mb();

	/* Remove old registry elements */
	rcu_gc_registry();

	/*
	 * Wait for previous parity to be empty of readers.
	 */
	update_counter_and_wait();	/* 0 -> 1, wait readers in parity 0 */

	/*
	 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	 * model easier to understand. It does not have a big performance impact
	 * anyway, given this is the write-side.
	 */
	cmm_smp_mb();

	/*
	 * Wait for previous parity to be empty of readers.
	 */
	update_counter_and_wait();	/* 1 -> 0, wait readers in parity 1 */

	/*
	 * Finish waiting for reader threads before letting the old ptr being
	 * freed.
	 */
	cmm_smp_mb();
out:
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

/*
 * library wrappers to be used by non-LGPL compatible source code.
 */

void rcu_read_lock(void)
{
	_rcu_read_lock();
}

void rcu_read_unlock(void)
{
	_rcu_read_unlock();
}

/*
 * only grow for now.
 */
static void resize_arena(struct registry_arena *arena, size_t len)
{
	void *new_arena;

	if (!arena->p)
		new_arena = mmap(arena->p, len,
				 PROT_READ | PROT_WRITE,
				 MAP_ANONYMOUS | MAP_PRIVATE,
				 -1, 0);
	else
		new_arena = mremap(arena->p, arena->len,
				   len, MREMAP_MAYMOVE);
	assert(new_arena != MAP_FAILED);

	/*
	 * re-used the same region ?
	 */
	if (new_arena == arena->p)
		return;

	bzero(new_arena + arena->len, len - arena->len);
	arena->p = new_arena;
}

/* Called with signals off and mutex locked */
static void add_thread(void)
{
	struct rcu_reader *rcu_reader_reg;

	if (registry_arena.len
	    < registry_arena.used + sizeof(struct rcu_reader))
		resize_arena(&registry_arena,
		caa_max(registry_arena.len << 1, ARENA_INIT_ALLOC));
	/*
	 * Find a free spot.
	 */
	for (rcu_reader_reg = registry_arena.p;
	     (void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	     rcu_reader_reg++) {
		if (!rcu_reader_reg->alloc)
			break;
	}
	rcu_reader_reg->alloc = 1;
	registry_arena.used += sizeof(struct rcu_reader);

	/* Add to registry */
	rcu_reader_reg->tid = pthread_self();
	assert(rcu_reader_reg->ctr == 0);
	cds_list_add(&rcu_reader_reg->node, &registry);
	rcu_reader = rcu_reader_reg;
}

/* Called with signals off and mutex locked */
static void rcu_gc_registry(void)
{
	struct rcu_reader *rcu_reader_reg;
	pthread_t tid;
	int ret;

	for (rcu_reader_reg = registry_arena.p;
	     (void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	     rcu_reader_reg++) {
		if (!rcu_reader_reg->alloc)
			continue;
		tid = rcu_reader_reg->tid;
		ret = pthread_kill(tid, 0);
		assert(ret != EINVAL);
		if (ret == ESRCH) {
			cds_list_del(&rcu_reader_reg->node);
			rcu_reader_reg->ctr = 0;
			rcu_reader_reg->alloc = 0;
			registry_arena.used -= sizeof(struct rcu_reader);
		}
	}
}

/* Disable signals, take mutex, add to registry */
void rcu_bp_register(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigemptyset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	assert(!ret);

	/*
	 * Check if a signal concurrently registered our thread since
	 * the check in rcu_read_lock(). */
	if (rcu_reader)
		goto end;

	mutex_lock(&rcu_gp_lock);
	add_thread();
	mutex_unlock(&rcu_gp_lock);
end:
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

void rcu_bp_exit(void)
{
	if (registry_arena.p)
		munmap(registry_arena.p, registry_arena.len);
}

/*
 * Holding the rcu_gp_lock across fork will make sure we fork() don't race with
 * a concurrent thread executing with this same lock held. This ensures that the
 * registry is in a coherent state in the child.
 */
void rcu_bp_before_fork(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigemptyset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	assert(!ret);
	mutex_lock(&rcu_gp_lock);
	saved_fork_signal_mask = oldmask;
}

void rcu_bp_after_fork_parent(void)
{
	sigset_t oldmask;
	int ret;

	oldmask = saved_fork_signal_mask;
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

void rcu_bp_after_fork_child(void)
{
	sigset_t oldmask;
	int ret;

	rcu_gc_registry();
	oldmask = saved_fork_signal_mask;
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

#include "urcu-call-rcu-impl.h"
#include "urcu-defer-impl.h"
Commit	Line	Data
	1	/*
	2	* urcu-bp.c
	3	*
	4	* Userspace RCU library, "bulletproof" version.
	5	*
	6	* Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	7	* Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
	8	*
	9	* This library is free software; you can redistribute it and/or
	10	* modify it under the terms of the GNU Lesser General Public
	11	* License as published by the Free Software Foundation; either
	12	* version 2.1 of the License, or (at your option) any later version.
	13	*
	14	* This library is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	17	* Lesser General Public License for more details.
	18	*
	19	* You should have received a copy of the GNU Lesser General Public
	20	* License along with this library; if not, write to the Free Software
	21	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	22	*
	23	* IBM's contributions to this file may be relicensed under LGPLv2 or later.
	24	*/
	25
	26	#define _GNU_SOURCE
	27	#define _LGPL_SOURCE
	28	#include <stdio.h>
	29	#include <pthread.h>
	30	#include <signal.h>
	31	#include <assert.h>
	32	#include <stdlib.h>
	33	#include <string.h>
	34	#include <errno.h>
	35	#include <poll.h>
	36	#include <unistd.h>
	37	#include <sys/mman.h>
	38
	39	#include "urcu/wfqueue.h"
	40	#include "urcu/map/urcu-bp.h"
	41	#include "urcu/static/urcu-bp.h"
	42
	43	/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
	44	#undef _LGPL_SOURCE
	45	#include "urcu-bp.h"
	46	#define _LGPL_SOURCE
	47
	48	#ifndef MAP_ANONYMOUS
	49	#define MAP_ANONYMOUS MAP_ANON
	50	#endif
	51
	52	#ifndef __linux__
	53
	54	#define MREMAP_MAYMOVE 1
	55	#define MREMAP_FIXED 2
	56
	57	/*
	58	* mremap wrapper for non-Linux systems. Maps a RW, anonymous private mapping.
	59	* This is not generic.
	60	*/
	61	void mremap(void old_address, size_t old_size, size_t new_size, int flags)
	62	{
	63	void *new_address;
	64
	65	assert(flags & MREMAP_MAYMOVE);
	66	assert(!(flags & MREMAP_FIXED));
	67	new_address = mmap(old_address, new_size,
	68	PROT_READ \| PROT_WRITE,
	69	MAP_ANONYMOUS \| MAP_PRIVATE,
	70	-1, 0);
	71	if (new_address == MAP_FAILED)
	72	return MAP_FAILED;
	73	if (old_address) {
	74	memcpy(new_address, old_address, old_size);
	75	munmap(old_address, old_size);
	76	}
	77	return new_address;
	78	}
	79	#endif
	80
	81	/* Sleep delay in us */
	82	#define RCU_SLEEP_DELAY 1000
	83	#define ARENA_INIT_ALLOC 16
	84
	85	/*
	86	* Active attempts to check for reader Q.S. before calling sleep().
	87	*/
	88	#define RCU_QS_ACTIVE_ATTEMPTS 100
	89
	90	void __attribute__((destructor)) rcu_bp_exit(void);
	91
	92	static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
	93
	94	#ifdef DEBUG_YIELD
	95	unsigned int yield_active;
	96	unsigned int __thread rand_yield;
	97	#endif
	98
	99	/*
	100	* Global grace period counter.
	101	* Contains the current RCU_GP_CTR_PHASE.
	102	* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
	103	* Written to only by writer with mutex taken. Read by both writer and readers.
	104	*/
	105	long rcu_gp_ctr = RCU_GP_COUNT;
	106
	107	/*
	108	* Pointer to registry elements. Written to only by each individual reader. Read
	109	* by both the reader and the writers.
	110	*/
	111	struct rcu_reader __thread *rcu_reader;
	112
	113	static CDS_LIST_HEAD(registry);
	114
	115	struct registry_arena {
	116	void *p;
	117	size_t len;
	118	size_t used;
	119	};
	120
	121	static struct registry_arena registry_arena;
	122
	123	/* Saved fork signal mask, protected by rcu_gp_lock */
	124	static sigset_t saved_fork_signal_mask;
	125
	126	static void rcu_gc_registry(void);
	127
	128	static void mutex_lock(pthread_mutex_t *mutex)
	129	{
	130	int ret;
	131
	132	#ifndef DISTRUST_SIGNALS_EXTREME
	133	ret = pthread_mutex_lock(mutex);
	134	if (ret) {
	135	perror("Error in pthread mutex lock");
	136	exit(-1);
	137	}
	138	#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
	139	while ((ret = pthread_mutex_trylock(mutex)) != 0) {
	140	if (ret != EBUSY && ret != EINTR) {
	141	printf("ret = %d, errno = %d\n", ret, errno);
	142	perror("Error in pthread mutex lock");
	143	exit(-1);
	144	}
	145	poll(NULL,0,10);
	146	}
	147	#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
	148	}
	149
	150	static void mutex_unlock(pthread_mutex_t *mutex)
	151	{
	152	int ret;
	153
	154	ret = pthread_mutex_unlock(mutex);
	155	if (ret) {
	156	perror("Error in pthread mutex unlock");
	157	exit(-1);
	158	}
	159	}
	160
	161	void update_counter_and_wait(void)
	162	{
	163	CDS_LIST_HEAD(qsreaders);
	164	int wait_loops = 0;
	165	struct rcu_reader index, tmp;
	166
	167	/* Switch parity: 0 -> 1, 1 -> 0 */
	168	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
	169
	170	/*
	171	* Must commit qparity update to memory before waiting for other parity
	172	* quiescent state. Failure to do so could result in the writer waiting
	173	* forever while new readers are always accessing data (no progress).
	174	* Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
	175	*/
	176
	177	/*
	178	* Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	179	* model easier to understand. It does not have a big performance impact
	180	* anyway, given this is the write-side.
	181	*/
	182	cmm_smp_mb();
	183
	184	/*
	185	* Wait for each thread rcu_reader.ctr count to become 0.
	186	*/
	187	for (;;) {
	188	wait_loops++;
	189	cds_list_for_each_entry_safe(index, tmp, &registry, node) {
	190	if (!rcu_old_gp_ongoing(&index->ctr))
	191	cds_list_move(&index->node, &qsreaders);
	192	}
	193
	194	if (cds_list_empty(&registry)) {
	195	break;
	196	} else {
	197	if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
	198	usleep(RCU_SLEEP_DELAY);
	199	else
	200	caa_cpu_relax();
	201	}
	202	}
	203	/* put back the reader list in the registry */
	204	cds_list_splice(&qsreaders, &registry);
	205	}
	206
	207	void synchronize_rcu(void)
	208	{
	209	sigset_t newmask, oldmask;
	210	int ret;
	211
	212	ret = sigemptyset(&newmask);
	213	assert(!ret);
	214	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	215	assert(!ret);
	216
	217	mutex_lock(&rcu_gp_lock);
	218
	219	if (cds_list_empty(&registry))
	220	goto out;
	221
	222	/* All threads should read qparity before accessing data structure
	223	* where new ptr points to. */
	224	/* Write new ptr before changing the qparity */
	225	cmm_smp_mb();
	226
	227	/* Remove old registry elements */
	228	rcu_gc_registry();
	229
	230	/*
	231	* Wait for previous parity to be empty of readers.
	232	*/
	233	update_counter_and_wait(); /* 0 -> 1, wait readers in parity 0 */
	234
	235	/*
	236	* Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	237	* model easier to understand. It does not have a big performance impact
	238	* anyway, given this is the write-side.
	239	*/
	240	cmm_smp_mb();
	241
	242	/*
	243	* Wait for previous parity to be empty of readers.
	244	*/
	245	update_counter_and_wait(); /* 1 -> 0, wait readers in parity 1 */
	246
	247	/*
	248	* Finish waiting for reader threads before letting the old ptr being
	249	* freed.
	250	*/
	251	cmm_smp_mb();
	252	out:
	253	mutex_unlock(&rcu_gp_lock);
	254	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	255	assert(!ret);
	256	}
	257
	258	/*
	259	* library wrappers to be used by non-LGPL compatible source code.
	260	*/
	261
	262	void rcu_read_lock(void)
	263	{
	264	_rcu_read_lock();
	265	}
	266
	267	void rcu_read_unlock(void)
	268	{
	269	_rcu_read_unlock();
	270	}
	271
	272	/*
	273	* only grow for now.
	274	*/
	275	static void resize_arena(struct registry_arena *arena, size_t len)
	276	{
	277	void *new_arena;
	278
	279	if (!arena->p)
	280	new_arena = mmap(arena->p, len,
	281	PROT_READ \| PROT_WRITE,
	282	MAP_ANONYMOUS \| MAP_PRIVATE,
	283	-1, 0);
	284	else
	285	new_arena = mremap(arena->p, arena->len,
	286	len, MREMAP_MAYMOVE);
	287	assert(new_arena != MAP_FAILED);
	288
	289	/*
	290	* re-used the same region ?
	291	*/
	292	if (new_arena == arena->p)
	293	return;
	294
	295	bzero(new_arena + arena->len, len - arena->len);
	296	arena->p = new_arena;
	297	}
	298
	299	/* Called with signals off and mutex locked */
	300	static void add_thread(void)
	301	{
	302	struct rcu_reader *rcu_reader_reg;
	303
	304	if (registry_arena.len
	305	< registry_arena.used + sizeof(struct rcu_reader))
	306	resize_arena(&registry_arena,
	307	caa_max(registry_arena.len << 1, ARENA_INIT_ALLOC));
	308	/*
	309	* Find a free spot.
	310	*/
	311	for (rcu_reader_reg = registry_arena.p;
	312	(void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	313	rcu_reader_reg++) {
	314	if (!rcu_reader_reg->alloc)
	315	break;
	316	}
	317	rcu_reader_reg->alloc = 1;
	318	registry_arena.used += sizeof(struct rcu_reader);
	319
	320	/* Add to registry */
	321	rcu_reader_reg->tid = pthread_self();
	322	assert(rcu_reader_reg->ctr == 0);
	323	cds_list_add(&rcu_reader_reg->node, &registry);
	324	rcu_reader = rcu_reader_reg;
	325	}
	326
	327	/* Called with signals off and mutex locked */
	328	static void rcu_gc_registry(void)
	329	{
	330	struct rcu_reader *rcu_reader_reg;
	331	pthread_t tid;
	332	int ret;
	333
	334	for (rcu_reader_reg = registry_arena.p;
	335	(void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	336	rcu_reader_reg++) {
	337	if (!rcu_reader_reg->alloc)
	338	continue;
	339	tid = rcu_reader_reg->tid;
	340	ret = pthread_kill(tid, 0);
	341	assert(ret != EINVAL);
	342	if (ret == ESRCH) {
	343	cds_list_del(&rcu_reader_reg->node);
	344	rcu_reader_reg->ctr = 0;
	345	rcu_reader_reg->alloc = 0;
	346	registry_arena.used -= sizeof(struct rcu_reader);
	347	}
	348	}
	349	}
	350
	351	/* Disable signals, take mutex, add to registry */
	352	void rcu_bp_register(void)
	353	{
	354	sigset_t newmask, oldmask;
	355	int ret;
	356
	357	ret = sigemptyset(&newmask);
	358	assert(!ret);
	359	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	360	assert(!ret);
	361
	362	/*
	363	* Check if a signal concurrently registered our thread since
	364	* the check in rcu_read_lock(). */
	365	if (rcu_reader)
	366	goto end;
	367
	368	mutex_lock(&rcu_gp_lock);
	369	add_thread();
	370	mutex_unlock(&rcu_gp_lock);
	371	end:
	372	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	373	assert(!ret);
	374	}
	375
	376	void rcu_bp_exit(void)
	377	{
	378	if (registry_arena.p)
	379	munmap(registry_arena.p, registry_arena.len);
	380	}
	381
	382	/*
	383	* Holding the rcu_gp_lock across fork will make sure we fork() don't race with
	384	* a concurrent thread executing with this same lock held. This ensures that the
	385	* registry is in a coherent state in the child.
	386	*/
	387	void rcu_bp_before_fork(void)
	388	{
	389	sigset_t newmask, oldmask;
	390	int ret;
	391
	392	ret = sigemptyset(&newmask);
	393	assert(!ret);
	394	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	395	assert(!ret);
	396	mutex_lock(&rcu_gp_lock);
	397	saved_fork_signal_mask = oldmask;
	398	}
	399
	400	void rcu_bp_after_fork_parent(void)
	401	{
	402	sigset_t oldmask;
	403	int ret;
	404
	405	oldmask = saved_fork_signal_mask;
	406	mutex_unlock(&rcu_gp_lock);
	407	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	408	assert(!ret);
	409	}
	410
	411	void rcu_bp_after_fork_child(void)
	412	{
	413	sigset_t oldmask;
	414	int ret;
	415
	416	rcu_gc_registry();
	417	oldmask = saved_fork_signal_mask;
	418	mutex_unlock(&rcu_gp_lock);
	419	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	420	assert(!ret);
	421	}
	422
	423	#include "urcu-call-rcu-impl.h"
	424	#include "urcu-defer-impl.h"