[urcu.git] / urcu-bp.c

/*
 * urcu-bp.c
 *
 * Userspace RCU library, "bulletproof" version.
 *
 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
 */

#define _GNU_SOURCE
#define _LGPL_SOURCE
#include <stdio.h>
#include <pthread.h>
#include <signal.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <poll.h>
#include <unistd.h>
#include <sys/mman.h>

#include "urcu/wfqueue.h"
#include "urcu/map/urcu-bp.h"
#include "urcu/static/urcu-bp.h"
#include "urcu-pointer.h"
#include "urcu/tls-compat.h"

/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#undef _LGPL_SOURCE
#include "urcu-bp.h"
#define _LGPL_SOURCE

#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif

#ifdef __linux__
static
void *mremap_wrapper(void *old_address, size_t old_size,
		size_t new_size, int flags)
{
	return mremap(old_address, old_size, new_size, flags);
}
#else

#define MREMAP_MAYMOVE	1
#define MREMAP_FIXED	2

/*
 * mremap wrapper for non-Linux systems. Maps a RW, anonymous private mapping.
 * This is not generic.
*/
static
void *mremap_wrapper(void *old_address, size_t old_size,
		size_t new_size, int flags)
{
	void *new_address;

	assert(flags & MREMAP_MAYMOVE);
	assert(!(flags & MREMAP_FIXED));
	new_address = mmap(old_address, new_size,
			   PROT_READ | PROT_WRITE,
			   MAP_ANONYMOUS | MAP_PRIVATE,
			   -1, 0);
	if (new_address == MAP_FAILED)
		return MAP_FAILED;
	if (old_address) {
		memcpy(new_address, old_address, old_size);
		munmap(old_address, old_size);
	}
	return new_address;
}
#endif

/* Sleep delay in us */
#define RCU_SLEEP_DELAY		1000
#define ARENA_INIT_ALLOC	16

/*
 * Active attempts to check for reader Q.S. before calling sleep().
 */
#define RCU_QS_ACTIVE_ATTEMPTS 100

void __attribute__((destructor)) rcu_bp_exit(void);

static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;

#ifdef DEBUG_YIELD
unsigned int yield_active;
DEFINE_URCU_TLS(unsigned int, rand_yield);
#endif

/*
 * Global grace period counter.
 * Contains the current RCU_GP_CTR_PHASE.
 * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
 * Written to only by writer with mutex taken. Read by both writer and readers.
 */
long rcu_gp_ctr = RCU_GP_COUNT;

/*
 * Pointer to registry elements. Written to only by each individual reader. Read
 * by both the reader and the writers.
 */
DEFINE_URCU_TLS(struct rcu_reader *, rcu_reader);

static CDS_LIST_HEAD(registry);

struct registry_arena {
	void *p;
	size_t len;
	size_t used;
};

static struct registry_arena registry_arena;

/* Saved fork signal mask, protected by rcu_gp_lock */
static sigset_t saved_fork_signal_mask;

static void rcu_gc_registry(void);

static void mutex_lock(pthread_mutex_t *mutex)
{
	int ret;

#ifndef DISTRUST_SIGNALS_EXTREME
	ret = pthread_mutex_lock(mutex);
	if (ret) {
		perror("Error in pthread mutex lock");
		exit(-1);
	}
#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
	while ((ret = pthread_mutex_trylock(mutex)) != 0) {
		if (ret != EBUSY && ret != EINTR) {
			printf("ret = %d, errno = %d\n", ret, errno);
			perror("Error in pthread mutex lock");
			exit(-1);
		}
		poll(NULL,0,10);
	}
#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
}

static void mutex_unlock(pthread_mutex_t *mutex)
{
	int ret;

	ret = pthread_mutex_unlock(mutex);
	if (ret) {
		perror("Error in pthread mutex unlock");
		exit(-1);
	}
}

void update_counter_and_wait(void)
{
	CDS_LIST_HEAD(qsreaders);
	int wait_loops = 0;
	struct rcu_reader *index, *tmp;

	/* Switch parity: 0 -> 1, 1 -> 0 */
	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);

	/*
	 * Must commit qparity update to memory before waiting for other parity
	 * quiescent state. Failure to do so could result in the writer waiting
	 * forever while new readers are always accessing data (no progress).
	 * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
	 */

	/*
	 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	 * model easier to understand. It does not have a big performance impact
	 * anyway, given this is the write-side.
	 */
	cmm_smp_mb();

	/*
	 * Wait for each thread rcu_reader.ctr count to become 0.
	 */
	for (;;) {
		wait_loops++;
		cds_list_for_each_entry_safe(index, tmp, &registry, node) {
			if (!rcu_old_gp_ongoing(&index->ctr))
				cds_list_move(&index->node, &qsreaders);
		}

		if (cds_list_empty(&registry)) {
			break;
		} else {
			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
				usleep(RCU_SLEEP_DELAY);
			else
				caa_cpu_relax();
		}
	}
	/* put back the reader list in the registry */
	cds_list_splice(&qsreaders, &registry);
}

void synchronize_rcu(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigemptyset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	assert(!ret);

	mutex_lock(&rcu_gp_lock);

	if (cds_list_empty(&registry))
		goto out;

	/* All threads should read qparity before accessing data structure
	 * where new ptr points to. */
	/* Write new ptr before changing the qparity */
	cmm_smp_mb();

	/* Remove old registry elements */
	rcu_gc_registry();

	/*
	 * Wait for previous parity to be empty of readers.
	 */
	update_counter_and_wait();	/* 0 -> 1, wait readers in parity 0 */

	/*
	 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	 * model easier to understand. It does not have a big performance impact
	 * anyway, given this is the write-side.
	 */
	cmm_smp_mb();

	/*
	 * Wait for previous parity to be empty of readers.
	 */
	update_counter_and_wait();	/* 1 -> 0, wait readers in parity 1 */

	/*
	 * Finish waiting for reader threads before letting the old ptr being
	 * freed.
	 */
	cmm_smp_mb();
out:
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

/*
 * library wrappers to be used by non-LGPL compatible source code.
 */

void rcu_read_lock(void)
{
	_rcu_read_lock();
}

void rcu_read_unlock(void)
{
	_rcu_read_unlock();
}

/*
 * only grow for now.
 */
static void resize_arena(struct registry_arena *arena, size_t len)
{
	void *new_arena;

	if (!arena->p)
		new_arena = mmap(arena->p, len,
				 PROT_READ | PROT_WRITE,
				 MAP_ANONYMOUS | MAP_PRIVATE,
				 -1, 0);
	else
		new_arena = mremap_wrapper(arena->p, arena->len,
				   	len, MREMAP_MAYMOVE);
	assert(new_arena != MAP_FAILED);

	/*
	 * re-used the same region ?
	 */
	if (new_arena == arena->p)
		return;

	bzero(new_arena + arena->len, len - arena->len);
	arena->p = new_arena;
}

/* Called with signals off and mutex locked */
static void add_thread(void)
{
	struct rcu_reader *rcu_reader_reg;

	if (registry_arena.len
	    < registry_arena.used + sizeof(struct rcu_reader))
		resize_arena(&registry_arena,
		caa_max(registry_arena.len << 1, ARENA_INIT_ALLOC));
	/*
	 * Find a free spot.
	 */
	for (rcu_reader_reg = registry_arena.p;
	     (void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	     rcu_reader_reg++) {
		if (!rcu_reader_reg->alloc)
			break;
	}
	rcu_reader_reg->alloc = 1;
	registry_arena.used += sizeof(struct rcu_reader);

	/* Add to registry */
	rcu_reader_reg->tid = pthread_self();
	assert(rcu_reader_reg->ctr == 0);
	cds_list_add(&rcu_reader_reg->node, &registry);
	URCU_TLS(rcu_reader) = rcu_reader_reg;
}

/* Called with signals off and mutex locked */
static void rcu_gc_registry(void)
{
	struct rcu_reader *rcu_reader_reg;
	pthread_t tid;
	int ret;

	for (rcu_reader_reg = registry_arena.p;
	     (void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	     rcu_reader_reg++) {
		if (!rcu_reader_reg->alloc)
			continue;
		tid = rcu_reader_reg->tid;
		ret = pthread_kill(tid, 0);
		assert(ret != EINVAL);
		if (ret == ESRCH) {
			cds_list_del(&rcu_reader_reg->node);
			rcu_reader_reg->ctr = 0;
			rcu_reader_reg->alloc = 0;
			registry_arena.used -= sizeof(struct rcu_reader);
		}
	}
}

/* Disable signals, take mutex, add to registry */
void rcu_bp_register(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigemptyset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	assert(!ret);

	/*
	 * Check if a signal concurrently registered our thread since
	 * the check in rcu_read_lock(). */
	if (URCU_TLS(rcu_reader))
		goto end;

	mutex_lock(&rcu_gp_lock);
	add_thread();
	mutex_unlock(&rcu_gp_lock);
end:
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

void rcu_bp_exit(void)
{
	if (registry_arena.p)
		munmap(registry_arena.p, registry_arena.len);
}

/*
 * Holding the rcu_gp_lock across fork will make sure we fork() don't race with
 * a concurrent thread executing with this same lock held. This ensures that the
 * registry is in a coherent state in the child.
 */
void rcu_bp_before_fork(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigemptyset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	assert(!ret);
	mutex_lock(&rcu_gp_lock);
	saved_fork_signal_mask = oldmask;
}

void rcu_bp_after_fork_parent(void)
{
	sigset_t oldmask;
	int ret;

	oldmask = saved_fork_signal_mask;
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

void rcu_bp_after_fork_child(void)
{
	sigset_t oldmask;
	int ret;

	rcu_gc_registry();
	oldmask = saved_fork_signal_mask;
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

void *rcu_dereference_sym_bp(void *p)
{
	return _rcu_dereference(p);
}

void *rcu_set_pointer_sym_bp(void **p, void *v)
{
	cmm_wmb();
	uatomic_set(p, v);
	return v;
}

void *rcu_xchg_pointer_sym_bp(void **p, void *v)
{
	cmm_wmb();
	return uatomic_xchg(p, v);
}

void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new)
{
	cmm_wmb();
	return uatomic_cmpxchg(p, old, _new);
}

DEFINE_RCU_FLAVOR(rcu_flavor);

#include "urcu-call-rcu-impl.h"
#include "urcu-defer-impl.h"
Commit	Line	Data
fdee2e6d MD	1	/*
	2	* urcu-bp.c
	3	*
	4	* Userspace RCU library, "bulletproof" version.
	5	*
6982d6d7	6	* Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
fdee2e6d MD	7	* Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
	8	*
	9	* This library is free software; you can redistribute it and/or
	10	* modify it under the terms of the GNU Lesser General Public
	11	* License as published by the Free Software Foundation; either
	12	* version 2.1 of the License, or (at your option) any later version.
	13	*
	14	* This library is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	17	* Lesser General Public License for more details.
	18	*
	19	* You should have received a copy of the GNU Lesser General Public
	20	* License along with this library; if not, write to the Free Software
	21	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	22	*
	23	* IBM's contributions to this file may be relicensed under LGPLv2 or later.
	24	*/
	25
0617bf4c	26	#define _GNU_SOURCE
71c811bf	27	#define _LGPL_SOURCE
fdee2e6d MD	28	#include <stdio.h>
	29	#include <pthread.h>
	30	#include <signal.h>
	31	#include <assert.h>
	32	#include <stdlib.h>
	33	#include <string.h>
	34	#include <errno.h>
	35	#include <poll.h>
	36	#include <unistd.h>
	37	#include <sys/mman.h>
	38
71c811bf	39	#include "urcu/wfqueue.h"
57760d44	40	#include "urcu/map/urcu-bp.h"
af7c2dbe	41	#include "urcu/static/urcu-bp.h"
618b2595	42	#include "urcu-pointer.h"
bd252a04	43	#include "urcu/tls-compat.h"
71c811bf	44
fdee2e6d	45	/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
71c811bf	46	#undef _LGPL_SOURCE
fdee2e6d	47	#include "urcu-bp.h"
71c811bf	48	#define _LGPL_SOURCE
fdee2e6d	49
4c1ae2ea MD	50	#ifndef MAP_ANONYMOUS
	51	#define MAP_ANONYMOUS MAP_ANON
	52	#endif
	53
c7eaf61c MD	54	#ifdef __linux__
	55	static
	56	void mremap_wrapper(void old_address, size_t old_size,
	57	size_t new_size, int flags)
	58	{
	59	return mremap(old_address, old_size, new_size, flags);
	60	}
	61	#else
45a4872f MD	62
	63	#define MREMAP_MAYMOVE 1
	64	#define MREMAP_FIXED 2
	65
	66	/*
	67	* mremap wrapper for non-Linux systems. Maps a RW, anonymous private mapping.
	68	* This is not generic.
	69	*/
c7eaf61c MD	70	static
	71	void mremap_wrapper(void old_address, size_t old_size,
	72	size_t new_size, int flags)
45a4872f	73	{
dc745ef6	74	void *new_address;
45a4872f MD	75
	76	assert(flags & MREMAP_MAYMOVE);
	77	assert(!(flags & MREMAP_FIXED));
	78	new_address = mmap(old_address, new_size,
	79	PROT_READ \| PROT_WRITE,
	80	MAP_ANONYMOUS \| MAP_PRIVATE,
	81	-1, 0);
	82	if (new_address == MAP_FAILED)
	83	return MAP_FAILED;
	84	if (old_address) {
	85	memcpy(new_address, old_address, old_size);
	86	munmap(old_address, old_size);
	87	}
	88	return new_address;
	89	}
	90	#endif
	91
fdee2e6d MD	92	/* Sleep delay in us */
	93	#define RCU_SLEEP_DELAY 1000
	94	#define ARENA_INIT_ALLOC 16
	95
b7b6a8f5 PB	96	/*
	97	* Active attempts to check for reader Q.S. before calling sleep().
	98	*/
	99	#define RCU_QS_ACTIVE_ATTEMPTS 100
	100
02be5561	101	void __attribute__((destructor)) rcu_bp_exit(void);
fdee2e6d	102
6abb4bd5	103	static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
fdee2e6d MD	104
	105	#ifdef DEBUG_YIELD
	106	unsigned int yield_active;
bd252a04	107	DEFINE_URCU_TLS(unsigned int, rand_yield);
fdee2e6d MD	108	#endif
	109
	110	/*
	111	* Global grace period counter.
02be5561	112	* Contains the current RCU_GP_CTR_PHASE.
fdee2e6d MD	113	* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
	114	* Written to only by writer with mutex taken. Read by both writer and readers.
	115	*/
02be5561	116	long rcu_gp_ctr = RCU_GP_COUNT;
fdee2e6d MD	117
	118	/*
	119	* Pointer to registry elements. Written to only by each individual reader. Read
	120	* by both the reader and the writers.
	121	*/
bd252a04	122	DEFINE_URCU_TLS(struct rcu_reader *, rcu_reader);
fdee2e6d	123
16aa9ee8	124	static CDS_LIST_HEAD(registry);
fdee2e6d MD	125
	126	struct registry_arena {
	127	void *p;
	128	size_t len;
	129	size_t used;
	130	};
	131
	132	static struct registry_arena registry_arena;
	133
4cf1675f MD	134	/* Saved fork signal mask, protected by rcu_gp_lock */
	135	static sigset_t saved_fork_signal_mask;
	136
fdee2e6d MD	137	static void rcu_gc_registry(void);
fdee2e6d MD	138
6abb4bd5	139	static void mutex_lock(pthread_mutex_t *mutex)
fdee2e6d MD	140	{
	141	int ret;
	142
	143	#ifndef DISTRUST_SIGNALS_EXTREME
6abb4bd5	144	ret = pthread_mutex_lock(mutex);
fdee2e6d MD	145	if (ret) {
	146	perror("Error in pthread mutex lock");
	147	exit(-1);
	148	}
	149	#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
6abb4bd5	150	while ((ret = pthread_mutex_trylock(mutex)) != 0) {
fdee2e6d MD	151	if (ret != EBUSY && ret != EINTR) {
	152	printf("ret = %d, errno = %d\n", ret, errno);
	153	perror("Error in pthread mutex lock");
	154	exit(-1);
	155	}
fdee2e6d MD	156	poll(NULL,0,10);
	157	}
	158	#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
	159	}
	160
6abb4bd5	161	static void mutex_unlock(pthread_mutex_t *mutex)
fdee2e6d MD	162	{
	163	int ret;
	164
6abb4bd5	165	ret = pthread_mutex_unlock(mutex);
fdee2e6d MD	166	if (ret) {
	167	perror("Error in pthread mutex unlock");
	168	exit(-1);
	169	}
	170	}
	171
2dfb8b5e	172	void update_counter_and_wait(void)
fdee2e6d	173	{
16aa9ee8	174	CDS_LIST_HEAD(qsreaders);
fdee2e6d	175	int wait_loops = 0;
02be5561	176	struct rcu_reader index, tmp;
fdee2e6d	177
32c15e4e	178	/* Switch parity: 0 -> 1, 1 -> 0 */
6cf3827c	179	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
2dfb8b5e MD	180
	181	/*
	182	* Must commit qparity update to memory before waiting for other parity
	183	* quiescent state. Failure to do so could result in the writer waiting
	184	* forever while new readers are always accessing data (no progress).
6cf3827c	185	* Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
2dfb8b5e MD	186	*/
	187
	188	/*
5481ddb3	189	* Adding a cmm_smp_mb() which is _not_ formally required, but makes the
2dfb8b5e MD	190	* model easier to understand. It does not have a big performance impact
	191	* anyway, given this is the write-side.
	192	*/
5481ddb3	193	cmm_smp_mb();
2dfb8b5e	194
fdee2e6d	195	/*
02be5561	196	* Wait for each thread rcu_reader.ctr count to become 0.
fdee2e6d MD	197	*/
	198	for (;;) {
	199	wait_loops++;
16aa9ee8	200	cds_list_for_each_entry_safe(index, tmp, &registry, node) {
fdee2e6d	201	if (!rcu_old_gp_ongoing(&index->ctr))
16aa9ee8	202	cds_list_move(&index->node, &qsreaders);
fdee2e6d MD	203	}
fdee2e6d MD	204
16aa9ee8	205	if (cds_list_empty(&registry)) {
fdee2e6d MD	206	break;
	207	} else {
	208	if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
	209	usleep(RCU_SLEEP_DELAY);
	210	else
06f22bdb	211	caa_cpu_relax();
fdee2e6d MD	212	}
	213	}
	214	/* put back the reader list in the registry */
16aa9ee8	215	cds_list_splice(&qsreaders, &registry);
fdee2e6d MD	216	}
	217
	218	void synchronize_rcu(void)
	219	{
	220	sigset_t newmask, oldmask;
	221	int ret;
	222
	223	ret = sigemptyset(&newmask);
	224	assert(!ret);
	225	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	226	assert(!ret);
	227
6abb4bd5	228	mutex_lock(&rcu_gp_lock);
fdee2e6d	229
16aa9ee8	230	if (cds_list_empty(&registry))
2dfb8b5e	231	goto out;
fdee2e6d MD	232
fdee2e6d MD	233	/* All threads should read qparity before accessing data structure
2dfb8b5e	234	* where new ptr points to. */
fdee2e6d	235	/* Write new ptr before changing the qparity */
5481ddb3	236	cmm_smp_mb();
fdee2e6d	237
2dfb8b5e MD	238	/* Remove old registry elements */
2dfb8b5e MD	239	rcu_gc_registry();
fdee2e6d MD	240
	241	/*
	242	* Wait for previous parity to be empty of readers.
	243	*/
2dfb8b5e	244	update_counter_and_wait(); /* 0 -> 1, wait readers in parity 0 */
fdee2e6d MD	245
fdee2e6d MD	246	/*
5481ddb3	247	* Adding a cmm_smp_mb() which is _not_ formally required, but makes the
fdee2e6d MD	248	* model easier to understand. It does not have a big performance impact
	249	* anyway, given this is the write-side.
	250	*/
5481ddb3	251	cmm_smp_mb();
fdee2e6d	252
fdee2e6d	253	/*
2dfb8b5e	254	* Wait for previous parity to be empty of readers.
fdee2e6d	255	*/
2dfb8b5e	256	update_counter_and_wait(); /* 1 -> 0, wait readers in parity 1 */
fdee2e6d MD	257
fdee2e6d MD	258	/*
2dfb8b5e MD	259	* Finish waiting for reader threads before letting the old ptr being
2dfb8b5e MD	260	* freed.
fdee2e6d	261	*/
5481ddb3	262	cmm_smp_mb();
2dfb8b5e	263	out:
6abb4bd5	264	mutex_unlock(&rcu_gp_lock);
fdee2e6d MD	265	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	266	assert(!ret);
	267	}
	268
	269	/*
	270	* library wrappers to be used by non-LGPL compatible source code.
	271	*/
	272
	273	void rcu_read_lock(void)
	274	{
	275	_rcu_read_lock();
	276	}
	277
	278	void rcu_read_unlock(void)
	279	{
	280	_rcu_read_unlock();
	281	}
	282
	283	/*
	284	* only grow for now.
	285	*/
	286	static void resize_arena(struct registry_arena *arena, size_t len)
	287	{
	288	void *new_arena;
	289
0617bf4c MD	290	if (!arena->p)
	291	new_arena = mmap(arena->p, len,
	292	PROT_READ \| PROT_WRITE,
	293	MAP_ANONYMOUS \| MAP_PRIVATE,
	294	-1, 0);
	295	else
c7eaf61c MD	296	new_arena = mremap_wrapper(arena->p, arena->len,
c7eaf61c MD	297	len, MREMAP_MAYMOVE);
0617bf4c MD	298	assert(new_arena != MAP_FAILED);
0617bf4c MD	299
fdee2e6d MD	300	/*
	301	* re-used the same region ?
	302	*/
	303	if (new_arena == arena->p)
	304	return;
	305
fdee2e6d MD	306	bzero(new_arena + arena->len, len - arena->len);
	307	arena->p = new_arena;
	308	}
	309
	310	/* Called with signals off and mutex locked */
	311	static void add_thread(void)
	312	{
02be5561	313	struct rcu_reader *rcu_reader_reg;
fdee2e6d MD	314
fdee2e6d MD	315	if (registry_arena.len
02be5561	316	< registry_arena.used + sizeof(struct rcu_reader))
fdee2e6d	317	resize_arena(&registry_arena,
2f8a5ae7	318	caa_max(registry_arena.len << 1, ARENA_INIT_ALLOC));
fdee2e6d MD	319	/*
	320	* Find a free spot.
	321	*/
02be5561 MD	322	for (rcu_reader_reg = registry_arena.p;
	323	(void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	324	rcu_reader_reg++) {
	325	if (!rcu_reader_reg->alloc)
fdee2e6d MD	326	break;
fdee2e6d MD	327	}
02be5561 MD	328	rcu_reader_reg->alloc = 1;
02be5561 MD	329	registry_arena.used += sizeof(struct rcu_reader);
fdee2e6d MD	330
fdee2e6d MD	331	/* Add to registry */
02be5561 MD	332	rcu_reader_reg->tid = pthread_self();
02be5561 MD	333	assert(rcu_reader_reg->ctr == 0);
16aa9ee8	334	cds_list_add(&rcu_reader_reg->node, &registry);
bd252a04	335	URCU_TLS(rcu_reader) = rcu_reader_reg;
fdee2e6d MD	336	}
	337
	338	/* Called with signals off and mutex locked */
	339	static void rcu_gc_registry(void)
	340	{
02be5561	341	struct rcu_reader *rcu_reader_reg;
fdee2e6d MD	342	pthread_t tid;
	343	int ret;
	344
02be5561 MD	345	for (rcu_reader_reg = registry_arena.p;
	346	(void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	347	rcu_reader_reg++) {
	348	if (!rcu_reader_reg->alloc)
fdee2e6d	349	continue;
02be5561	350	tid = rcu_reader_reg->tid;
fdee2e6d MD	351	ret = pthread_kill(tid, 0);
	352	assert(ret != EINVAL);
	353	if (ret == ESRCH) {
16aa9ee8	354	cds_list_del(&rcu_reader_reg->node);
79266051	355	rcu_reader_reg->ctr = 0;
02be5561 MD	356	rcu_reader_reg->alloc = 0;
02be5561 MD	357	registry_arena.used -= sizeof(struct rcu_reader);
fdee2e6d MD	358	}
	359	}
	360	}
	361
	362	/* Disable signals, take mutex, add to registry */
	363	void rcu_bp_register(void)
	364	{
	365	sigset_t newmask, oldmask;
	366	int ret;
	367
	368	ret = sigemptyset(&newmask);
	369	assert(!ret);
	370	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	371	assert(!ret);
	372
	373	/*
	374	* Check if a signal concurrently registered our thread since
	375	* the check in rcu_read_lock(). */
bd252a04	376	if (URCU_TLS(rcu_reader))
fdee2e6d MD	377	goto end;
fdee2e6d MD	378
6abb4bd5	379	mutex_lock(&rcu_gp_lock);
fdee2e6d	380	add_thread();
6abb4bd5	381	mutex_unlock(&rcu_gp_lock);
fdee2e6d MD	382	end:
	383	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	384	assert(!ret);
	385	}
	386
9380711a	387	void rcu_bp_exit(void)
fdee2e6d	388	{
9380711a MD	389	if (registry_arena.p)
9380711a MD	390	munmap(registry_arena.p, registry_arena.len);
fdee2e6d	391	}
4cf1675f MD	392
	393	/*
	394	* Holding the rcu_gp_lock across fork will make sure we fork() don't race with
	395	* a concurrent thread executing with this same lock held. This ensures that the
	396	* registry is in a coherent state in the child.
	397	*/
	398	void rcu_bp_before_fork(void)
	399	{
	400	sigset_t newmask, oldmask;
	401	int ret;
	402
	403	ret = sigemptyset(&newmask);
	404	assert(!ret);
	405	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	406	assert(!ret);
	407	mutex_lock(&rcu_gp_lock);
	408	saved_fork_signal_mask = oldmask;
	409	}
	410
	411	void rcu_bp_after_fork_parent(void)
	412	{
	413	sigset_t oldmask;
	414	int ret;
	415
	416	oldmask = saved_fork_signal_mask;
	417	mutex_unlock(&rcu_gp_lock);
	418	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	419	assert(!ret);
	420	}
	421
	422	void rcu_bp_after_fork_child(void)
	423	{
	424	sigset_t oldmask;
	425	int ret;
	426
	427	rcu_gc_registry();
	428	oldmask = saved_fork_signal_mask;
	429	mutex_unlock(&rcu_gp_lock);
	430	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	431	assert(!ret);
	432	}
5e77fc1f	433
9b7981bb MD	434	void rcu_dereference_sym_bp(void p)
	435	{
	436	return _rcu_dereference(p);
	437	}
	438
5efd3cd2 MD	439	void rcu_set_pointer_sym_bp(void p, void v)
	440	{
	441	cmm_wmb();
424d4ed5 MD	442	uatomic_set(p, v);
424d4ed5 MD	443	return v;
5efd3cd2 MD	444	}
	445
	446	void rcu_xchg_pointer_sym_bp(void p, void v)
	447	{
	448	cmm_wmb();
	449	return uatomic_xchg(p, v);
	450	}
	451
	452	void rcu_cmpxchg_pointer_sym_bp(void p, void old, void *_new)
	453	{
	454	cmm_wmb();
	455	return uatomic_cmpxchg(p, old, _new);
	456	}
	457
5e6b23a6	458	DEFINE_RCU_FLAVOR(rcu_flavor);
541d828d	459
5e77fc1f	460	#include "urcu-call-rcu-impl.h"
0376e7b2	461	#include "urcu-defer-impl.h"