[userspace-rcu.git] / urcu / static / urcu-qsbr.h

#ifndef _URCU_QSBR_STATIC_H
#define _URCU_QSBR_STATIC_H

/*
 * urcu-qsbr-static.h
 *
 * Userspace RCU QSBR header.
 *
 * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
 * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
 *
 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
 */

#include <stdlib.h>
#include <pthread.h>
#include <assert.h>
#include <limits.h>
#include <unistd.h>
#include <stdint.h>

#include <urcu/compiler.h>
#include <urcu/arch.h>
#include <urcu/system.h>
#include <urcu/uatomic.h>
#include <urcu/list.h>
#include <urcu/futex.h>
#include <urcu/tls-compat.h>

#ifdef __cplusplus
extern "C" {
#endif 

/*
 * This code section can only be included in LGPL 2.1 compatible source code.
 * See below for the function call wrappers which can be used in code meant to
 * be only linked with the Userspace RCU library. This comes with a small
 * performance degradation on the read-side due to the added function calls.
 * This is required to permit relinking with newer versions of the library.
 */

#ifdef DEBUG_RCU
#define rcu_assert(args...)	assert(args)
#else
#define rcu_assert(args...)
#endif

#ifdef DEBUG_YIELD
#include <sched.h>
#include <time.h>
#include <pthread.h>
#include <unistd.h>

#define YIELD_READ 	(1 << 0)
#define YIELD_WRITE	(1 << 1)

/* maximum sleep delay, in us */
#define MAX_SLEEP 50

extern unsigned int yield_active;
extern DECLARE_URCU_TLS(unsigned int, rand_yield);

static inline void debug_yield_read(void)
{
	if (yield_active & YIELD_READ)
		if (rand_r(&URCU_TLS(rand_yield)) & 0x1)
			usleep(rand_r(&URCU_TLS(rand_yield)) % MAX_SLEEP);
}

static inline void debug_yield_write(void)
{
	if (yield_active & YIELD_WRITE)
		if (rand_r(&URCU_TLS(rand_yield)) & 0x1)
			usleep(rand_r(&URCU_TLS(rand_yield)) % MAX_SLEEP);
}

static inline void debug_yield_init(void)
{
	URCU_TLS(rand_yield) = time(NULL) ^ (unsigned long) pthread_self();
}
#else
static inline void debug_yield_read(void)
{
}

static inline void debug_yield_write(void)
{
}

static inline void debug_yield_init(void)
{

}
#endif

#define RCU_GP_ONLINE		(1UL << 0)
#define RCU_GP_CTR		(1UL << 1)

/*
 * Global quiescent period counter with low-order bits unused.
 * Using a int rather than a char to eliminate false register dependencies
 * causing stalls on some architectures.
 */
extern unsigned long rcu_gp_ctr;

struct rcu_reader {
	/* Data used by both reader and synchronize_rcu() */
	unsigned long ctr;
	/* Data used for registry */
	struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
	int waiting;
	pthread_t tid;
};

extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);

extern int32_t gp_futex;

/*
 * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
 */
static inline void wake_up_gp(void)
{
	if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(rcu_reader).waiting))) {
		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).waiting, 0);
		cmm_smp_mb();
		if (uatomic_read(&gp_futex) != -1)
			return;
		uatomic_set(&gp_futex, 0);
		/*
		 * Ignoring return value until we can make this function
		 * return something (because urcu_die() is not publicly
		 * exposed).
		 */
		(void) futex_noasync(&gp_futex, FUTEX_WAKE, 1,
				NULL, NULL, 0);
	}
}

static inline int rcu_gp_ongoing(unsigned long *ctr)
{
	unsigned long v;

	v = CMM_LOAD_SHARED(*ctr);
	return v && (v != rcu_gp_ctr);
}

/*
 * Enter an RCU read-side critical section.
 *
 * This function is less than 10 lines long.  The intent is that this
 * function meets the 10-line criterion for LGPL, allowing this function
 * to be invoked directly from non-LGPL code.
 */
static inline void _rcu_read_lock(void)
{
	rcu_assert(URCU_TLS(rcu_reader).ctr);
}

/*
 * Exit an RCU read-side critical section.
 *
 * This function is less than 10 lines long.  The intent is that this
 * function meets the 10-line criterion for LGPL, allowing this function
 * to be invoked directly from non-LGPL code.
 */
static inline void _rcu_read_unlock(void)
{
}

/*
 * Inform RCU of a quiescent state.
 *
 * This function is less than 10 lines long.  The intent is that this
 * function meets the 10-line criterion for LGPL, allowing this function
 * to be invoked directly from non-LGPL code.
 */
static inline void _rcu_quiescent_state(void)
{
	cmm_smp_mb();
	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
	cmm_smp_mb();	/* write URCU_TLS(rcu_reader).ctr before read futex */
	wake_up_gp();
	cmm_smp_mb();
}

/*
 * Take a thread offline, prohibiting it from entering further RCU
 * read-side critical sections.
 *
 * This function is less than 10 lines long.  The intent is that this
 * function meets the 10-line criterion for LGPL, allowing this function
 * to be invoked directly from non-LGPL code.
 */
static inline void _rcu_thread_offline(void)
{
	cmm_smp_mb();
	CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, 0);
	cmm_smp_mb();	/* write URCU_TLS(rcu_reader).ctr before read futex */
	wake_up_gp();
	cmm_barrier();	/* Ensure the compiler does not reorder us with mutex */
}

/*
 * Bring a thread online, allowing it to once again enter RCU
 * read-side critical sections.
 *
 * This function is less than 10 lines long.  The intent is that this
 * function meets the 10-line criterion for LGPL, allowing this function
 * to be invoked directly from non-LGPL code.
 */
static inline void _rcu_thread_online(void)
{
	cmm_barrier();	/* Ensure the compiler does not reorder us with mutex */
	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp_ctr));
	cmm_smp_mb();
}

#ifdef __cplusplus 
}
#endif

#endif /* _URCU_QSBR_STATIC_H */
Commit	Line	Data
	1	#ifndef _URCU_QSBR_STATIC_H
	2	#define _URCU_QSBR_STATIC_H
	3
	4	/*
	5	* urcu-qsbr-static.h
	6	*
	7	* Userspace RCU QSBR header.
	8	*
	9	* TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
	10	* RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
	11	*
	12	* Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	13	* Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
	14	*
	15	* This library is free software; you can redistribute it and/or
	16	* modify it under the terms of the GNU Lesser General Public
	17	* License as published by the Free Software Foundation; either
	18	* version 2.1 of the License, or (at your option) any later version.
	19	*
	20	* This library is distributed in the hope that it will be useful,
	21	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	22	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	23	* Lesser General Public License for more details.
	24	*
	25	* You should have received a copy of the GNU Lesser General Public
	26	* License along with this library; if not, write to the Free Software
	27	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	28	*
	29	* IBM's contributions to this file may be relicensed under LGPLv2 or later.
	30	*/
	31
	32	#include <stdlib.h>
	33	#include <pthread.h>
	34	#include <assert.h>
	35	#include <limits.h>
	36	#include <unistd.h>
	37	#include <stdint.h>
	38
	39	#include <urcu/compiler.h>
	40	#include <urcu/arch.h>
	41	#include <urcu/system.h>
	42	#include <urcu/uatomic.h>
	43	#include <urcu/list.h>
	44	#include <urcu/futex.h>
	45	#include <urcu/tls-compat.h>
	46
	47	#ifdef __cplusplus
	48	extern "C" {
	49	#endif
	50
	51	/*
	52	* This code section can only be included in LGPL 2.1 compatible source code.
	53	* See below for the function call wrappers which can be used in code meant to
	54	* be only linked with the Userspace RCU library. This comes with a small
	55	* performance degradation on the read-side due to the added function calls.
	56	* This is required to permit relinking with newer versions of the library.
	57	*/
	58
	59	#ifdef DEBUG_RCU
	60	#define rcu_assert(args...) assert(args)
	61	#else
	62	#define rcu_assert(args...)
	63	#endif
	64
	65	#ifdef DEBUG_YIELD
	66	#include <sched.h>
	67	#include <time.h>
	68	#include <pthread.h>
	69	#include <unistd.h>
	70
	71	#define YIELD_READ (1 << 0)
	72	#define YIELD_WRITE (1 << 1)
	73
	74	/* maximum sleep delay, in us */
	75	#define MAX_SLEEP 50
	76
	77	extern unsigned int yield_active;
	78	extern DECLARE_URCU_TLS(unsigned int, rand_yield);
	79
	80	static inline void debug_yield_read(void)
	81	{
	82	if (yield_active & YIELD_READ)
	83	if (rand_r(&URCU_TLS(rand_yield)) & 0x1)
	84	usleep(rand_r(&URCU_TLS(rand_yield)) % MAX_SLEEP);
	85	}
	86
	87	static inline void debug_yield_write(void)
	88	{
	89	if (yield_active & YIELD_WRITE)
	90	if (rand_r(&URCU_TLS(rand_yield)) & 0x1)
	91	usleep(rand_r(&URCU_TLS(rand_yield)) % MAX_SLEEP);
	92	}
	93
	94	static inline void debug_yield_init(void)
	95	{
	96	URCU_TLS(rand_yield) = time(NULL) ^ (unsigned long) pthread_self();
	97	}
	98	#else
	99	static inline void debug_yield_read(void)
	100	{
	101	}
	102
	103	static inline void debug_yield_write(void)
	104	{
	105	}
	106
	107	static inline void debug_yield_init(void)
	108	{
	109
	110	}
	111	#endif
	112
	113	#define RCU_GP_ONLINE (1UL << 0)
	114	#define RCU_GP_CTR (1UL << 1)
	115
	116	/*
	117	* Global quiescent period counter with low-order bits unused.
	118	* Using a int rather than a char to eliminate false register dependencies
	119	* causing stalls on some architectures.
	120	*/
	121	extern unsigned long rcu_gp_ctr;
	122
	123	struct rcu_reader {
	124	/* Data used by both reader and synchronize_rcu() */
	125	unsigned long ctr;
	126	/* Data used for registry */
	127	struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
	128	int waiting;
	129	pthread_t tid;
	130	};
	131
	132	extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
	133
	134	extern int32_t gp_futex;
	135
	136	/*
	137	* Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
	138	*/
	139	static inline void wake_up_gp(void)
	140	{
	141	if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(rcu_reader).waiting))) {
	142	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).waiting, 0);
	143	cmm_smp_mb();
	144	if (uatomic_read(&gp_futex) != -1)
	145	return;
	146	uatomic_set(&gp_futex, 0);
	147	/*
	148	* Ignoring return value until we can make this function
	149	* return something (because urcu_die() is not publicly
	150	* exposed).
	151	*/
	152	(void) futex_noasync(&gp_futex, FUTEX_WAKE, 1,
	153	NULL, NULL, 0);
	154	}
	155	}
	156
	157	static inline int rcu_gp_ongoing(unsigned long *ctr)
	158	{
	159	unsigned long v;
	160
	161	v = CMM_LOAD_SHARED(*ctr);
	162	return v && (v != rcu_gp_ctr);
	163	}
	164
	165	/*
	166	* Enter an RCU read-side critical section.
	167	*
	168	* This function is less than 10 lines long. The intent is that this
	169	* function meets the 10-line criterion for LGPL, allowing this function
	170	* to be invoked directly from non-LGPL code.
	171	*/
	172	static inline void _rcu_read_lock(void)
	173	{
	174	rcu_assert(URCU_TLS(rcu_reader).ctr);
	175	}
	176
	177	/*
	178	* Exit an RCU read-side critical section.
	179	*
	180	* This function is less than 10 lines long. The intent is that this
	181	* function meets the 10-line criterion for LGPL, allowing this function
	182	* to be invoked directly from non-LGPL code.
	183	*/
	184	static inline void _rcu_read_unlock(void)
	185	{
	186	}
	187
	188	/*
	189	* Inform RCU of a quiescent state.
	190	*
	191	* This function is less than 10 lines long. The intent is that this
	192	* function meets the 10-line criterion for LGPL, allowing this function
	193	* to be invoked directly from non-LGPL code.
	194	*/
	195	static inline void _rcu_quiescent_state(void)
	196	{
	197	cmm_smp_mb();
	198	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
	199	cmm_smp_mb(); /* write URCU_TLS(rcu_reader).ctr before read futex */
	200	wake_up_gp();
	201	cmm_smp_mb();
	202	}
	203
	204	/*
	205	* Take a thread offline, prohibiting it from entering further RCU
	206	* read-side critical sections.
	207	*
	208	* This function is less than 10 lines long. The intent is that this
	209	* function meets the 10-line criterion for LGPL, allowing this function
	210	* to be invoked directly from non-LGPL code.
	211	*/
	212	static inline void _rcu_thread_offline(void)
	213	{
	214	cmm_smp_mb();
	215	CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, 0);
	216	cmm_smp_mb(); /* write URCU_TLS(rcu_reader).ctr before read futex */
	217	wake_up_gp();
	218	cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
	219	}
	220
	221	/*
	222	* Bring a thread online, allowing it to once again enter RCU
	223	* read-side critical sections.
	224	*
	225	* This function is less than 10 lines long. The intent is that this
	226	* function meets the 10-line criterion for LGPL, allowing this function
	227	* to be invoked directly from non-LGPL code.
	228	*/
	229	static inline void _rcu_thread_online(void)
	230	{
	231	cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
	232	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp_ctr));
	233	cmm_smp_mb();
	234	}
	235
	236	#ifdef __cplusplus
	237	}
	238	#endif
	239
	240	#endif /* _URCU_QSBR_STATIC_H */