[urcu.git] / urcu / static / urcu-qsbr.h

#ifndef _URCU_QSBR_STATIC_H
#define _URCU_QSBR_STATIC_H

/*
 * urcu-qsbr-static.h
 *
 * Userspace RCU QSBR header.
 *
 * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
 * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
 *
 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
 */

#include <stdlib.h>
#include <pthread.h>
#include <assert.h>
#include <limits.h>
#include <unistd.h>
#include <stdint.h>

#include <urcu/compiler.h>
#include <urcu/arch.h>
#include <urcu/system.h>
#include <urcu/uatomic.h>
#include <urcu/list.h>
#include <urcu/futex.h>
#include <urcu/tls-compat.h>

#ifdef __cplusplus
extern "C" {
#endif 

/*
 * This code section can only be included in LGPL 2.1 compatible source code.
 * See below for the function call wrappers which can be used in code meant to
 * be only linked with the Userspace RCU library. This comes with a small
 * performance degradation on the read-side due to the added function calls.
 * This is required to permit relinking with newer versions of the library.
 */

#ifdef DEBUG_RCU
#define rcu_assert(args...)	assert(args)
#else
#define rcu_assert(args...)
#endif

#ifdef DEBUG_YIELD
#include <sched.h>
#include <time.h>
#include <pthread.h>
#include <unistd.h>

#define RCU_YIELD_READ 	(1 << 0)
#define RCU_YIELD_WRITE	(1 << 1)

/* maximum sleep delay, in us */
#define MAX_SLEEP 50

extern unsigned int rcu_yield_active;
extern DECLARE_URCU_TLS(unsigned int, rcu_rand_yield);

static inline void rcu_debug_yield_read(void)
{
	if (rcu_yield_active & RCU_YIELD_READ)
		if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1)
			usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP);
}

static inline void rcu_debug_yield_write(void)
{
	if (rcu_yield_active & RCU_YIELD_WRITE)
		if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1)
			usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP);
}

static inline void rcu_debug_yield_init(void)
{
	URCU_TLS(rcu_rand_yield) = time(NULL) ^ (unsigned long) pthread_self();
}
#else
static inline void rcu_debug_yield_read(void)
{
}

static inline void rcu_debug_yield_write(void)
{
}

static inline void rcu_debug_yield_init(void)
{

}
#endif

#define RCU_GP_ONLINE		(1UL << 0)
#define RCU_GP_CTR		(1UL << 1)

/*
 * Global quiescent period counter with low-order bits unused.
 * Using a int rather than a char to eliminate false register dependencies
 * causing stalls on some architectures.
 */
extern unsigned long rcu_gp_ctr;

struct rcu_reader {
	/* Data used by both reader and synchronize_rcu() */
	unsigned long ctr;
	/* Data used for registry */
	struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
	int waiting;
	pthread_t tid;
};

extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);

extern int32_t rcu_gp_futex;

/*
 * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
 */
static inline void wake_up_gp(void)
{
	if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(rcu_reader).waiting))) {
		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).waiting, 0);
		cmm_smp_mb();
		if (uatomic_read(&rcu_gp_futex) != -1)
			return;
		uatomic_set(&rcu_gp_futex, 0);
		futex_noasync(&rcu_gp_futex, FUTEX_WAKE, 1,
		      NULL, NULL, 0);
	}
}

static inline int rcu_gp_ongoing(unsigned long *ctr)
{
	unsigned long v;

	v = CMM_LOAD_SHARED(*ctr);
	return v && (v != rcu_gp_ctr);
}

/*
 * Enter an RCU read-side critical section.
 *
 * This function is less than 10 lines long.  The intent is that this
 * function meets the 10-line criterion for LGPL, allowing this function
 * to be invoked directly from non-LGPL code.
 */
static inline void _rcu_read_lock(void)
{
	rcu_assert(URCU_TLS(rcu_reader).ctr);
}

/*
 * Exit an RCU read-side critical section.
 *
 * This function is less than 10 lines long.  The intent is that this
 * function meets the 10-line criterion for LGPL, allowing this function
 * to be invoked directly from non-LGPL code.
 */
static inline void _rcu_read_unlock(void)
{
}

/*
 * Inform RCU of a quiescent state.
 *
 * This function is less than 10 lines long.  The intent is that this
 * function meets the 10-line criterion for LGPL, allowing this function
 * to be invoked directly from non-LGPL code.
 */
static inline void _rcu_quiescent_state(void)
{
	cmm_smp_mb();
	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
	cmm_smp_mb();	/* write URCU_TLS(rcu_reader).ctr before read futex */
	wake_up_gp();
	cmm_smp_mb();
}

/*
 * Take a thread offline, prohibiting it from entering further RCU
 * read-side critical sections.
 *
 * This function is less than 10 lines long.  The intent is that this
 * function meets the 10-line criterion for LGPL, allowing this function
 * to be invoked directly from non-LGPL code.
 */
static inline void _rcu_thread_offline(void)
{
	cmm_smp_mb();
	CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, 0);
	cmm_smp_mb();	/* write URCU_TLS(rcu_reader).ctr before read futex */
	wake_up_gp();
	cmm_barrier();	/* Ensure the compiler does not reorder us with mutex */
}

/*
 * Bring a thread online, allowing it to once again enter RCU
 * read-side critical sections.
 *
 * This function is less than 10 lines long.  The intent is that this
 * function meets the 10-line criterion for LGPL, allowing this function
 * to be invoked directly from non-LGPL code.
 */
static inline void _rcu_thread_online(void)
{
	cmm_barrier();	/* Ensure the compiler does not reorder us with mutex */
	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp_ctr));
	cmm_smp_mb();
}

#ifdef __cplusplus 
}
#endif

#endif /* _URCU_QSBR_STATIC_H */
Commit	Line	Data
	1	#ifndef _URCU_QSBR_STATIC_H
	2	#define _URCU_QSBR_STATIC_H
	3
	4	/*
	5	* urcu-qsbr-static.h
	6	*
	7	* Userspace RCU QSBR header.
	8	*
	9	* TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
	10	* RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
	11	*
	12	* Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	13	* Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
	14	*
	15	* This library is free software; you can redistribute it and/or
	16	* modify it under the terms of the GNU Lesser General Public
	17	* License as published by the Free Software Foundation; either
	18	* version 2.1 of the License, or (at your option) any later version.
	19	*
	20	* This library is distributed in the hope that it will be useful,
	21	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	22	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	23	* Lesser General Public License for more details.
	24	*
	25	* You should have received a copy of the GNU Lesser General Public
	26	* License along with this library; if not, write to the Free Software
	27	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	28	*
	29	* IBM's contributions to this file may be relicensed under LGPLv2 or later.
	30	*/
	31
	32	#include <stdlib.h>
	33	#include <pthread.h>
	34	#include <assert.h>
	35	#include <limits.h>
	36	#include <unistd.h>
	37	#include <stdint.h>
	38
	39	#include <urcu/compiler.h>
	40	#include <urcu/arch.h>
	41	#include <urcu/system.h>
	42	#include <urcu/uatomic.h>
	43	#include <urcu/list.h>
	44	#include <urcu/futex.h>
	45	#include <urcu/tls-compat.h>
	46
	47	#ifdef __cplusplus
	48	extern "C" {
	49	#endif
	50
	51	/*
	52	* This code section can only be included in LGPL 2.1 compatible source code.
	53	* See below for the function call wrappers which can be used in code meant to
	54	* be only linked with the Userspace RCU library. This comes with a small
	55	* performance degradation on the read-side due to the added function calls.
	56	* This is required to permit relinking with newer versions of the library.
	57	*/
	58
	59	#ifdef DEBUG_RCU
	60	#define rcu_assert(args...) assert(args)
	61	#else
	62	#define rcu_assert(args...)
	63	#endif
	64
	65	#ifdef DEBUG_YIELD
	66	#include <sched.h>
	67	#include <time.h>
	68	#include <pthread.h>
	69	#include <unistd.h>
	70
	71	#define RCU_YIELD_READ (1 << 0)
	72	#define RCU_YIELD_WRITE (1 << 1)
	73
	74	/* maximum sleep delay, in us */
	75	#define MAX_SLEEP 50
	76
	77	extern unsigned int rcu_yield_active;
	78	extern DECLARE_URCU_TLS(unsigned int, rcu_rand_yield);
	79
	80	static inline void rcu_debug_yield_read(void)
	81	{
	82	if (rcu_yield_active & RCU_YIELD_READ)
	83	if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1)
	84	usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP);
	85	}
	86
	87	static inline void rcu_debug_yield_write(void)
	88	{
	89	if (rcu_yield_active & RCU_YIELD_WRITE)
	90	if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1)
	91	usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP);
	92	}
	93
	94	static inline void rcu_debug_yield_init(void)
	95	{
	96	URCU_TLS(rcu_rand_yield) = time(NULL) ^ (unsigned long) pthread_self();
	97	}
	98	#else
	99	static inline void rcu_debug_yield_read(void)
	100	{
	101	}
	102
	103	static inline void rcu_debug_yield_write(void)
	104	{
	105	}
	106
	107	static inline void rcu_debug_yield_init(void)
	108	{
	109
	110	}
	111	#endif
	112
	113	#define RCU_GP_ONLINE (1UL << 0)
	114	#define RCU_GP_CTR (1UL << 1)
	115
	116	/*
	117	* Global quiescent period counter with low-order bits unused.
	118	* Using a int rather than a char to eliminate false register dependencies
	119	* causing stalls on some architectures.
	120	*/
	121	extern unsigned long rcu_gp_ctr;
	122
	123	struct rcu_reader {
	124	/* Data used by both reader and synchronize_rcu() */
	125	unsigned long ctr;
	126	/* Data used for registry */
	127	struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
	128	int waiting;
	129	pthread_t tid;
	130	};
	131
	132	extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
	133
	134	extern int32_t rcu_gp_futex;
	135
	136	/*
	137	* Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
	138	*/
	139	static inline void wake_up_gp(void)
	140	{
	141	if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(rcu_reader).waiting))) {
	142	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).waiting, 0);
	143	cmm_smp_mb();
	144	if (uatomic_read(&rcu_gp_futex) != -1)
	145	return;
	146	uatomic_set(&rcu_gp_futex, 0);
	147	futex_noasync(&rcu_gp_futex, FUTEX_WAKE, 1,
	148	NULL, NULL, 0);
	149	}
	150	}
	151
	152	static inline int rcu_gp_ongoing(unsigned long *ctr)
	153	{
	154	unsigned long v;
	155
	156	v = CMM_LOAD_SHARED(*ctr);
	157	return v && (v != rcu_gp_ctr);
	158	}
	159
	160	/*
	161	* Enter an RCU read-side critical section.
	162	*
	163	* This function is less than 10 lines long. The intent is that this
	164	* function meets the 10-line criterion for LGPL, allowing this function
	165	* to be invoked directly from non-LGPL code.
	166	*/
	167	static inline void _rcu_read_lock(void)
	168	{
	169	rcu_assert(URCU_TLS(rcu_reader).ctr);
	170	}
	171
	172	/*
	173	* Exit an RCU read-side critical section.
	174	*
	175	* This function is less than 10 lines long. The intent is that this
	176	* function meets the 10-line criterion for LGPL, allowing this function
	177	* to be invoked directly from non-LGPL code.
	178	*/
	179	static inline void _rcu_read_unlock(void)
	180	{
	181	}
	182
	183	/*
	184	* Inform RCU of a quiescent state.
	185	*
	186	* This function is less than 10 lines long. The intent is that this
	187	* function meets the 10-line criterion for LGPL, allowing this function
	188	* to be invoked directly from non-LGPL code.
	189	*/
	190	static inline void _rcu_quiescent_state(void)
	191	{
	192	cmm_smp_mb();
	193	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
	194	cmm_smp_mb(); /* write URCU_TLS(rcu_reader).ctr before read futex */
	195	wake_up_gp();
	196	cmm_smp_mb();
	197	}
	198
	199	/*
	200	* Take a thread offline, prohibiting it from entering further RCU
	201	* read-side critical sections.
	202	*
	203	* This function is less than 10 lines long. The intent is that this
	204	* function meets the 10-line criterion for LGPL, allowing this function
	205	* to be invoked directly from non-LGPL code.
	206	*/
	207	static inline void _rcu_thread_offline(void)
	208	{
	209	cmm_smp_mb();
	210	CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, 0);
	211	cmm_smp_mb(); /* write URCU_TLS(rcu_reader).ctr before read futex */
	212	wake_up_gp();
	213	cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
	214	}
	215
	216	/*
	217	* Bring a thread online, allowing it to once again enter RCU
	218	* read-side critical sections.
	219	*
	220	* This function is less than 10 lines long. The intent is that this
	221	* function meets the 10-line criterion for LGPL, allowing this function
	222	* to be invoked directly from non-LGPL code.
	223	*/
	224	static inline void _rcu_thread_online(void)
	225	{
	226	cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
	227	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp_ctr));
	228	cmm_smp_mb();
	229	}
	230
	231	#ifdef __cplusplus
	232	}
	233	#endif
	234
	235	#endif /* _URCU_QSBR_STATIC_H */