[userspace-rcu.git] / rcuja / rcuja-internal.h

#ifndef _URCU_RCUJA_INTERNAL_H
#define _URCU_RCUJA_INTERNAL_H

/*
 * rcuja/rcuja-internal.h
 *
 * Userspace RCU library - RCU Judy Array Internal Header
 *
 * Copyright 2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#define _GNU_SOURCE
#include <pthread.h>
#include <stdio.h>
#include <inttypes.h>
#include <unistd.h>
#include <urcu/rculfhash.h>

/*
 * Number of least significant pointer bits reserved to represent the
 * child type.
 */
#define JA_TYPE_BITS	3
#define JA_TYPE_MAX_NR	(1UL << JA_TYPE_BITS)
#define JA_TYPE_MASK	(JA_TYPE_MAX_NR - 1)
#define JA_PTR_MASK	(~JA_TYPE_MASK)

#define JA_ENTRY_PER_NODE	256UL
#define JA_LOG2_BITS_PER_BYTE	3U
#define JA_BITS_PER_BYTE	(1U << JA_LOG2_BITS_PER_BYTE)

#define JA_POOL_1D_MASK	((JA_BITS_PER_BYTE - 1) << JA_TYPE_BITS)
#define JA_POOL_2D_MASK	(JA_POOL_1D_MASK << JA_LOG2_BITS_PER_BYTE)

#define JA_MAX_DEPTH	9	/* Maximum depth, including leafs */

/*
 * Entry for NULL node is at index 8 of the table. It is never encoded
 * in flags.
 */
#define NODE_INDEX_NULL		8

/*
 * Number of removals needed on a fallback node before we try to shrink
 * it.
 */
#define JA_FALLBACK_REMOVAL_COUNT	8

/* Never declared. Opaque type used to store flagged node pointers. */
struct cds_ja_inode_flag;
struct cds_ja_inode;

/*
 * Shadow node contains mutex and call_rcu head associated with a node.
 */
struct cds_ja_shadow_node {
	struct cds_lfht_node ht_node;	/* hash table node */
	struct cds_ja_inode_flag *node_flag;	/* reverse mapping and hash table key */
	/*
	 * mutual exclusion on all nodes belonging to the same tree
	 * position (e.g. both nodes before and after recompaction
	 * use the same lock).
	 */
	pthread_mutex_t *lock;
	unsigned int nr_child;		/* number of children in node */
	struct rcu_head head;		/* for deferred node and shadow node reclaim */
	int fallback_removal_count;	/* removals left keeping fallback */
	int level;			/* level in the tree */
	struct cds_ja *ja;		/* toplevel judy array */
};

struct cds_ja {
	struct cds_ja_inode_flag *root;
	unsigned int tree_depth;
	uint64_t key_max;
	/*
	 * We use a hash table to associate node keys to their
	 * respective shadow node. This helps reducing lookup hot path
	 * cache footprint, especially for very small nodes.
	 */
	struct cds_lfht *ht;
	unsigned long nr_fallback;	/* Number of fallback nodes used */

	/* For debugging */
	unsigned long node_fallback_count_distribution[JA_ENTRY_PER_NODE];
	unsigned long nr_nodes_allocated, nr_nodes_freed;
};

static inline
struct cds_ja_inode_flag *ja_node_flag(struct cds_ja_inode *node,
		unsigned long type)
{
	assert(type < (1UL << JA_TYPE_BITS));
	return (struct cds_ja_inode_flag *) (((unsigned long) node) | type);
}

static inline
struct cds_ja_inode_flag *ja_node_flag_pool_1d(struct cds_ja_inode *node,
		unsigned long type, unsigned long bitsel)
{
	assert(type < (1UL << JA_TYPE_BITS));
	assert(bitsel < JA_BITS_PER_BYTE);
	return (struct cds_ja_inode_flag *) (((unsigned long) node) | (bitsel << JA_TYPE_BITS) | type);
}

static inline
struct cds_ja_inode_flag *ja_node_flag_pool_2d(struct cds_ja_inode *node,
		unsigned long type, unsigned int bitsel[2])
{
	assert(type < (1UL << JA_TYPE_BITS));
	assert(bitsel[0] < JA_BITS_PER_BYTE);
	assert(bitsel[1] < JA_BITS_PER_BYTE);
	return (struct cds_ja_inode_flag *) (((unsigned long) node) | (bitsel[0] << (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE)) | (bitsel[1] << JA_TYPE_BITS) | type);
}

static inline
unsigned long ja_node_pool_1d_bitsel(struct cds_ja_inode_flag *node)
{
	return ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
}

static inline
void ja_node_pool_2d_bitsel(struct cds_ja_inode_flag *node, unsigned long *bits)
{
	bits[0] = ((unsigned long) node & JA_POOL_2D_MASK) >> (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE);
	bits[1] = ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
}

/* Hardcoded pool indexes for fast path */
#define RCU_JA_POOL_IDX_5	5
#define RCU_JA_POOL_IDX_6	6
static inline
struct cds_ja_inode *ja_node_ptr(struct cds_ja_inode_flag *node)
{
	unsigned long v, type_idx;

	if (!node)
		return NULL;	/* RCU_JA_NULL */
	v = (unsigned long) node;
	type_idx = v & JA_TYPE_MASK;

	switch (type_idx) {
	case RCU_JA_POOL_IDX_5:
		v &= ~(JA_POOL_1D_MASK | JA_TYPE_MASK);
		break;
	case RCU_JA_POOL_IDX_6:
		v &= ~(JA_POOL_2D_MASK | JA_POOL_1D_MASK | JA_TYPE_MASK);
		break;
	default:
		/* RCU_JA_LINEAR or RCU_JA_PIGEON */
		v &= JA_PTR_MASK;
		break;
	}
	return (struct cds_ja_inode *) v;
}

__attribute__((visibility("protected")))
unsigned long ja_node_type(struct cds_ja_inode_flag *node);

__attribute__((visibility("protected")))
void rcuja_free_all_children(struct cds_ja_shadow_node *shadow_node,
		struct cds_ja_inode_flag *node_flag,
		void (*free_node_cb)(struct cds_ja_node *node));

__attribute__((visibility("protected")))
struct cds_ja_shadow_node *rcuja_shadow_lookup_lock(struct cds_lfht *ht,
		struct cds_ja_inode_flag *node_flag);

__attribute__((visibility("protected")))
void rcuja_shadow_unlock(struct cds_ja_shadow_node *shadow_node);

__attribute__((visibility("protected")))
struct cds_ja_shadow_node *rcuja_shadow_set(struct cds_lfht *ht,
		struct cds_ja_inode_flag *new_node_flag,
		struct cds_ja_shadow_node *inherit_from,
		struct cds_ja *ja, int level);

/* rcuja_shadow_clear flags */
enum {
	RCUJA_SHADOW_CLEAR_FREE_NODE = (1U << 0),
	RCUJA_SHADOW_CLEAR_FREE_LOCK = (1U << 1),
};

__attribute__((visibility("protected")))
int rcuja_shadow_clear(struct cds_lfht *ht,
		struct cds_ja_inode_flag *node_flag,
		struct cds_ja_shadow_node *shadow_node,
		unsigned int flags);

__attribute__((visibility("protected")))
void rcuja_shadow_prune(struct cds_lfht *ht,
		unsigned int flags,
		void (*free_node_cb)(struct cds_ja_node *node));

__attribute__((visibility("protected")))
struct cds_lfht *rcuja_create_ht(const struct rcu_flavor_struct *flavor);

__attribute__((visibility("protected")))
int rcuja_delete_ht(struct cds_lfht *ht);

__attribute__((visibility("protected")))
void free_cds_ja_node(struct cds_ja *ja, struct cds_ja_inode *node);

/*
 * Iterate through duplicates returned by cds_ja_lookup*()
 * Receives a struct cds_ja_node * as parameter, which is used as start
 * of duplicate list and loop cursor.
 */
#define cds_ja_for_each_duplicate(pos)					\
	for (; (pos) != NULL; (pos) = (pos)->next)

/*
 * Iterate through duplicates returned by cds_ja_lookup*()
 * Safe against removal of entries during traversal.
 */
#define cds_ja_for_each_duplicate_safe(_pos, _next)			\
	for (; (_pos) != NULL ? ((_next) = (_pos)->next, 1) : 0;	\
		(_pos) = (_next))

//#define DEBUG

#ifdef __linux__
#include <syscall.h>
#endif

#if defined(_syscall0)
_syscall0(pid_t, gettid)
#elif defined(__NR_gettid)
static inline pid_t gettid(void)
{
	return syscall(__NR_gettid);
}
#else
#warning "use pid as tid"
static inline pid_t gettid(void)
{
	return getpid();
}
#endif

#ifdef DEBUG
#define dbg_printf(fmt, args...)				\
	fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt,	\
		(unsigned long) gettid(), __func__,		\
		__FILE__, __LINE__, ## args)
#else
#define dbg_printf(fmt, args...)				\
do {								\
	/* do nothing but check printf format */		\
	if (0)							\
		fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \
			(unsigned long) gettid(), __func__,	\
			__FILE__, __LINE__, ## args);		\
} while (0)
#endif

#endif /* _URCU_RCUJA_INTERNAL_H */
Commit	Line	Data
61009379 MD	1	#ifndef _URCU_RCUJA_INTERNAL_H
	2	#define _URCU_RCUJA_INTERNAL_H
	3
	4	/*
	5	* rcuja/rcuja-internal.h
	6	*
	7	* Userspace RCU library - RCU Judy Array Internal Header
	8	*
	9	* Copyright 2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	10	*
	11	* This library is free software; you can redistribute it and/or
	12	* modify it under the terms of the GNU Lesser General Public
	13	* License as published by the Free Software Foundation; either
	14	* version 2.1 of the License, or (at your option) any later version.
	15	*
	16	* This library is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* Lesser General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU Lesser General Public
	22	* License along with this library; if not, write to the Free Software
	23	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
7d67da99	26	#define _GNU_SOURCE
511b4dcc	27	#include <pthread.h>
a2a7ff59 MD	28	#include <stdio.h>
a2a7ff59 MD	29	#include <inttypes.h>
7d67da99	30	#include <unistd.h>
511b4dcc MD	31	#include <urcu/rculfhash.h>
511b4dcc MD	32
3d8fe307 MD	33	/*
	34	* Number of least significant pointer bits reserved to represent the
	35	* child type.
	36	*/
	37	#define JA_TYPE_BITS 3
	38	#define JA_TYPE_MAX_NR (1UL << JA_TYPE_BITS)
	39	#define JA_TYPE_MASK (JA_TYPE_MAX_NR - 1)
	40	#define JA_PTR_MASK (~JA_TYPE_MASK)
	41
	42	#define JA_ENTRY_PER_NODE 256UL
	43	#define JA_LOG2_BITS_PER_BYTE 3U
	44	#define JA_BITS_PER_BYTE (1U << JA_LOG2_BITS_PER_BYTE)
	45
b1a90ce3 MD	46	#define JA_POOL_1D_MASK ((JA_BITS_PER_BYTE - 1) << JA_TYPE_BITS)
	47	#define JA_POOL_2D_MASK (JA_POOL_1D_MASK << JA_LOG2_BITS_PER_BYTE)
	48
3d8fe307 MD	49	#define JA_MAX_DEPTH 9 /* Maximum depth, including leafs */
	50
	51	/*
	52	* Entry for NULL node is at index 8 of the table. It is never encoded
	53	* in flags.
	54	*/
	55	#define NODE_INDEX_NULL 8
	56
	57	/*
	58	* Number of removals needed on a fallback node before we try to shrink
	59	* it.
	60	*/
	61	#define JA_FALLBACK_REMOVAL_COUNT 8
	62
511b4dcc	63	/* Never declared. Opaque type used to store flagged node pointers. */
b4540e8a	64	struct cds_ja_inode_flag;
3d8fe307	65	struct cds_ja_inode;
511b4dcc MD	66
	67	/*
	68	* Shadow node contains mutex and call_rcu head associated with a node.
	69	*/
d96bfb0d	70	struct cds_ja_shadow_node {
d22c185b	71	struct cds_lfht_node ht_node; /* hash table node */
3d8fe307	72	struct cds_ja_inode_flag node_flag; / reverse mapping and hash table key */
e1db2db5 MD	73	/*
	74	* mutual exclusion on all nodes belonging to the same tree
	75	* position (e.g. both nodes before and after recompaction
	76	* use the same lock).
	77	*/
	78	pthread_mutex_t *lock;
	79	unsigned int nr_child; /* number of children in node */
d22c185b	80	struct rcu_head head; /* for deferred node and shadow node reclaim */
f07b240f	81	int fallback_removal_count; /* removals left keeping fallback */
3d8fe307 MD	82	int level; /* level in the tree */
3d8fe307 MD	83	struct cds_ja ja; / toplevel judy array */
511b4dcc MD	84	};
511b4dcc MD	85
d96bfb0d	86	struct cds_ja {
b4540e8a MD	87	struct cds_ja_inode_flag *root;
	88	unsigned int tree_depth;
	89	uint64_t key_max;
511b4dcc	90	/*
e1db2db5 MD	91	* We use a hash table to associate node keys to their
	92	* respective shadow node. This helps reducing lookup hot path
	93	* cache footprint, especially for very small nodes.
511b4dcc MD	94	*/
511b4dcc MD	95	struct cds_lfht *ht;
f07b240f	96	unsigned long nr_fallback; /* Number of fallback nodes used */
354981c2 MD	97
	98	/* For debugging */
	99	unsigned long node_fallback_count_distribution[JA_ENTRY_PER_NODE];
	100	unsigned long nr_nodes_allocated, nr_nodes_freed;
511b4dcc	101	};
61009379	102
3d8fe307 MD	103	static inline
	104	struct cds_ja_inode_flag ja_node_flag(struct cds_ja_inode node,
	105	unsigned long type)
	106	{
	107	assert(type < (1UL << JA_TYPE_BITS));
	108	return (struct cds_ja_inode_flag *) (((unsigned long) node) \| type);
	109	}
	110
	111	static inline
b1a90ce3 MD	112	struct cds_ja_inode_flag ja_node_flag_pool_1d(struct cds_ja_inode node,
b1a90ce3 MD	113	unsigned long type, unsigned long bitsel)
3d8fe307	114	{
b1a90ce3 MD	115	assert(type < (1UL << JA_TYPE_BITS));
	116	assert(bitsel < JA_BITS_PER_BYTE);
	117	return (struct cds_ja_inode_flag *) (((unsigned long) node) \| (bitsel << JA_TYPE_BITS) \| type);
3d8fe307 MD	118	}
3d8fe307 MD	119
19ddcd04 MD	120	static inline
	121	struct cds_ja_inode_flag ja_node_flag_pool_2d(struct cds_ja_inode node,
	122	unsigned long type, unsigned int bitsel[2])
	123	{
	124	assert(type < (1UL << JA_TYPE_BITS));
	125	assert(bitsel[0] < JA_BITS_PER_BYTE);
	126	assert(bitsel[1] < JA_BITS_PER_BYTE);
	127	return (struct cds_ja_inode_flag *) (((unsigned long) node) \| (bitsel[0] << (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE)) \| (bitsel[1] << JA_TYPE_BITS) \| type);
	128	}
	129
3d8fe307	130	static inline
b1a90ce3	131	unsigned long ja_node_pool_1d_bitsel(struct cds_ja_inode_flag *node)
3d8fe307	132	{
b1a90ce3 MD	133	return ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
b1a90ce3 MD	134	}
3d8fe307	135
b1a90ce3 MD	136	static inline
	137	void ja_node_pool_2d_bitsel(struct cds_ja_inode_flag node, unsigned long bits)
	138	{
	139	bits[0] = ((unsigned long) node & JA_POOL_2D_MASK) >> (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE);
	140	bits[1] = ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
3d8fe307 MD	141	}
3d8fe307 MD	142
1cee749c MD	143	/* Hardcoded pool indexes for fast path */
	144	#define RCU_JA_POOL_IDX_5 5
	145	#define RCU_JA_POOL_IDX_6 6
	146	static inline
	147	struct cds_ja_inode ja_node_ptr(struct cds_ja_inode_flag node)
	148	{
	149	unsigned long v, type_idx;
	150
	151	if (!node)
	152	return NULL; /* RCU_JA_NULL */
	153	v = (unsigned long) node;
	154	type_idx = v & JA_TYPE_MASK;
	155
	156	switch (type_idx) {
	157	case RCU_JA_POOL_IDX_5:
	158	v &= ~(JA_POOL_1D_MASK \| JA_TYPE_MASK);
	159	break;
	160	case RCU_JA_POOL_IDX_6:
	161	v &= ~(JA_POOL_2D_MASK \| JA_POOL_1D_MASK \| JA_TYPE_MASK);
	162	break;
	163	default:
	164	/* RCU_JA_LINEAR or RCU_JA_PIGEON */
	165	v &= JA_PTR_MASK;
	166	break;
	167	}
0c0113e0	168	return (struct cds_ja_inode *) v;
1cee749c	169	}
b1a90ce3 MD	170
b1a90ce3 MD	171	__attribute__((visibility("protected")))
1cee749c	172	unsigned long ja_node_type(struct cds_ja_inode_flag *node);
b1a90ce3	173
3d8fe307 MD	174	__attribute__((visibility("protected")))
	175	void rcuja_free_all_children(struct cds_ja_shadow_node *shadow_node,
	176	struct cds_ja_inode_flag *node_flag,
dc0e9798	177	void (free_node_cb)(struct cds_ja_node node));
3d8fe307	178
25fde237	179	__attribute__((visibility("protected")))
d96bfb0d	180	struct cds_ja_shadow_node rcuja_shadow_lookup_lock(struct cds_lfht ht,
3d8fe307	181	struct cds_ja_inode_flag *node_flag);
be9a7474	182
25fde237	183	__attribute__((visibility("protected")))
d96bfb0d	184	void rcuja_shadow_unlock(struct cds_ja_shadow_node *shadow_node);
be9a7474	185
25fde237	186	__attribute__((visibility("protected")))
f07b240f	187	struct cds_ja_shadow_node rcuja_shadow_set(struct cds_lfht ht,
3d8fe307 MD	188	struct cds_ja_inode_flag *new_node_flag,
3d8fe307 MD	189	struct cds_ja_shadow_node *inherit_from,
48cbe001	190	struct cds_ja *ja, int level);
e1db2db5 MD	191
	192	/* rcuja_shadow_clear flags */
	193	enum {
	194	RCUJA_SHADOW_CLEAR_FREE_NODE = (1U << 0),
	195	RCUJA_SHADOW_CLEAR_FREE_LOCK = (1U << 1),
	196	};
	197
be9a7474	198	__attribute__((visibility("protected")))
e1db2db5	199	int rcuja_shadow_clear(struct cds_lfht *ht,
3d8fe307	200	struct cds_ja_inode_flag *node_flag,
a2a7ff59	201	struct cds_ja_shadow_node *shadow_node,
e1db2db5	202	unsigned int flags);
be9a7474 MD	203
	204	__attribute__((visibility("protected")))
	205	void rcuja_shadow_prune(struct cds_lfht *ht,
3d8fe307	206	unsigned int flags,
dc0e9798	207	void (free_node_cb)(struct cds_ja_node node));
be9a7474	208
25fde237	209	__attribute__((visibility("protected")))
5eb692c0	210	struct cds_lfht rcuja_create_ht(const struct rcu_flavor_struct flavor);
be9a7474	211
25fde237	212	__attribute__((visibility("protected")))
be9a7474	213	int rcuja_delete_ht(struct cds_lfht *ht);
25fde237	214
48cbe001	215	__attribute__((visibility("protected")))
354981c2	216	void free_cds_ja_node(struct cds_ja ja, struct cds_ja_inode node);
48cbe001	217
03ec1aeb MD	218	/*
	219	* Iterate through duplicates returned by cds_ja_lookup*()
	220	* Receives a struct cds_ja_node * as parameter, which is used as start
	221	* of duplicate list and loop cursor.
	222	*/
	223	#define cds_ja_for_each_duplicate(pos) \
	224	for (; (pos) != NULL; (pos) = (pos)->next)
	225
	226	/*
	227	* Iterate through duplicates returned by cds_ja_lookup*()
	228	* Safe against removal of entries during traversal.
	229	*/
	230	#define cds_ja_for_each_duplicate_safe(_pos, _next) \
	231	for (; (_pos) != NULL ? ((_next) = (_pos)->next, 1) : 0; \
	232	(_pos) = (_next))
	233
1a0c0717	234	//#define DEBUG
a2a7ff59	235
7d67da99 MD	236	#ifdef __linux__
	237	#include <syscall.h>
	238	#endif
	239
	240	#if defined(_syscall0)
	241	_syscall0(pid_t, gettid)
	242	#elif defined(__NR_gettid)
	243	static inline pid_t gettid(void)
	244	{
	245	return syscall(__NR_gettid);
	246	}
	247	#else
	248	#warning "use pid as tid"
	249	static inline pid_t gettid(void)
	250	{
	251	return getpid();
	252	}
	253	#endif
	254
a2a7ff59	255	#ifdef DEBUG
7d67da99 MD	256	#define dbg_printf(fmt, args...) \
	257	fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \
	258	(unsigned long) gettid(), __func__, \
	259	__FILE__, __LINE__, ## args)
a2a7ff59 MD	260	#else
	261	#define dbg_printf(fmt, args...) \
	262	do { \
	263	/* do nothing but check printf format */ \
	264	if (0) \
7d67da99 MD	265	fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \
	266	(unsigned long) gettid(), __func__, \
	267	__FILE__, __LINE__, ## args); \
a2a7ff59 MD	268	} while (0)
	269	#endif
	270
61009379	271	#endif /* _URCU_RCUJA_INTERNAL_H */