[userspace-rcu.git] / rcuja / rcuja-internal.h

#ifndef _URCU_RCUJA_INTERNAL_H
#define _URCU_RCUJA_INTERNAL_H

/*
 * rcuja/rcuja-internal.h
 *
 * Userspace RCU library - RCU Judy Array Internal Header
 *
 * Copyright 2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#define _GNU_SOURCE
#include <pthread.h>
#include <stdio.h>
#include <inttypes.h>
#include <unistd.h>
#include <urcu/rculfhash.h>

/*
 * Number of least significant pointer bits reserved to represent the
 * child type.
 */
#define JA_TYPE_BITS	3
#define JA_TYPE_MAX_NR	(1UL << JA_TYPE_BITS)
#define JA_TYPE_MASK	(JA_TYPE_MAX_NR - 1)
#define JA_PTR_MASK	(~JA_TYPE_MASK)

#define JA_ENTRY_PER_NODE	256UL
#define JA_LOG2_BITS_PER_BYTE	3U
#define JA_BITS_PER_BYTE	(1U << JA_LOG2_BITS_PER_BYTE)

#define JA_POOL_1D_MASK	((JA_BITS_PER_BYTE - 1) << JA_TYPE_BITS)
#define JA_POOL_2D_MASK	(JA_POOL_1D_MASK << JA_LOG2_BITS_PER_BYTE)

#define JA_MAX_DEPTH	9	/* Maximum depth, including leafs */

/*
 * Entry for NULL node is at index 8 of the table. It is never encoded
 * in flags.
 */
#define NODE_INDEX_NULL		8

/*
 * Number of removals needed on a fallback node before we try to shrink
 * it.
 */
#define JA_FALLBACK_REMOVAL_COUNT	8

/* Never declared. Opaque type used to store flagged node pointers. */
struct cds_ja_inode_flag;
struct cds_ja_inode;

/*
 * Shadow node contains mutex and call_rcu head associated with a node.
 */
struct cds_ja_shadow_node {
	struct cds_lfht_node ht_node;	/* hash table node */
	struct cds_ja_inode_flag *node_flag;	/* reverse mapping and hash table key */
	/*
	 * mutual exclusion on all nodes belonging to the same tree
	 * position (e.g. both nodes before and after recompaction
	 * use the same lock).
	 */
	pthread_mutex_t *lock;
	unsigned int nr_child;		/* number of children in node */
	struct rcu_head head;		/* for deferred node and shadow node reclaim */
	int fallback_removal_count;	/* removals left keeping fallback */
	int level;			/* level in the tree */
	struct cds_ja *ja;		/* toplevel judy array */
};

struct cds_ja {
	struct cds_ja_inode_flag *root;
	unsigned int tree_depth;
	uint64_t key_max;
	/*
	 * We use a hash table to associate node keys to their
	 * respective shadow node. This helps reducing lookup hot path
	 * cache footprint, especially for very small nodes.
	 */
	struct cds_lfht *ht;
	unsigned long nr_fallback;	/* Number of fallback nodes used */
};

static inline
struct cds_ja_inode_flag *ja_node_flag(struct cds_ja_inode *node,
		unsigned long type)
{
	assert(type < (1UL << JA_TYPE_BITS));
	return (struct cds_ja_inode_flag *) (((unsigned long) node) | type);
}

static inline
struct cds_ja_inode_flag *ja_node_flag_pool_1d(struct cds_ja_inode *node,
		unsigned long type, unsigned long bitsel)
{
	assert(type < (1UL << JA_TYPE_BITS));
	assert(bitsel < JA_BITS_PER_BYTE);
	return (struct cds_ja_inode_flag *) (((unsigned long) node) | (bitsel << JA_TYPE_BITS) | type);
}

static inline
unsigned long ja_node_pool_1d_bitsel(struct cds_ja_inode_flag *node)
{
	return ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
}

static inline
void ja_node_pool_2d_bitsel(struct cds_ja_inode_flag *node, unsigned long *bits)
{
	bits[0] = ((unsigned long) node & JA_POOL_2D_MASK) >> (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE);
	bits[1] = ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
}

__attribute__((visibility("protected")))
unsigned long ja_node_type(struct cds_ja_inode_flag *node);

__attribute__((visibility("protected")))
struct cds_ja_inode *ja_node_ptr(struct cds_ja_inode_flag *node);

__attribute__((visibility("protected")))
void rcuja_free_all_children(struct cds_ja_shadow_node *shadow_node,
		struct cds_ja_inode_flag *node_flag,
		void (*free_node_cb)(struct rcu_head *head));

__attribute__((visibility("protected")))
struct cds_ja_shadow_node *rcuja_shadow_lookup_lock(struct cds_lfht *ht,
		struct cds_ja_inode_flag *node_flag);

__attribute__((visibility("protected")))
void rcuja_shadow_unlock(struct cds_ja_shadow_node *shadow_node);

__attribute__((visibility("protected")))
struct cds_ja_shadow_node *rcuja_shadow_set(struct cds_lfht *ht,
		struct cds_ja_inode_flag *new_node_flag,
		struct cds_ja_shadow_node *inherit_from,
		struct cds_ja *ja);

/* rcuja_shadow_clear flags */
enum {
	RCUJA_SHADOW_CLEAR_FREE_NODE = (1U << 0),
	RCUJA_SHADOW_CLEAR_FREE_LOCK = (1U << 1),
};

__attribute__((visibility("protected")))
int rcuja_shadow_clear(struct cds_lfht *ht,
		struct cds_ja_inode_flag *node_flag,
		struct cds_ja_shadow_node *shadow_node,
		unsigned int flags);

__attribute__((visibility("protected")))
void rcuja_shadow_prune(struct cds_lfht *ht,
		unsigned int flags,
		void (*free_node_cb)(struct rcu_head *head));

__attribute__((visibility("protected")))
struct cds_lfht *rcuja_create_ht(const struct rcu_flavor_struct *flavor);

__attribute__((visibility("protected")))
int rcuja_delete_ht(struct cds_lfht *ht);

//#define DEBUG

#ifdef __linux__
#include <syscall.h>
#endif

#if defined(_syscall0)
_syscall0(pid_t, gettid)
#elif defined(__NR_gettid)
static inline pid_t gettid(void)
{
	return syscall(__NR_gettid);
}
#else
#warning "use pid as tid"
static inline pid_t gettid(void)
{
	return getpid();
}
#endif

#ifdef DEBUG
#define dbg_printf(fmt, args...)				\
	fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt,	\
		(unsigned long) gettid(), __func__,		\
		__FILE__, __LINE__, ## args)
#else
#define dbg_printf(fmt, args...)				\
do {								\
	/* do nothing but check printf format */		\
	if (0)							\
		fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \
			(unsigned long) gettid(), __func__,	\
			__FILE__, __LINE__, ## args);		\
} while (0)
#endif

#endif /* _URCU_RCUJA_INTERNAL_H */
Commit	Line	Data
61009379 MD	1	#ifndef _URCU_RCUJA_INTERNAL_H
	2	#define _URCU_RCUJA_INTERNAL_H
	3
	4	/*
	5	* rcuja/rcuja-internal.h
	6	*
	7	* Userspace RCU library - RCU Judy Array Internal Header
	8	*
	9	* Copyright 2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	10	*
	11	* This library is free software; you can redistribute it and/or
	12	* modify it under the terms of the GNU Lesser General Public
	13	* License as published by the Free Software Foundation; either
	14	* version 2.1 of the License, or (at your option) any later version.
	15	*
	16	* This library is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* Lesser General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU Lesser General Public
	22	* License along with this library; if not, write to the Free Software
	23	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
7d67da99	26	#define _GNU_SOURCE
511b4dcc	27	#include <pthread.h>
a2a7ff59 MD	28	#include <stdio.h>
a2a7ff59 MD	29	#include <inttypes.h>
7d67da99	30	#include <unistd.h>
511b4dcc MD	31	#include <urcu/rculfhash.h>
511b4dcc MD	32
3d8fe307 MD	33	/*
	34	* Number of least significant pointer bits reserved to represent the
	35	* child type.
	36	*/
	37	#define JA_TYPE_BITS 3
	38	#define JA_TYPE_MAX_NR (1UL << JA_TYPE_BITS)
	39	#define JA_TYPE_MASK (JA_TYPE_MAX_NR - 1)
	40	#define JA_PTR_MASK (~JA_TYPE_MASK)
	41
	42	#define JA_ENTRY_PER_NODE 256UL
	43	#define JA_LOG2_BITS_PER_BYTE 3U
	44	#define JA_BITS_PER_BYTE (1U << JA_LOG2_BITS_PER_BYTE)
	45
b1a90ce3 MD	46	#define JA_POOL_1D_MASK ((JA_BITS_PER_BYTE - 1) << JA_TYPE_BITS)
	47	#define JA_POOL_2D_MASK (JA_POOL_1D_MASK << JA_LOG2_BITS_PER_BYTE)
	48
3d8fe307 MD	49	#define JA_MAX_DEPTH 9 /* Maximum depth, including leafs */
	50
	51	/*
	52	* Entry for NULL node is at index 8 of the table. It is never encoded
	53	* in flags.
	54	*/
	55	#define NODE_INDEX_NULL 8
	56
	57	/*
	58	* Number of removals needed on a fallback node before we try to shrink
	59	* it.
	60	*/
	61	#define JA_FALLBACK_REMOVAL_COUNT 8
	62
511b4dcc	63	/* Never declared. Opaque type used to store flagged node pointers. */
b4540e8a	64	struct cds_ja_inode_flag;
3d8fe307	65	struct cds_ja_inode;
511b4dcc MD	66
	67	/*
	68	* Shadow node contains mutex and call_rcu head associated with a node.
	69	*/
d96bfb0d	70	struct cds_ja_shadow_node {
d22c185b	71	struct cds_lfht_node ht_node; /* hash table node */
3d8fe307	72	struct cds_ja_inode_flag node_flag; / reverse mapping and hash table key */
e1db2db5 MD	73	/*
	74	* mutual exclusion on all nodes belonging to the same tree
	75	* position (e.g. both nodes before and after recompaction
	76	* use the same lock).
	77	*/
	78	pthread_mutex_t *lock;
	79	unsigned int nr_child; /* number of children in node */
d22c185b	80	struct rcu_head head; /* for deferred node and shadow node reclaim */
f07b240f	81	int fallback_removal_count; /* removals left keeping fallback */
3d8fe307 MD	82	int level; /* level in the tree */
3d8fe307 MD	83	struct cds_ja ja; / toplevel judy array */
511b4dcc MD	84	};
511b4dcc MD	85
d96bfb0d	86	struct cds_ja {
b4540e8a MD	87	struct cds_ja_inode_flag *root;
	88	unsigned int tree_depth;
	89	uint64_t key_max;
511b4dcc	90	/*
e1db2db5 MD	91	* We use a hash table to associate node keys to their
	92	* respective shadow node. This helps reducing lookup hot path
	93	* cache footprint, especially for very small nodes.
511b4dcc MD	94	*/
511b4dcc MD	95	struct cds_lfht *ht;
f07b240f	96	unsigned long nr_fallback; /* Number of fallback nodes used */
511b4dcc	97	};
61009379	98
3d8fe307 MD	99	static inline
	100	struct cds_ja_inode_flag ja_node_flag(struct cds_ja_inode node,
	101	unsigned long type)
	102	{
	103	assert(type < (1UL << JA_TYPE_BITS));
	104	return (struct cds_ja_inode_flag *) (((unsigned long) node) \| type);
	105	}
	106
	107	static inline
b1a90ce3 MD	108	struct cds_ja_inode_flag ja_node_flag_pool_1d(struct cds_ja_inode node,
b1a90ce3 MD	109	unsigned long type, unsigned long bitsel)
3d8fe307	110	{
b1a90ce3 MD	111	assert(type < (1UL << JA_TYPE_BITS));
	112	assert(bitsel < JA_BITS_PER_BYTE);
	113	return (struct cds_ja_inode_flag *) (((unsigned long) node) \| (bitsel << JA_TYPE_BITS) \| type);
3d8fe307 MD	114	}
	115
	116	static inline
b1a90ce3	117	unsigned long ja_node_pool_1d_bitsel(struct cds_ja_inode_flag *node)
3d8fe307	118	{
b1a90ce3 MD	119	return ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
b1a90ce3 MD	120	}
3d8fe307	121
b1a90ce3 MD	122	static inline
	123	void ja_node_pool_2d_bitsel(struct cds_ja_inode_flag node, unsigned long bits)
	124	{
	125	bits[0] = ((unsigned long) node & JA_POOL_2D_MASK) >> (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE);
	126	bits[1] = ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
3d8fe307 MD	127	}
3d8fe307 MD	128
b1a90ce3 MD	129	__attribute__((visibility("protected")))
	130	unsigned long ja_node_type(struct cds_ja_inode_flag *node);
	131
	132	__attribute__((visibility("protected")))
	133	struct cds_ja_inode ja_node_ptr(struct cds_ja_inode_flag node);
	134
3d8fe307 MD	135	__attribute__((visibility("protected")))
	136	void rcuja_free_all_children(struct cds_ja_shadow_node *shadow_node,
	137	struct cds_ja_inode_flag *node_flag,
	138	void (free_node_cb)(struct rcu_head head));
	139
25fde237	140	__attribute__((visibility("protected")))
d96bfb0d	141	struct cds_ja_shadow_node rcuja_shadow_lookup_lock(struct cds_lfht ht,
3d8fe307	142	struct cds_ja_inode_flag *node_flag);
be9a7474	143
25fde237	144	__attribute__((visibility("protected")))
d96bfb0d	145	void rcuja_shadow_unlock(struct cds_ja_shadow_node *shadow_node);
be9a7474	146
25fde237	147	__attribute__((visibility("protected")))
f07b240f	148	struct cds_ja_shadow_node rcuja_shadow_set(struct cds_lfht ht,
3d8fe307 MD	149	struct cds_ja_inode_flag *new_node_flag,
	150	struct cds_ja_shadow_node *inherit_from,
	151	struct cds_ja *ja);
e1db2db5 MD	152
	153	/* rcuja_shadow_clear flags */
	154	enum {
	155	RCUJA_SHADOW_CLEAR_FREE_NODE = (1U << 0),
	156	RCUJA_SHADOW_CLEAR_FREE_LOCK = (1U << 1),
	157	};
	158
be9a7474	159	__attribute__((visibility("protected")))
e1db2db5	160	int rcuja_shadow_clear(struct cds_lfht *ht,
3d8fe307	161	struct cds_ja_inode_flag *node_flag,
a2a7ff59	162	struct cds_ja_shadow_node *shadow_node,
e1db2db5	163	unsigned int flags);
be9a7474 MD	164
	165	__attribute__((visibility("protected")))
	166	void rcuja_shadow_prune(struct cds_lfht *ht,
3d8fe307 MD	167	unsigned int flags,
3d8fe307 MD	168	void (free_node_cb)(struct rcu_head head));
be9a7474	169
25fde237	170	__attribute__((visibility("protected")))
5eb692c0	171	struct cds_lfht rcuja_create_ht(const struct rcu_flavor_struct flavor);
be9a7474	172
25fde237	173	__attribute__((visibility("protected")))
be9a7474	174	int rcuja_delete_ht(struct cds_lfht *ht);
25fde237	175
1a0c0717	176	//#define DEBUG
a2a7ff59	177
7d67da99 MD	178	#ifdef __linux__
	179	#include <syscall.h>
	180	#endif
	181
	182	#if defined(_syscall0)
	183	_syscall0(pid_t, gettid)
	184	#elif defined(__NR_gettid)
	185	static inline pid_t gettid(void)
	186	{
	187	return syscall(__NR_gettid);
	188	}
	189	#else
	190	#warning "use pid as tid"
	191	static inline pid_t gettid(void)
	192	{
	193	return getpid();
	194	}
	195	#endif
	196
a2a7ff59	197	#ifdef DEBUG
7d67da99 MD	198	#define dbg_printf(fmt, args...) \
	199	fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \
	200	(unsigned long) gettid(), __func__, \
	201	__FILE__, __LINE__, ## args)
a2a7ff59 MD	202	#else
	203	#define dbg_printf(fmt, args...) \
	204	do { \
	205	/* do nothing but check printf format */ \
	206	if (0) \
7d67da99 MD	207	fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \
	208	(unsigned long) gettid(), __func__, \
	209	__FILE__, __LINE__, ## args); \
a2a7ff59 MD	210	} while (0)
	211	#endif
	212
61009379	213	#endif /* _URCU_RCUJA_INTERNAL_H */