[urcu.git] / urcu / rcuwfqueue.h

#ifndef _URCU_RCUWFQUEUE_H
#define _URCU_RCUWFQUEUE_H

/*
 * rcuwfqueue.h
 *
 * Userspace RCU library - RCU Queue with Wait-Free Enqueue/Blocking Dequeue
 *
 * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <urcu/urcu_ref.h>
#include <assert.h>

#ifdef __cplusplus
extern "C" {
#endif

#if (!defined(_GNU_SOURCE) && !defined(_LGPL_SOURCE))
#error "Dynamic loader LGPL wrappers not implemented yet"
#endif

/*
 * RCU queue with wait-free enqueue/blocking dequeue using reference counting.
 * Enqueue and dequeue operations hold a RCU read lock to deal with cmpxchg ABA
 * problem. This implementation keeps a dummy head node to ensure we can always
 * update the queue locklessly. Given that this is a queue, the dummy head node
 * must always advance as we dequeue entries. Therefore, we keep a reference
 * count on each entry we are dequeueing, so they can be kept as dummy head node
 * until the next dequeue, at which point their reference count will be
 * decremented.
 */

#define URCU_WFQ_PERMANENT_REF		128

struct rcu_wfq_node {
	struct rcu_wfq_node *next;
	struct urcu_ref ref;
};

struct rcu_wfq_queue {
	struct rcu_wfq_node *head, *tail;
	struct rcu_wfq_node init;	/* Dummy initialization node */
};

void rcu_wfq_node_init(struct rcu_wfq_node *node)
{
	node->next = NULL;
	urcu_ref_init(&node->ref);
}

void rcu_wfq_init(struct rcu_wfq_queue *q)
{
	rcu_wfq_node_init(&q->init);
	/* Make sure the initial node is never freed. */
	urcu_ref_set(&q->init.ref, URCU_WFQ_PERMANENT_REF);
	/* Set queue end */
	q->head = q->tail = &q->init;
}

void rcu_wfq_enqueue(struct rcu_wfq_queue *q, struct rcu_wfq_node *node)
{
	struct rcu_wfq_node *old_tail;

	urcu_ref_get(&node->ref);
	/*
	 * uatomic_xchg() implicit memory barrier orders earlier stores to node
	 * (setting it to NULL and incrementing the refcount) before
	 * publication.
	 */
	old_tail = uatomic_xchg(&q->tail, node);
	/*
	 * At this point, dequeuers see a NULL old_tail->next, which indicates
	 * end of queue. The following store will append "node" to the queue
	 * from a dequeuer perspective.
	 */
	STORE_SHARED(old_tail->next, node);
}

/*
 * The entry returned by dequeue must be taken care of by doing a urcu_ref_put,
 * which calls the release primitive when the reference count drops to zero. A
 * grace period must be waited before performing the actual memory reclamation
 * in the release primitive. The wfq node returned by dequeue must not be
 * modified/re-used/freed until the reference count reaches zero and a grace
 * period has elapsed (after the refcount reached 0).
 *
 * No need to go on a waitqueue here, as there is no possible state in which the
 * list could cause dequeue to busy-loop needlessly while waiting for another
 * thread to be scheduled. The queue appears empty until tail->next is set by
 * enqueue.
 */
struct rcu_wfq_node *
rcu_wfq_dequeue_blocking(struct rcu_wfq_queue *q,
			 void (*release)(struct urcu_ref *))
{
	for (;;) {
		struct rcu_wfq_node *head, *next;

		rcu_read_lock();
		head = rcu_dereference(q->head);
		next = rcu_dereference(head->next);
		if (next) {
			if (uatomic_cmpxchg(&q->head, head, next) == head) {
				rcu_read_unlock();
				urcu_ref_put(&head->ref, release);
				return next;
			} else {
				/* Concurrently pushed, retry */
				rcu_read_unlock();
				continue;
			}
		} else {
			/* Empty */
			rcu_read_unlock();
			return NULL;
		}
	}
}

#ifdef __cplusplus
}
#endif

#endif /* _URCU_RCUWFQUEUE_H */
Commit	Line	Data
0e2e9380 MD	1	#ifndef _URCU_RCUWFQUEUE_H
	2	#define _URCU_RCUWFQUEUE_H
	3
ee566d46 MD	4	/*
	5	* rcuwfqueue.h
	6	*
	7	* Userspace RCU library - RCU Queue with Wait-Free Enqueue/Blocking Dequeue
	8	*
	9	* Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	10	*
	11	* This library is free software; you can redistribute it and/or
	12	* modify it under the terms of the GNU Lesser General Public
	13	* License as published by the Free Software Foundation; either
	14	* version 2.1 of the License, or (at your option) any later version.
	15	*
	16	* This library is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* Lesser General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU Lesser General Public
	22	* License along with this library; if not, write to the Free Software
	23	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
	26	#include <urcu/urcu_ref.h>
	27	#include <assert.h>
	28
0e2e9380 MD	29	#ifdef __cplusplus
	30	extern "C" {
	31	#endif
	32
ee566d46 MD	33	#if (!defined(_GNU_SOURCE) && !defined(_LGPL_SOURCE))
	34	#error "Dynamic loader LGPL wrappers not implemented yet"
	35	#endif
	36
	37	/*
	38	* RCU queue with wait-free enqueue/blocking dequeue using reference counting.
	39	* Enqueue and dequeue operations hold a RCU read lock to deal with cmpxchg ABA
	40	* problem. This implementation keeps a dummy head node to ensure we can always
	41	* update the queue locklessly. Given that this is a queue, the dummy head node
	42	* must always advance as we dequeue entries. Therefore, we keep a reference
	43	* count on each entry we are dequeueing, so they can be kept as dummy head node
	44	* until the next dequeue, at which point their reference count will be
	45	* decremented.
	46	*/
	47
	48	#define URCU_WFQ_PERMANENT_REF 128
	49
	50	struct rcu_wfq_node {
	51	struct rcu_wfq_node *next;
	52	struct urcu_ref ref;
	53	};
	54
	55	struct rcu_wfq_queue {
	56	struct rcu_wfq_node head, tail;
	57	struct rcu_wfq_node init; /* Dummy initialization node */
	58	};
	59
	60	void rcu_wfq_node_init(struct rcu_wfq_node *node)
	61	{
	62	node->next = NULL;
	63	urcu_ref_init(&node->ref);
	64	}
	65
	66	void rcu_wfq_init(struct rcu_wfq_queue *q)
	67	{
	68	rcu_wfq_node_init(&q->init);
	69	/* Make sure the initial node is never freed. */
	70	urcu_ref_set(&q->init.ref, URCU_WFQ_PERMANENT_REF);
	71	/* Set queue end */
	72	q->head = q->tail = &q->init;
	73	}
	74
	75	void rcu_wfq_enqueue(struct rcu_wfq_queue q, struct rcu_wfq_node node)
	76	{
	77	struct rcu_wfq_node *old_tail;
	78
	79	urcu_ref_get(&node->ref);
	80	/*
	81	* uatomic_xchg() implicit memory barrier orders earlier stores to node
	82	* (setting it to NULL and incrementing the refcount) before
	83	* publication.
	84	*/
	85	old_tail = uatomic_xchg(&q->tail, node);
	86	/*
	87	* At this point, dequeuers see a NULL old_tail->next, which indicates
	88	* end of queue. The following store will append "node" to the queue
	89	* from a dequeuer perspective.
	90	*/
	91	STORE_SHARED(old_tail->next, node);
	92	}
	93
	94	/*
	95	* The entry returned by dequeue must be taken care of by doing a urcu_ref_put,
	96	* which calls the release primitive when the reference count drops to zero. A
97	* grace period must be waited before performing the actual memory reclamation
98	* in the release primitive. The wfq node returned by dequeue must not be
99	* modified/re-used/freed until the reference count reaches zero and a grace
100	* period has elapsed (after the refcount reached 0).
101	*
0e2e9380 MD	102	* No need to go on a waitqueue here, as there is no possible state in which the
	103	* list could cause dequeue to busy-loop needlessly while waiting for another
	104	* thread to be scheduled. The queue appears empty until tail->next is set by
	105	* enqueue.
ee566d46 MD	106	*/
	107	struct rcu_wfq_node *
	108	rcu_wfq_dequeue_blocking(struct rcu_wfq_queue *q,
	109	void (release)(struct urcu_ref ))
	110	{
	111	for (;;) {
	112	struct rcu_wfq_node head, next;
	113
	114	rcu_read_lock();
	115	head = rcu_dereference(q->head);
	116	next = rcu_dereference(head->next);
	117	if (next) {
	118	if (uatomic_cmpxchg(&q->head, head, next) == head) {
	119	rcu_read_unlock();
	120	urcu_ref_put(&head->ref, release);
	121	return next;
	122	} else {
	123	/* Concurrently pushed, retry */
	124	rcu_read_unlock();
	125	continue;
	126	}
	127	} else {
	128	/* Empty */
	129	rcu_read_unlock();
	130	return NULL;
	131	}
	132	}
	133	}
0e2e9380 MD	134
	135	#ifdef __cplusplus
	136	}
	137	#endif
	138
	139	#endif /* _URCU_RCUWFQUEUE_H */