4 * Userspace RCU library - Red-Black Tree
6 * Copyright (c) 2010 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
8 * This library is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
13 * This library is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with this library; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 * Implementation of RCU-adapted data structures and operations based on the RB
23 * tree algorithms found in chapter 12 of:
25 * Thomas H. Cormen, Charles E. Leiserson, Ronald L. Rivest, and
26 * Clifford Stein. Introduction to Algorithms, Third Edition. The MIT
27 * Press, September 2009.
39 #include <urcu/rcurbtree.h>
40 #include <urcu-pointer.h>
41 #include <urcu-call-rcu.h>
42 #include <urcu/compiler.h>
45 * Explanation of next/prev walk coherency and search coherency when
46 * performed concurrently with updates.
48 * next/prev walk coherency with respect to concurrent updates:
50 * There are 3 scenarios for which we need to model and validate this:
51 * rotation, transplant and "teleportation" (the latter being a remote
52 * transplant in a remove non-nil case).
54 * - rotation left (right is symmetric)
55 * xl and yr point to the same parent nodes before/after left
56 * rotation. yll and ylr also point to the same parent node
57 * before/after left rotation.
58 * As we are copying x, y and yl (the 3 nodes which parent/child
59 * relationship are changed) to "new" version of this node cluster,
60 * all external references to the cluster either point to the old
61 * cluster or the new one. If we take this cluster as a "black box"
62 * from the point of view of next/prev traversal, all we have to
63 * ensure is that the old and the new cluster behave in the exact same
64 * way with respect to traversal order.
67 * In this operation, we transplant a copy of "v" into its parent
68 * location (u), thus replacing it. The children of "v", vl and vr,
69 * still point to v (new version) after the transplant, so it does not
70 * change the behavior when considering the next/prev traversal. "v"
71 * being copied to a new version ensures that the parent pointers of v
72 * are pointing to its new parent (parent of u) before it is published
73 * to readers (by setting the child pointer of u's parent to the new
77 * This one is probably the most tricky and will require some ascii
80 * We want to remove z from this tree:
98 * What we are going to do is to "teleport" y into z's location,
99 * reparenting yr to b. We are taking care to create a new cluster
100 * copy that is isolated from any reader. We will represent the new
101 * members of the cluster with capital letters.
117 * In this transient state, we notice that the pointers within the
118 * cluster all point to the new cluster nodes, and they point to the
119 * correct external nodes. However, no external pointer point to the
120 * cluster (yet). The first pointer to point to this cluster will be
121 * "zp->right". It will therefore make the cluster visible for search.
123 * In this intermediate state, we can walk through the new cluster
124 * when coming from the top (in a next/prev traversal), but can come
125 * back to the old cluster when going back up from the children nodes.
126 * All we have to ensure is that the two clusters, taken as a black
127 * box from a next/prev traversal perspective, yield to the exact same
130 * Search coherency with concurrent updates:
132 * Simple "search" (only going down the tree) is also handled by this
133 * cluster scheme. The explanation is a subset of the prev/next
134 * explanation, where we don't have to care about the intermediate
135 * stages where the children point to the old cluster, because we only
136 * ever use the top level pointers to go down into the children nodes,
137 * we never go back up. So by simply making sure that all the cluster
138 * internal nodes pointers are setup correctly before making the
139 * cluster visible to the readers (by setting the parent pointer to
140 * the topmost new node in the cluster), we are sure that readers will
141 * see a coherent view of the cluster at all times.
145 #define dbg_printf(args...) printf(args)
146 #define dbg_usleep(usecs) usleep(usecs)
148 #define dbg_printf(args...)
149 #define dbg_usleep(usecs)
153 * Undefine this to enable the non-RCU rotate and transplant functions
154 * (for debugging). Note that these versions don't support the tree
155 * max_end updates, so lookups must be performed with
156 * rcu_rbtree_search_begin_key when using this debug facility.
158 #define RBTREE_RCU_SUPPORT_ROTATE_LEFT
159 #define RBTREE_RCU_SUPPORT_ROTATE_RIGHT
160 #define RBTREE_RCU_SUPPORT_TRANSPLANT
163 static pthread_mutex_t test_mutex
= PTHREAD_MUTEX_INITIALIZER
;
164 static pthread_mutex_t outer_mutex
= PTHREAD_MUTEX_INITIALIZER
;
167 void lock_outer_mutex(void)
169 pthread_mutex_lock(&outer_mutex
);
173 void unlock_outer_mutex(void)
175 pthread_mutex_unlock(&outer_mutex
);
179 void lock_test_mutex(void)
181 pthread_mutex_lock(&test_mutex
);
185 void unlock_test_mutex(void)
187 pthread_mutex_unlock(&test_mutex
);
192 void set_parent(struct rcu_rbtree_node
*node
,
193 struct rcu_rbtree_node
*parent
,
196 _CMM_STORE_SHARED(node
->parent
, ((unsigned long) parent
) | pos
);
200 struct rcu_rbtree_node
*get_parent(struct rcu_rbtree_node
*node
)
202 return (struct rcu_rbtree_node
*) (node
->parent
& ~1UL);
206 unsigned int get_pos(struct rcu_rbtree_node
*node
)
208 return (unsigned int) (node
->parent
& 1UL);
212 struct rcu_rbtree_node
*get_parent_and_pos(struct rcu_rbtree_node
*node
,
215 unsigned long parent_pos
= rcu_dereference(node
->parent
);
217 *pos
= (unsigned int) (parent_pos
& 1UL);
218 return (struct rcu_rbtree_node
*) (parent_pos
& ~1UL);
222 void set_decay(struct rcu_rbtree_node
*x
, struct rcu_rbtree_node
*xc
)
228 struct rcu_rbtree_node
*get_decay(struct rcu_rbtree_node
*x
)
232 while (x
->decay_next
)
238 struct rcu_rbtree_node
*is_decay(struct rcu_rbtree_node
*x
)
240 return x
->decay_next
;
244 struct rcu_rbtree_node
*dup_decay_node(struct rcu_rbtree
*rbtree
,
245 struct rcu_rbtree_node
*x
)
247 struct rcu_rbtree_node
*xc
;
249 if (rcu_rbtree_is_nil(rbtree
, x
))
252 xc
= rbtree
->rballoc();
253 memcpy(xc
, x
, sizeof(struct rcu_rbtree_node
));
254 xc
->decay_next
= NULL
;
256 call_rcu(&x
->head
, rbtree
->rbfree
);
261 * Info for range lookups:
262 * Range lookup information is only valid when used when searching for
263 * ranges. It should never be used in next/prev traversal because the
264 * pointers to parents are not in sync with the parent vision of the
268 void set_left(struct rcu_rbtree
*rbtree
, struct rcu_rbtree_node
*node
,
269 struct rcu_rbtree_node
*left
)
275 void set_right(struct rcu_rbtree
*rbtree
, struct rcu_rbtree_node
*node
,
276 struct rcu_rbtree_node
*right
)
278 node
->_right
= right
;
282 void *calculate_node_max_end(struct rcu_rbtree
*rbtree
, struct rcu_rbtree_node
*node
)
287 if (!rcu_rbtree_is_nil(rbtree
, node
->_right
)) {
288 if (rbtree
->comp(max_end
, node
->_right
->max_end
) < 0)
289 max_end
= node
->_right
->max_end
;
291 if (!rcu_rbtree_is_nil(rbtree
, node
->_left
)) {
292 if (rbtree
->comp(max_end
, node
->_left
->max_end
) < 0)
293 max_end
= node
->_left
->max_end
;
300 * Deal with memory allocation errors.
301 * Can be ensured by reserving a pool of memory entries before doing the
302 * insertion, which will have to be function of number of
303 * transplantations/rotations required for the operation (which is a
304 * multiple of the tree height).
309 void show_tree(struct rcu_rbtree
*rbtree
)
311 struct rcu_rbtree_node
*node
;
313 node
= rcu_rbtree_min(rbtree
, rbtree
->root
);
314 while (!rcu_rbtree_is_nil(rbtree
, node
)) {
315 assert(!is_decay(node
));
316 printf("{ b:%lX e:%lX pb: %lX r:%lX l:%lX %s %s %s} ",
317 (unsigned long) node
->begin
,
318 (unsigned long) node
->end
,
319 (unsigned long) get_parent(node
)->begin
,
320 (unsigned long) node
->_right
->begin
,
321 (unsigned long) node
->_left
->begin
,
322 node
->color
? "red" : "black",
323 get_pos(node
) ? "right" : "left",
324 rcu_rbtree_is_nil(rbtree
, node
) ? "nil" : "");
325 node
= rcu_rbtree_next(rbtree
, node
);
330 #define check_max_end(rbtree, x) \
332 if (rcu_rbtree_is_nil(rbtree, x)) \
334 assert(rbtree->comp(x->max_end, \
335 calculate_node_max_end(rbtree, x)) == 0); \
340 void show_tree(struct rcu_rbtree
*rbtree
)
345 void check_max_end(struct rcu_rbtree
*rbtree
, struct rcu_rbtree_node
*x
)
351 struct rcu_rbtree_node
*make_nil(struct rcu_rbtree
*rbtree
)
353 return &rbtree
->nil_node
;
357 * Iterative rbtree search.
359 struct rcu_rbtree_node
*rcu_rbtree_search(struct rcu_rbtree
*rbtree
,
360 struct rcu_rbtree_node
*x
,
363 struct rcu_rbtree_node
*xl
;
365 dbg_printf("searching point 0x%lx\n", (unsigned long) point
);
366 x
= rcu_dereference(x
);
368 while (!rcu_rbtree_is_nil(rbtree
, x
)) {
370 xl
= rcu_dereference(x
->_left
);
371 dbg_printf("search x %lx x_end %lx x_max_end %lx\n", (unsigned long) x
->begin
,
372 (unsigned long) x
->end
, (unsigned long) x
->max_end
);
373 dbg_printf("search xl %lx xl_end %lx xl_max_end %lx\n", (unsigned long) xl
->begin
,
374 (unsigned long) xl
->end
, (unsigned long) xl
->max_end
);
375 if (!rcu_rbtree_is_nil(rbtree
, xl
)
376 && (rbtree
->comp(xl
->max_end
, point
) > 0)) {
377 dbg_printf("go left\n");
379 } else if (rbtree
->comp(x
->begin
, point
) <= 0
380 && rbtree
->comp(point
, x
->end
) < 0) {
381 dbg_printf("got it!\n");
383 } else if (rbtree
->comp(point
, x
->begin
) > 0) {
384 dbg_printf("go right\n");
385 x
= rcu_dereference(x
->_right
);
387 dbg_printf("not found!\n");
388 x
= make_nil(rbtree
);
391 if (rcu_rbtree_is_nil(rbtree
, x
))
392 dbg_printf("Reached bottom of tree.\n");
397 struct rcu_rbtree_node
*rcu_rbtree_search_range(struct rcu_rbtree
*rbtree
,
398 struct rcu_rbtree_node
*x
,
399 void *begin
, void *end
)
401 struct rcu_rbtree_node
*node
;
403 node
= rcu_rbtree_search(rbtree
, x
, begin
);
404 if (rcu_rbtree_is_nil(rbtree
, node
))
406 if (rbtree
->comp(node
->end
, end
) < 0)
407 return NULL
; /* High is outside lookup range */
412 * Search by exact range start value.
414 struct rcu_rbtree_node
*rcu_rbtree_search_begin_key(struct rcu_rbtree
*rbtree
,
415 struct rcu_rbtree_node
*x
,
418 x
= rcu_dereference(x
);
421 while (!rcu_rbtree_is_nil(rbtree
, x
) && (comp
= rbtree
->comp(k
, x
->begin
)) != 0) {
424 x
= rcu_dereference(x
->_left
);
426 x
= rcu_dereference(x
->_right
);
432 struct rcu_rbtree_node
*rcu_rbtree_min_dup_decay(struct rcu_rbtree
*rbtree
,
433 struct rcu_rbtree_node
*x
,
434 struct rcu_rbtree_node
**zr
)
436 struct rcu_rbtree_node
*xl
;
438 x
= rcu_dereference(x
);
440 if (rcu_rbtree_is_nil(rbtree
, x
)) {
444 *zr
= x
= dup_decay_node(rbtree
, x
);
446 while (!rcu_rbtree_is_nil(rbtree
, xl
= rcu_dereference(x
->_left
))) {
447 x
= dup_decay_node(rbtree
, xl
);
448 set_parent(x
, get_decay(get_parent(x
)), get_pos(x
));
449 get_parent(x
)->_left
= get_decay(get_parent(x
)->_left
);
455 struct rcu_rbtree_node
*rcu_rbtree_min_update_decay(struct rcu_rbtree
*rbtree
,
456 struct rcu_rbtree_node
*x
)
458 struct rcu_rbtree_node
*xl
;
460 x
= rcu_dereference(x
);
462 if (rcu_rbtree_is_nil(rbtree
, x
))
465 set_parent(x
->_right
, get_decay(get_parent(x
->_right
)),
467 set_parent(x
->_left
, get_decay(get_parent(x
->_left
)),
471 while (!rcu_rbtree_is_nil(rbtree
, xl
= rcu_dereference(x
->_left
))) {
473 set_parent(x
->_right
, get_decay(get_parent(x
->_right
)),
475 set_parent(x
->_left
, get_decay(get_parent(x
->_left
)),
481 struct rcu_rbtree_node
*rcu_rbtree_min(struct rcu_rbtree
*rbtree
,
482 struct rcu_rbtree_node
*x
)
484 struct rcu_rbtree_node
*xl
;
486 x
= rcu_dereference(x
);
488 if (rcu_rbtree_is_nil(rbtree
, x
))
491 while (!rcu_rbtree_is_nil(rbtree
, xl
= rcu_dereference(x
->_left
)))
496 struct rcu_rbtree_node
*rcu_rbtree_max(struct rcu_rbtree
*rbtree
,
497 struct rcu_rbtree_node
*x
)
499 struct rcu_rbtree_node
*xr
;
501 x
= rcu_dereference(x
);
503 if (rcu_rbtree_is_nil(rbtree
, x
))
506 while (!rcu_rbtree_is_nil(rbtree
, xr
= rcu_dereference(x
->_right
)))
512 * RCU read lock must be held across the next/prev calls to ensure validity of
515 struct rcu_rbtree_node
*rcu_rbtree_next(struct rcu_rbtree
*rbtree
,
516 struct rcu_rbtree_node
*x
)
518 struct rcu_rbtree_node
*xr
, *y
;
521 x
= rcu_dereference(x
);
523 if (!rcu_rbtree_is_nil(rbtree
, xr
= rcu_dereference(x
->_right
)))
524 return rcu_rbtree_min(rbtree
, xr
);
525 y
= get_parent_and_pos(x
, &x_pos
);
526 while (!rcu_rbtree_is_nil(rbtree
, y
) && x_pos
== IS_RIGHT
) {
528 y
= get_parent_and_pos(y
, &x_pos
);
533 struct rcu_rbtree_node
*rcu_rbtree_prev(struct rcu_rbtree
*rbtree
,
534 struct rcu_rbtree_node
*x
)
536 struct rcu_rbtree_node
*xl
, *y
;
539 x
= rcu_dereference(x
);
541 if (!rcu_rbtree_is_nil(rbtree
, xl
= rcu_dereference(x
->_left
)))
542 return rcu_rbtree_max(rbtree
, xl
);
543 y
= get_parent_and_pos(x
, &x_pos
);
544 while (!rcu_rbtree_is_nil(rbtree
, y
) && x_pos
== IS_LEFT
) {
546 y
= get_parent_and_pos(y
, &x_pos
);
552 * "node" needs to be non-visible by readers.
555 void populate_node_end(struct rcu_rbtree
*rbtree
, struct rcu_rbtree_node
*node
,
556 unsigned int copy_parents
, struct rcu_rbtree_node
*stop
)
558 struct rcu_rbtree_node
*prev
= NULL
, *orig_node
= node
, *top
;
564 assert(!rcu_rbtree_is_nil(rbtree
, node
));
566 if (prev
&& copy_parents
) {
567 node
= dup_decay_node(rbtree
, node
);
568 if (get_pos(prev
) == IS_RIGHT
)
572 set_parent(prev
, node
, get_pos(prev
));
575 max_end
= calculate_node_max_end(rbtree
, node
);
577 * Compare the node max_end keys to make sure we replace
578 * references to a key belonging to a node we remove
579 * from the tree. Otherwise we would still be using this
580 * pointer as an invalid reference after garbage
581 * collection of the node and of its associated
582 * begin/end pointers.
584 if (max_end
!= node
->max_end
) {
585 node
->max_end
= max_end
;
587 top
= get_parent(node
);
588 cmm_smp_wmb(); /* write into node before publish */
589 /* make new branch visible to readers */
590 if (rcu_rbtree_is_nil(rbtree
, top
))
591 _CMM_STORE_SHARED(rbtree
->root
, node
);
592 if (get_pos(node
) == IS_LEFT
)
593 _CMM_STORE_SHARED(top
->_left
, node
);
595 _CMM_STORE_SHARED(top
->_right
, node
);
599 /* Check for propagation stop */
604 node
= get_parent(node
);
605 } while (!rcu_rbtree_is_nil(rbtree
, node
));
607 top
= node
; /* nil */
608 cmm_smp_wmb(); /* write into node before publish */
609 /* make new branch visible to readers */
610 _CMM_STORE_SHARED(rbtree
->root
, prev
);
615 /* update children */
618 assert(!rcu_rbtree_is_nil(rbtree
, node
));
619 set_parent(node
->_left
, get_decay(get_parent(node
->_left
)), IS_LEFT
);
620 set_parent(node
->_right
, get_decay(get_parent(node
->_right
)), IS_RIGHT
);
621 } while ((node
= get_parent(node
)) != top
);
625 * We have to ensure these assumptions are correct for prev/next
628 * with x being a right child, the assumption that:
629 * get_parent(x)->_right == x
630 * or if x is a left child, the assumption that:
631 * get_parent(x)->_left == x
633 * This explains why we have to allocate a vc copy of the node for left_rotate,
634 * right_rotate and transplant operations.
636 * We always ensure that the right/left child and correct parent is set in the
637 * node copies *before* we reparent the children and make the upper-level point
641 /* RCU: copy x and y, atomically point to new versions. GC old. */
642 /* Should be eventually followed by a cmm_smp_wmc() */
644 #ifdef RBTREE_RCU_SUPPORT_ROTATE_LEFT
647 void left_rotate(struct rcu_rbtree
*rbtree
,
648 struct rcu_rbtree_node
*x
)
650 struct rcu_rbtree_node
*y
, *y_left
;
652 dbg_printf("left rotate %lx\n", (unsigned long) x
->begin
);
657 /* Now operate on new copy, decay old versions */
658 x
= dup_decay_node(rbtree
, x
);
659 y
= dup_decay_node(rbtree
, y
);
660 y_left
= dup_decay_node(rbtree
, y_left
);
662 check_max_end(rbtree
, get_parent(x
));
663 check_max_end(rbtree
, x
);
664 check_max_end(rbtree
, y
);
666 /* Internal node modifications */
667 set_parent(y
, get_parent(x
), get_pos(x
));
668 set_parent(x
, y
, IS_LEFT
);
669 set_left(rbtree
, y
, x
);
670 set_right(rbtree
, x
, y_left
);
672 if (!rcu_rbtree_is_nil(rbtree
, y_left
))
673 set_parent(y_left
, x
, IS_RIGHT
);
676 * We only changed the relative position of x and y wrt their
677 * children, and reparented y (but are keeping the same nodes in
678 * place, so its parent does not need to have end value
681 x
->max_end
= calculate_node_max_end(rbtree
, x
);
682 y
->max_end
= calculate_node_max_end(rbtree
, y
);
684 cmm_smp_wmb(); /* write into node before publish */
686 /* External references update (visible by readers) */
687 if (rcu_rbtree_is_nil(rbtree
, get_parent(y
)))
688 _CMM_STORE_SHARED(rbtree
->root
, y
);
689 else if (get_pos(y
) == IS_LEFT
)
690 _CMM_STORE_SHARED(get_parent(y
)->_left
, y
);
692 _CMM_STORE_SHARED(get_parent(y
)->_right
, y
);
694 /* Point children to new copy (parent only used by updates/next/prev) */
695 set_parent(x
->_left
, get_decay(get_parent(x
->_left
)),
697 set_parent(y
->_right
, get_decay(get_parent(y
->_right
)),
699 if (!rcu_rbtree_is_nil(rbtree
, y_left
)) {
700 set_parent(y_left
->_right
,
701 get_decay(get_parent(y_left
->_right
)),
702 get_pos(y_left
->_right
));
703 set_parent(y_left
->_left
,
704 get_decay(get_parent(y_left
->_left
)),
705 get_pos(y_left
->_left
));
709 assert(y
== rbtree
->root
|| get_parent(y
)->_left
== y
710 || get_parent(y
)->_right
== y
);
711 assert(x
== rbtree
->root
|| get_parent(x
)->_left
== x
712 || get_parent(x
)->_right
== x
);
713 assert(rcu_rbtree_is_nil(rbtree
, x
->_right
) || get_parent(x
->_right
) == x
);
714 assert(rcu_rbtree_is_nil(rbtree
, x
->_left
) || get_parent(x
->_left
) == x
);
715 assert(rcu_rbtree_is_nil(rbtree
, y
->_right
) || get_parent(y
->_right
) == y
);
716 assert(rcu_rbtree_is_nil(rbtree
, y
->_left
) || get_parent(y
->_left
) == y
);
717 assert(!is_decay(rbtree
->root
));
718 assert(!is_decay(x
));
719 assert(!is_decay(y
));
720 assert(!is_decay(x
->_right
));
721 assert(!is_decay(x
->_left
));
722 assert(!is_decay(y
->_right
));
723 assert(!is_decay(y
->_left
));
724 check_max_end(rbtree
, get_parent(y
));
725 check_max_end(rbtree
, x
);
726 check_max_end(rbtree
, y
);
731 /* non-rcu version */
733 void left_rotate(struct rcu_rbtree
*rbtree
,
734 struct rcu_rbtree_node
*x
)
736 struct rcu_rbtree_node
*y
;
740 x
->_right
= y
->_left
;
741 if (!rcu_rbtree_is_nil(rbtree
, y
->_left
))
742 set_parent(y
->_left
, x
, IS_RIGHT
);
743 set_parent(y
, get_parent(x
), get_pos(x
));
744 if (rcu_rbtree_is_nil(rbtree
, get_parent(x
)))
746 else if (x
== get_parent(x
)->_left
) {
747 get_parent(x
)->_left
= y
;
749 get_parent(x
)->_right
= y
;
752 set_parent(x
, y
, IS_LEFT
);
758 #ifdef RBTREE_RCU_SUPPORT_ROTATE_RIGHT
760 void right_rotate(struct rcu_rbtree
*rbtree
,
761 struct rcu_rbtree_node
*x
)
763 struct rcu_rbtree_node
*y
, *y_right
;
765 dbg_printf("right rotate %lx\n", (unsigned long) x
->begin
);
770 /* Now operate on new copy, decay old versions */
771 x
= dup_decay_node(rbtree
, x
);
772 y
= dup_decay_node(rbtree
, y
);
773 y_right
= dup_decay_node(rbtree
, y_right
);
775 check_max_end(rbtree
, get_parent(x
));
776 check_max_end(rbtree
, x
);
777 check_max_end(rbtree
, y
);
779 /* Internal node modifications */
780 set_parent(y
, get_parent(x
), get_pos(x
));
781 set_parent(x
, y
, IS_RIGHT
);
782 set_right(rbtree
, y
, x
);
783 set_left(rbtree
, x
, y_right
);
785 if (!rcu_rbtree_is_nil(rbtree
, y_right
))
786 set_parent(y_right
, x
, IS_LEFT
);
789 * We only changed the relative position of x and y wrt their
790 * children, and reparented y (but are keeping the same nodes in
791 * place, so its parent does not need to have end value
794 x
->max_end
= calculate_node_max_end(rbtree
, x
);
795 y
->max_end
= calculate_node_max_end(rbtree
, y
);
797 cmm_smp_wmb(); /* write into node before publish */
799 /* External references update (visible by readers) */
800 if (rcu_rbtree_is_nil(rbtree
, get_parent(y
)))
801 _CMM_STORE_SHARED(rbtree
->root
, y
);
802 else if (get_pos(y
) == IS_RIGHT
)
803 _CMM_STORE_SHARED(get_parent(y
)->_right
, y
);
805 _CMM_STORE_SHARED(get_parent(y
)->_left
, y
);
807 /* Point children to new copy (parent only used by updates/next/prev) */
808 set_parent(x
->_right
, get_decay(get_parent(x
->_right
)),
810 set_parent(y
->_left
, get_decay(get_parent(y
->_left
)),
812 if (!rcu_rbtree_is_nil(rbtree
, y_right
)) {
813 set_parent(y_right
->_left
,
814 get_decay(get_parent(y_right
->_left
)),
815 get_pos(y_right
->_left
));
816 set_parent(y_right
->_right
,
817 get_decay(get_parent(y_right
->_right
)),
818 get_pos(y_right
->_right
));
822 assert(y
== rbtree
->root
|| get_parent(y
)->_right
== y
823 || get_parent(y
)->_left
== y
);
824 assert(x
== rbtree
->root
|| get_parent(x
)->_right
== x
825 || get_parent(x
)->_left
== x
);
826 assert(rcu_rbtree_is_nil(rbtree
, x
->_left
) || get_parent(x
->_left
) == x
);
827 assert(rcu_rbtree_is_nil(rbtree
, x
->_right
) || get_parent(x
->_right
) == x
);
828 assert(rcu_rbtree_is_nil(rbtree
, y
->_left
) || get_parent(y
->_left
) == y
);
829 assert(rcu_rbtree_is_nil(rbtree
, y
->_right
) || get_parent(y
->_right
) == y
);
830 assert(!is_decay(rbtree
->root
));
831 assert(!is_decay(x
));
832 assert(!is_decay(y
));
833 assert(!is_decay(x
->_left
));
834 assert(!is_decay(x
->_right
));
835 assert(!is_decay(y
->_left
));
836 assert(!is_decay(y
->_right
));
837 check_max_end(rbtree
, x
);
838 check_max_end(rbtree
, y
);
839 check_max_end(rbtree
, get_parent(y
));
844 /* non-rcu version */
846 void right_rotate(struct rcu_rbtree
*rbtree
,
847 struct rcu_rbtree_node
*x
)
849 struct rcu_rbtree_node
*y
;
853 x
->_left
= y
->_right
;
854 if (!rcu_rbtree_is_nil(rbtree
, y
->_right
))
855 set_parent(y
->_right
, x
, IS_LEFT
);
856 set_parent(y
, get_parent(x
), get_pos(x
));
857 if (rcu_rbtree_is_nil(rbtree
, get_parent(x
)))
859 else if (x
== get_parent(x
)->_right
) {
860 get_parent(x
)->_right
= y
;
862 get_parent(x
)->_left
= y
;
865 set_parent(x
, y
, IS_RIGHT
);
871 static void rcu_rbtree_insert_fixup(struct rcu_rbtree
*rbtree
,
872 struct rcu_rbtree_node
*z
)
874 struct rcu_rbtree_node
*y
;
876 dbg_printf("insert fixup %p\n", z
->begin
);
877 assert(!is_decay(rbtree
->root
));
879 while (get_parent(z
)->color
== COLOR_RED
) {
880 if (get_parent(z
) == get_parent(get_parent(z
))->_left
) {
881 y
= get_parent(get_parent(z
))->_right
;
882 if (y
->color
== COLOR_RED
) {
883 get_parent(z
)->color
= COLOR_BLACK
;
884 y
->color
= COLOR_BLACK
;
885 get_parent(get_parent(z
))->color
= COLOR_RED
;
886 z
= get_parent(get_parent(z
));
888 if (z
== get_parent(z
)->_right
) {
890 left_rotate(rbtree
, z
);
892 assert(!is_decay(rbtree
->root
));
894 get_parent(z
)->color
= COLOR_BLACK
;
895 get_parent(get_parent(z
))->color
= COLOR_RED
;
896 assert(!is_decay(z
));
897 assert(!is_decay(get_parent(z
)));
898 assert(!is_decay(get_parent(get_parent(z
))));
899 right_rotate(rbtree
, get_parent(get_parent(z
)));
900 assert(!is_decay(z
));
901 assert(!is_decay(rbtree
->root
));
904 y
= get_parent(get_parent(z
))->_left
;
905 if (y
->color
== COLOR_RED
) {
906 get_parent(z
)->color
= COLOR_BLACK
;
907 y
->color
= COLOR_BLACK
;
908 get_parent(get_parent(z
))->color
= COLOR_RED
;
909 z
= get_parent(get_parent(z
));
911 if (z
== get_parent(z
)->_left
) {
913 right_rotate(rbtree
, z
);
915 assert(!is_decay(rbtree
->root
));
917 get_parent(z
)->color
= COLOR_BLACK
;
918 get_parent(get_parent(z
))->color
= COLOR_RED
;
919 left_rotate(rbtree
, get_parent(get_parent(z
)));
920 assert(!is_decay(z
));
921 assert(!is_decay(rbtree
->root
));
925 rbtree
->root
->color
= COLOR_BLACK
;
929 * rcu_rbtree_insert - Insert a node in the RCU rbtree
931 * Returns 0 on success, or < 0 on error.
933 int rcu_rbtree_insert(struct rcu_rbtree
*rbtree
,
934 struct rcu_rbtree_node
*z
)
936 struct rcu_rbtree_node
*x
, *y
;
938 dbg_printf("insert %p\n", z
->begin
);
939 assert(!is_decay(rbtree
->root
));
941 y
= make_nil(rbtree
);
943 while (!rcu_rbtree_is_nil(rbtree
, x
)) {
945 if (rbtree
->comp(z
->begin
, x
->begin
) < 0)
951 z
->_left
= make_nil(rbtree
);
952 z
->_right
= make_nil(rbtree
);
953 z
->color
= COLOR_RED
;
954 z
->decay_next
= NULL
;
957 if (rcu_rbtree_is_nil(rbtree
, y
)) {
958 set_parent(z
, y
, IS_RIGHT
); /* pos arbitrary for root node */
960 * Order stores to z (children/parents) before stores
961 * that will make it visible to the rest of the tree.
964 _CMM_STORE_SHARED(rbtree
->root
, z
);
965 } else if (rbtree
->comp(z
->begin
, y
->begin
) < 0) {
966 y
= dup_decay_node(rbtree
, y
);
967 set_parent(z
, y
, IS_LEFT
);
968 if (get_pos(z
) == IS_LEFT
)
969 _CMM_STORE_SHARED(y
->_left
, z
);
971 _CMM_STORE_SHARED(y
->_right
, z
);
972 populate_node_end(rbtree
, y
, 1, NULL
);
974 y
= dup_decay_node(rbtree
, y
);
975 set_parent(z
, y
, IS_RIGHT
);
976 if (get_pos(z
) == IS_LEFT
)
977 _CMM_STORE_SHARED(y
->_left
, z
);
979 _CMM_STORE_SHARED(y
->_right
, z
);
980 populate_node_end(rbtree
, y
, 1, NULL
);
982 rcu_rbtree_insert_fixup(rbtree
, z
);
984 * Make sure to commit all _CMM_STORE_SHARED() for non-coherent caches.
988 check_max_end(rbtree
, z
);
989 check_max_end(rbtree
, y
);
995 * Transplant v into u position.
998 #ifdef RBTREE_RCU_SUPPORT_TRANSPLANT
1001 void rcu_rbtree_transplant(struct rcu_rbtree
*rbtree
,
1002 struct rcu_rbtree_node
*u
,
1003 struct rcu_rbtree_node
*v
,
1004 unsigned int copy_parents
,
1005 struct rcu_rbtree_node
*stop
)
1007 dbg_printf("transplant %p\n", v
->begin
);
1009 if (!rcu_rbtree_is_nil(rbtree
, v
))
1010 v
= dup_decay_node(rbtree
, v
);
1012 if (rcu_rbtree_is_nil(rbtree
, get_parent(u
))) {
1013 /* pos is arbitrary for root node */
1014 set_parent(v
, get_parent(u
), IS_RIGHT
);
1015 cmm_smp_wmb(); /* write into node before publish */
1016 _CMM_STORE_SHARED(rbtree
->root
, v
);
1018 struct rcu_rbtree_node
*vp
;
1022 vp
= dup_decay_node(rbtree
, vp
);
1023 set_parent(v
, vp
, get_pos(u
));
1024 if (get_pos(v
) == IS_LEFT
)
1025 _CMM_STORE_SHARED(vp
->_left
, v
);
1027 _CMM_STORE_SHARED(vp
->_right
, v
);
1028 populate_node_end(rbtree
, vp
, copy_parents
, stop
);
1029 check_max_end(rbtree
, vp
);
1032 /* Point children to new copy (parent only used by updates/next/prev) */
1033 if (!rcu_rbtree_is_nil(rbtree
, v
)) {
1034 set_parent(v
->_right
, get_decay(get_parent(v
->_right
)),
1035 get_pos(v
->_right
));
1036 set_parent(v
->_left
, get_decay(get_parent(v
->_left
)),
1039 assert(!is_decay(rbtree
->root
));
1040 check_max_end(rbtree
, v
);
1045 /* Non-RCU version */
1047 void rcu_rbtree_transplant(struct rcu_rbtree
*rbtree
,
1048 struct rcu_rbtree_node
*u
,
1049 struct rcu_rbtree_node
*v
,
1050 unsigned int copy_parents
,
1051 struct rcu_rbtree_node
*stop
)
1053 dbg_printf("transplant %p\n", v
->begin
);
1056 if (rcu_rbtree_is_nil(rbtree
, get_parent(u
)))
1058 else if (u
== get_parent(u
)->_left
)
1059 get_parent(u
)->_left
= v
;
1061 get_parent(u
)->_right
= v
;
1062 set_parent(v
, get_parent(u
), get_pos(u
));
1063 unlock_test_mutex();
1068 static void rcu_rbtree_remove_fixup(struct rcu_rbtree
*rbtree
,
1069 struct rcu_rbtree_node
*x
)
1071 dbg_printf("remove fixup %p\n", x
->begin
);
1073 while (x
!= rbtree
->root
&& x
->color
== COLOR_BLACK
) {
1074 assert(!is_decay(get_parent(x
)));
1075 assert(!is_decay(get_parent(x
)->_left
));
1076 if (x
== get_parent(x
)->_left
) {
1077 struct rcu_rbtree_node
*w
;
1079 w
= get_parent(x
)->_right
;
1081 if (w
->color
== COLOR_RED
) {
1082 w
->color
= COLOR_BLACK
;
1083 get_parent(x
)->color
= COLOR_RED
;
1084 left_rotate(rbtree
, get_parent(x
));
1086 assert(!is_decay(rbtree
->root
));
1087 w
= get_parent(x
)->_right
;
1089 if (w
->_left
->color
== COLOR_BLACK
1090 && w
->_right
->color
== COLOR_BLACK
) {
1091 w
->color
= COLOR_RED
;
1093 assert(!is_decay(rbtree
->root
));
1094 assert(!is_decay(x
));
1096 if (w
->_right
->color
== COLOR_BLACK
) {
1097 w
->_left
->color
= COLOR_BLACK
;
1098 w
->color
= COLOR_RED
;
1099 right_rotate(rbtree
, w
);
1100 assert(!is_decay(rbtree
->root
));
1102 w
= get_parent(x
)->_right
;
1104 w
->color
= get_parent(x
)->color
;
1105 get_parent(x
)->color
= COLOR_BLACK
;
1106 w
->_right
->color
= COLOR_BLACK
;
1107 left_rotate(rbtree
, get_parent(x
));
1108 assert(!is_decay(rbtree
->root
));
1112 struct rcu_rbtree_node
*w
;
1114 w
= get_parent(x
)->_left
;
1116 if (w
->color
== COLOR_RED
) {
1117 w
->color
= COLOR_BLACK
;
1118 get_parent(x
)->color
= COLOR_RED
;
1119 right_rotate(rbtree
, get_parent(x
));
1120 assert(!is_decay(rbtree
->root
));
1122 w
= get_parent(x
)->_left
;
1124 if (w
->_right
->color
== COLOR_BLACK
1125 && w
->_left
->color
== COLOR_BLACK
) {
1126 w
->color
= COLOR_RED
;
1128 assert(!is_decay(rbtree
->root
));
1129 assert(!is_decay(x
));
1131 if (w
->_left
->color
== COLOR_BLACK
) {
1132 w
->_right
->color
= COLOR_BLACK
;
1133 w
->color
= COLOR_RED
;
1134 left_rotate(rbtree
, w
);
1135 assert(!is_decay(rbtree
->root
));
1137 w
= get_parent(x
)->_left
;
1139 w
->color
= get_parent(x
)->color
;
1140 get_parent(x
)->color
= COLOR_BLACK
;
1141 w
->_left
->color
= COLOR_BLACK
;
1142 right_rotate(rbtree
, get_parent(x
));
1143 assert(!is_decay(rbtree
->root
));
1148 x
->color
= COLOR_BLACK
;
1152 * Delete z. All non-copied children left/right positions are unchanged.
1155 void rcu_rbtree_remove_nonil(struct rcu_rbtree
*rbtree
,
1156 struct rcu_rbtree_node
*z
,
1157 struct rcu_rbtree_node
*y
)
1159 struct rcu_rbtree_node
*x
;
1161 dbg_printf("remove nonil %p\n", z
->begin
);
1164 assert(!is_decay(z
));
1165 assert(!is_decay(y
));
1166 assert(!is_decay(y
->_right
));
1167 assert(!is_decay(get_parent(y
)));
1169 assert(!is_decay(x
));
1170 if (get_parent(y
) == z
) {
1171 y
= dup_decay_node(rbtree
, y
);
1172 set_parent(x
, y
, get_pos(x
)); /* parent for nil */
1173 /* y is z's right node */
1174 set_left(rbtree
, y
, z
->_left
);
1175 y
->max_end
= calculate_node_max_end(rbtree
, y
);
1176 rcu_rbtree_transplant(rbtree
, z
, y
, 1, NULL
);
1178 struct rcu_rbtree_node
*oy_right
, *z_right
;
1181 * Need to make sure y is always visible by readers.
1183 y
= rcu_rbtree_min_dup_decay(rbtree
, z
->_right
, &z_right
);
1184 assert(!is_decay(y
));
1185 assert(!is_decay(z
));
1186 oy_right
= y
->_right
;
1189 * The max child begin of z_right does not change, because
1190 * we're only changing its left children.
1192 y
->_right
= z_right
;
1193 set_parent(y
->_right
, y
, IS_RIGHT
);
1194 assert(!is_decay(z
->_left
));
1195 y
->_left
= z
->_left
;
1196 assert(!is_decay(oy_right
));
1198 * Transplant of oy_right to old y's location will only
1199 * trigger a "end" value update of the already copied branch
1200 * (which is not visible yet). We are transplanting
1201 * oy_right as a left child of old y's parent, so the
1202 * min values update propagated upward necessarily stops
1205 rcu_rbtree_transplant(rbtree
, y
, oy_right
, 0, y
);
1206 y
->max_end
= calculate_node_max_end(rbtree
, y
);
1207 rcu_rbtree_transplant(rbtree
, z
, y
, 1, NULL
);
1208 /* Update children */
1209 (void) rcu_rbtree_min_update_decay(rbtree
, y
->_right
);
1212 assert(!is_decay(z
));
1213 assert(!is_decay(z
->_left
));
1214 y
->color
= z
->color
;
1215 set_parent(y
->_left
, y
, IS_LEFT
);
1216 set_parent(y
->_right
, get_decay(get_parent(y
->_right
)), IS_RIGHT
);
1217 assert(!is_decay(y
->_left
));
1218 assert(!is_decay(y
->_right
));
1221 int rcu_rbtree_remove(struct rcu_rbtree
*rbtree
,
1222 struct rcu_rbtree_node
*z
)
1224 struct rcu_rbtree_node
*x
, *y
;
1225 unsigned int y_original_color
;
1227 assert(!is_decay(rbtree
->root
));
1228 dbg_printf("remove %p\n", z
->begin
);
1231 assert(!is_decay(z
));
1233 y_original_color
= y
->color
;
1235 if (rcu_rbtree_is_nil(rbtree
, z
->_left
)) {
1236 rcu_rbtree_transplant(rbtree
, z
, z
->_right
, 1, NULL
);
1237 assert(!is_decay(z
));
1238 x
= get_decay(z
->_right
);
1240 } else if (rcu_rbtree_is_nil(rbtree
, z
->_right
)) {
1241 rcu_rbtree_transplant(rbtree
, z
, z
->_left
, 1, NULL
);
1242 assert(!is_decay(z
));
1243 x
= get_decay(z
->_left
);
1246 y
= rcu_rbtree_min(rbtree
, z
->_right
);
1247 assert(!is_decay(y
));
1248 y_original_color
= y
->color
;
1250 rcu_rbtree_remove_nonil(rbtree
, z
, y
);
1254 if (y_original_color
== COLOR_BLACK
)
1255 rcu_rbtree_remove_fixup(rbtree
, x
);
1257 check_max_end(rbtree
, x
);
1258 check_max_end(rbtree
, get_decay(y
));
1260 * Commit all _CMM_STORE_SHARED().