- if (next) {
- if (uatomic_cmpxchg(&q->head, head, next) == head) {
- urcu_ref_put(&head->ref, q->release);
- return next;
- } else {
- /* Concurrently pushed, retry */
- continue;
- }
- } else {
- /* Empty */
- return NULL;
+ dummy = rcu_dereference(q->dummy);
+ if (head == dummy && next == NULL)
+ return NULL; /* empty */
+ /*
+ * We never, ever allow dequeue to get to a state where
+ * the queue is empty (we need at least one node in the
+ * queue). This is ensured by checking if the head next
+ * is NULL. This means a concurrent dummy node
+ * re-enqueue is in progress. We help it reach
+ * completion, and retry.
+ */
+ if (!next) {
+ /*
+ * Dummy node re-enqueue is in progress. Try to
+ * help.
+ */
+ reenqueue_dummy(q);
+ continue; /* try again */
+ }
+ if (uatomic_cmpxchg(&q->head, head, next) != head)
+ continue; /* Concurrently pushed. */
+ if (head == dummy) {
+ /* Free old and requeue new dummy. */
+ rcu_set_pointer(&q->dummy, NULL);
+ rcu_free_dummy(dummy);
+ reenqueue_dummy(q);
+ continue; /* try again */