+ CDS_LIST_HEAD(cur_snap_readers);
+ CDS_LIST_HEAD(qsreaders);
+ DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
+ struct urcu_waiters waiters;
+
+ /*
+ * Add ourself to gp_waiters queue of threads awaiting to wait
+ * for a grace period. Proceed to perform the grace period only
+ * if we are the first thread added into the queue.
+ * The implicit memory barrier before urcu_wait_add()
+ * orders prior memory accesses of threads put into the wait
+ * queue before their insertion into the wait queue.
+ */
+ if (urcu_wait_add(&gp_waiters, &wait) != 0) {
+ /* Not first in queue: will be awakened by another thread. */
+ urcu_adaptative_busy_wait(&wait);
+ /* Order following memory accesses after grace period. */
+ cmm_smp_mb();
+ return;
+ }
+ /* We won't need to wake ourself up */
+ urcu_wait_set_state(&wait, URCU_WAIT_RUNNING);
+
+ mutex_lock(&rcu_gp_lock);
+
+ /*
+ * Move all waiters into our local queue.
+ */
+ urcu_move_waiters(&waiters, &gp_waiters);
+
+ if (cds_list_empty(®istry))
+ goto out;
+
+ /* All threads should read qparity before accessing data structure
+ * where new ptr points to. Must be done within rcu_gp_lock because it
+ * iterates on reader threads.*/
+ /* Write new ptr before changing the qparity */
+ smp_mb_master(RCU_MB_GROUP);
+
+ /*
+ * Wait for readers to observe original parity or be quiescent.
+ */
+ wait_for_readers(®istry, &cur_snap_readers, &qsreaders);
+
+ /*
+ * Must finish waiting for quiescent state for original parity before
+ * committing next rcu_gp.ctr update to memory. Failure to do so could
+ * result in the writer waiting forever while new readers are always
+ * accessing data (no progress). Enforce compiler-order of load
+ * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
+ */
+ cmm_barrier();
+
+ /*
+ * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
+ * model easier to understand. It does not have a big performance impact
+ * anyway, given this is the write-side.
+ */
+ cmm_smp_mb();
+
+ /* Switch parity: 0 -> 1, 1 -> 0 */
+ CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
+
+ /*
+ * Must commit rcu_gp.ctr update to memory before waiting for quiescent
+ * state. Failure to do so could result in the writer waiting forever
+ * while new readers are always accessing data (no progress). Enforce
+ * compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
+ */
+ cmm_barrier();
+
+ /*
+ *
+ * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
+ * model easier to understand. It does not have a big performance impact
+ * anyway, given this is the write-side.
+ */
+ cmm_smp_mb();
+
+ /*
+ * Wait for readers to observe new parity or be quiescent.
+ */
+ wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
+
+ /*
+ * Put quiescent reader list back into registry.
+ */
+ cds_list_splice(&qsreaders, ®istry);
+
+ /* Finish waiting for reader threads before letting the old ptr being
+ * freed. Must be done within rcu_gp_lock because it iterates on reader
+ * threads. */
+ smp_mb_master(RCU_MB_GROUP);
+out:
+ mutex_unlock(&rcu_gp_lock);