- /* All threads should read qparity before accessing data structure. */
- /* Write ptr before changing the qparity */
- force_mb_all_threads();
- debug_yield_write();
- switch_next_urcu_qparity();
- debug_yield_write();
+ mutex_lock(&rcu_gp_lock);
+ mutex_lock(&rcu_registry_lock);
+
+ if (cds_list_empty(®istry))
+ goto out;
+
+ /*
+ * All threads should read qparity before accessing data structure
+ * where new ptr points to. Must be done within rcu_registry_lock
+ * because it iterates on reader threads.
+ */
+ /* Write new ptr before changing the qparity */
+ smp_mb_master(RCU_MB_GROUP);
+
+ /*
+ * Wait for previous parity to be empty of readers.
+ * update_counter_and_wait() can release and grab again
+ * rcu_registry_lock interally.
+ */
+ update_counter_and_wait(); /* 0 -> 1, wait readers in parity 0 */
+
+ /*
+ * Must finish waiting for quiescent state for parity 0 before
+ * committing next rcu_gp_ctr update to memory. Failure to do so could
+ * result in the writer waiting forever while new readers are always
+ * accessing data (no progress). Enforce compiler-order of load
+ * URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr.
+ */
+ cmm_barrier();
+
+ /*
+ * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
+ * model easier to understand. It does not have a big performance impact
+ * anyway, given this is the write-side.
+ */
+ cmm_smp_mb();