Add missing rcu_cmpxchg_pointer define

[urcu.git] / urcu-qsbr.c

diff --git a/urcu-qsbr.c b/urcu-qsbr.c
index a86f6e94d5ae13eb24e5d34610a3d9c5954373b4..dac664935d7b898bd3dcce494845c7471c09749d 100644 (file)
--- a/urcu-qsbr.c
+++ b/urcu-qsbr.c
@@ -37,7 +37,9 @@
 /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
 #include "urcu-qsbr.h"
 
-pthread_mutex_t urcu_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t urcu_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+int gp_futex;
 
 /*
  * Global grace period counter.
@@ -99,6 +101,27 @@ static void internal_urcu_unlock(void)
        }
 }
 
+/*
+ * synchronize_rcu() waiting. Single thread.
+ */
+static void wait_gp(struct reader_registry *index)
+{
+       atomic_dec(&gp_futex);
+       smp_mb(); /* Write futex before read reader_gp */
+       if (!rcu_gp_ongoing(index->rcu_reader_qs_gp)) {
+               /* Read reader_gp before write futex */
+               smp_mb();
+               /* Callbacks are queued, don't wait. */
+               atomic_set(&gp_futex, 0);
+       } else {
+               /* Read reader_gp before read futex */
+               smp_rmb();
+               if (atomic_read(&gp_futex) == -1)
+                       futex(&gp_futex, FUTEX_WAIT, -1,
+                             NULL, NULL, 0);
+       }
+}
+
 static void wait_for_quiescent_state(void)
 {
        struct reader_registry *index;
@@ -109,13 +132,19 @@ static void wait_for_quiescent_state(void)
         * Wait for each thread rcu_reader_qs_gp count to become 0.
         */
        for (index = registry; index < registry + num_readers; index++) {
+               int wait_loops = 0;
+
+               while (rcu_gp_ongoing(index->rcu_reader_qs_gp)) {
+                       if (wait_loops++ == RCU_QS_ACTIVE_ATTEMPTS) {
+                               wait_gp(index);
+                       } else {
 #ifndef HAS_INCOHERENT_CACHES
-               while (rcu_gp_ongoing(index->rcu_reader_qs_gp))
-                       cpu_relax();
+                               cpu_relax();
 #else /* #ifndef HAS_INCOHERENT_CACHES */
-               while (rcu_gp_ongoing(index->rcu_reader_qs_gp))
-                       smp_mb();
+                               smp_mb();
 #endif /* #else #ifndef HAS_INCOHERENT_CACHES */
+                       }
+               }
        }
 }
 
@@ -135,12 +164,24 @@ static void switch_next_urcu_qparity(void)
 
 void synchronize_rcu(void)
 {
+       unsigned long was_online;
+
+       was_online = rcu_reader_qs_gp;
+
        /* All threads should read qparity before accessing data structure
         * where new ptr points to.
         */
        /* Write new ptr before changing the qparity */
        smp_mb();
 
+       /*
+        * Mark the writer thread offline to make sure we don't wait for
+        * our own quiescent state. This allows using synchronize_rcu() in
+        * threads registered as readers.
+        */
+       if (was_online)
+               STORE_SHARED(rcu_reader_qs_gp, 0);
+
        internal_urcu_lock();
 
        switch_next_urcu_qparity();     /* 0 -> 1 */
@@ -183,9 +224,12 @@ void synchronize_rcu(void)
 
        internal_urcu_unlock();
 
-       /* Finish waiting for reader threads before letting the old ptr being
+       /*
+        * Finish waiting for reader threads before letting the old ptr being
         * freed.
         */
+       if (was_online)
+               _STORE_SHARED(rcu_reader_qs_gp, LOAD_SHARED(urcu_gp_ctr));
        smp_mb();
 }
 #else /* !(BITS_PER_LONG < 64) */
@@ -240,6 +284,12 @@ void *rcu_assign_pointer_sym(void **p, void *v)
        return STORE_SHARED(p, v);
 }
 
+void *rcu_cmpxchg_pointer_sym(void **p, void *old, void *_new)
+{
+       wmb();
+       return cmpxchg(p, old, _new);
+}
+
 void *rcu_xchg_pointer_sym(void **p, void *v)
 {
        wmb();