X-Git-Url: http://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu-qsbr.c;h=dac664935d7b898bd3dcce494845c7471c09749d;hp=a86f6e94d5ae13eb24e5d34610a3d9c5954373b4;hb=ba59a0c7b244a0939a2298fc76a9002436ef9674;hpb=47d2f29ead2c92123069ebf7b84c6e48cf91612a

diff --git a/urcu-qsbr.c b/urcu-qsbr.c
index a86f6e9..dac6649 100644
--- a/urcu-qsbr.c
+++ b/urcu-qsbr.c
@@ -37,7 +37,9 @@
 /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
 #include "urcu-qsbr.h"
 
-pthread_mutex_t urcu_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t urcu_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+int gp_futex;
 
 /*
  * Global grace period counter.
@@ -99,6 +101,27 @@ static void internal_urcu_unlock(void)
 	}
 }
 
+/*
+ * synchronize_rcu() waiting. Single thread.
+ */
+static void wait_gp(struct reader_registry *index)
+{
+	atomic_dec(&gp_futex);
+	smp_mb(); /* Write futex before read reader_gp */
+	if (!rcu_gp_ongoing(index->rcu_reader_qs_gp)) {
+		/* Read reader_gp before write futex */
+		smp_mb();
+		/* Callbacks are queued, don't wait. */
+		atomic_set(&gp_futex, 0);
+	} else {
+		/* Read reader_gp before read futex */
+		smp_rmb();
+		if (atomic_read(&gp_futex) == -1)
+			futex(&gp_futex, FUTEX_WAIT, -1,
+			      NULL, NULL, 0);
+	}
+}
+
 static void wait_for_quiescent_state(void)
 {
 	struct reader_registry *index;
@@ -109,13 +132,19 @@ static void wait_for_quiescent_state(void)
 	 * Wait for each thread rcu_reader_qs_gp count to become 0.
 	 */
 	for (index = registry; index < registry + num_readers; index++) {
+		int wait_loops = 0;
+
+		while (rcu_gp_ongoing(index->rcu_reader_qs_gp)) {
+			if (wait_loops++ == RCU_QS_ACTIVE_ATTEMPTS) {
+				wait_gp(index);
+			} else {
 #ifndef HAS_INCOHERENT_CACHES
-		while (rcu_gp_ongoing(index->rcu_reader_qs_gp))
-			cpu_relax();
+				cpu_relax();
 #else /* #ifndef HAS_INCOHERENT_CACHES */
-		while (rcu_gp_ongoing(index->rcu_reader_qs_gp))
-			smp_mb();
+				smp_mb();
 #endif /* #else #ifndef HAS_INCOHERENT_CACHES */
+			}
+		}
 	}
 }
 
@@ -135,12 +164,24 @@ static void switch_next_urcu_qparity(void)
 
 void synchronize_rcu(void)
 {
+	unsigned long was_online;
+
+	was_online = rcu_reader_qs_gp;
+
 	/* All threads should read qparity before accessing data structure
 	 * where new ptr points to.
 	 */
 	/* Write new ptr before changing the qparity */
 	smp_mb();
 
+	/*
+	 * Mark the writer thread offline to make sure we don't wait for
+	 * our own quiescent state. This allows using synchronize_rcu() in
+	 * threads registered as readers.
+	 */
+	if (was_online)
+		STORE_SHARED(rcu_reader_qs_gp, 0);
+
 	internal_urcu_lock();
 
 	switch_next_urcu_qparity();	/* 0 -> 1 */
@@ -183,9 +224,12 @@ void synchronize_rcu(void)
 
 	internal_urcu_unlock();
 
-	/* Finish waiting for reader threads before letting the old ptr being
+	/*
+	 * Finish waiting for reader threads before letting the old ptr being
 	 * freed.
 	 */
+	if (was_online)
+		_STORE_SHARED(rcu_reader_qs_gp, LOAD_SHARED(urcu_gp_ctr));
 	smp_mb();
 }
 #else /* !(BITS_PER_LONG < 64) */
@@ -240,6 +284,12 @@ void *rcu_assign_pointer_sym(void **p, void *v)
 	return STORE_SHARED(p, v);
 }
 
+void *rcu_cmpxchg_pointer_sym(void **p, void *old, void *_new)
+{
+	wmb();
+	return cmpxchg(p, old, _new);
+}
+
 void *rcu_xchg_pointer_sym(void **p, void *v)
 {
 	wmb();