qsbr: only mark reader thread as being waited for in contended case

[userspace-rcu.git] / urcu.c

diff --git a/urcu.c b/urcu.c
index 32f74880492ce40527c8b5b40560249eac485b90..f74304ca51fa3c2c0aed789585192b06646eb5c9 100644 (file)
--- a/urcu.c
+++ b/urcu.c
@@ -61,14 +61,14 @@ long urcu_gp_ctr = RCU_GP_COUNT;
  * Written to only by each individual reader. Read by both the reader and the
  * writers.
  */
-long __thread urcu_active_readers;
+struct urcu_reader_status __thread urcu_reader_status;
 
 /* Thread IDs of registered readers */
 #define INIT_NUM_THREADS 4
 
 struct reader_registry {
        pthread_t tid;
-       long *urcu_active_readers;
+       struct urcu_reader_status *urcu_reader_status;
        char *need_mb;
 };
 
@@ -215,27 +215,46 @@ void wait_for_quiescent_state(void)
        if (!registry)
                return;
        /*
-        * Wait for each thread urcu_active_readers count to become 0.
+        * Wait for each thread active_readers count to become 0.
         */
        for (index = registry; index < registry + num_readers; index++) {
+               int wait_loops = 0;
+
+               if (likely(!rcu_old_gp_ongoing(
+                               &index->urcu_reader_status->active_readers)))
+                       continue;
+
+               index->urcu_reader_status->gp_waiting = 1;
 #ifndef HAS_INCOHERENT_CACHES
-               while (rcu_old_gp_ongoing(index->urcu_active_readers))
-                       cpu_relax();
+               while (rcu_old_gp_ongoing(
+                               &index->urcu_reader_status->active_readers)) {
+                       if (wait_loops++ == RCU_QS_ACTIVE_ATTEMPTS) {
+                               sched_yield();  /* ideally sched_yield_to() */
+                       } else {
+                               cpu_relax();
+                       }
+               }
 #else /* #ifndef HAS_INCOHERENT_CACHES */
-               int wait_loops = 0;
                /*
                 * BUSY-LOOP. Force the reader thread to commit its
-                * urcu_active_readers update to memory if we wait for too long.
+                * active_readers update to memory if we wait for too long.
                 */
-               while (rcu_old_gp_ongoing(index->urcu_active_readers)) {
-                       if (wait_loops++ == KICK_READER_LOOPS) {
+               while (rcu_old_gp_ongoing(
+                               &index->urcu_reader_status->active_readers)) {
+                       switch (wait_loops++) {
+                       case RCU_QS_ACTIVE_ATTEMPTS:
+                               sched_yield();  /* ideally sched_yield_to() */
+                               break;
+                       case KICK_READER_LOOPS:
                                force_mb_single_thread(index);
                                wait_loops = 0;
-                       } else {
+                               break;
+                       default:
                                cpu_relax();
                        }
                }
 #endif /* #else #ifndef HAS_INCOHERENT_CACHES */
+               index->urcu_reader_status->gp_waiting = 0;
        }
 }
 
@@ -249,6 +268,8 @@ void synchronize_rcu(void)
        /* Write new ptr before changing the qparity */
        force_mb_all_threads();
 
+       STORE_SHARED(urcu_gp_ctr, urcu_gp_ctr ^ RCU_GP_ONGOING);
+
        switch_next_urcu_qparity();     /* 0 -> 1 */
 
        /*
@@ -308,6 +329,8 @@ void synchronize_rcu(void)
         */
        wait_for_quiescent_state();     /* Wait readers in parity 1 */
 
+       STORE_SHARED(urcu_gp_ctr, urcu_gp_ctr ^ RCU_GP_ONGOING);
+
        /* Finish waiting for reader threads before letting the old ptr being
         * freed. Must be done within internal_urcu_lock because it iterates on
         * reader threads. */
@@ -383,7 +406,7 @@ static void rcu_add_reader(pthread_t id)
        }
        registry[num_readers].tid = id;
        /* reference to the TLS of _this_ reader thread. */
-       registry[num_readers].urcu_active_readers = &urcu_active_readers;
+       registry[num_readers].urcu_reader_status = &urcu_reader_status;
        registry[num_readers].need_mb = &need_mb;
        num_readers++;
 }
@@ -402,7 +425,7 @@ static void rcu_remove_reader(pthread_t id)
                        memcpy(index, &registry[num_readers - 1],
                                sizeof(struct reader_registry));
                        registry[num_readers - 1].tid = 0;
-                       registry[num_readers - 1].urcu_active_readers = NULL;
+                       registry[num_readers - 1].urcu_reader_status = NULL;
                        num_readers--;
                        return;
                }