qsbr: only mark reader thread as being waited for in contended case

[userspace-rcu.git] / urcu-static.h

diff --git a/urcu-static.h b/urcu-static.h
index 1921097dbb8596a9d134fecdea5e2bcfe9444c2d..efb8225161b2c3c99c450d68a83289f1acb0379d 100644 (file)
--- a/urcu-static.h
+++ b/urcu-static.h
@@ -31,6 +31,7 @@
 
 #include <stdlib.h>
 #include <pthread.h>
+#include <sched.h>
 
 #include <compiler.h>
 #include <arch.h>
@@ -76,6 +77,15 @@
  * Inserts memory barriers on architectures that require them (currently only
  * Alpha) and documents which pointers are protected by RCU.
  *
+ * The compiler memory barrier in LOAD_SHARED() ensures that value-speculative
+ * optimizations (e.g. VSS: Value Speculation Scheduling) does not perform the
+ * data read before the pointer read by speculating the value of the pointer.
+ * Correct ordering is ensured because the pointer is read as a volatile access.
+ * This acts as a global side-effect operation, which forbids reordering of
+ * dependent memory operations. Note that such concern about dependency-breaking
+ * optimizations will eventually be taken care of by the "memory_order_consume"
+ * addition to forthcoming C++ standard.
+ *
  * Should match rcu_assign_pointer() or rcu_xchg_pointer().
  */
 
@@ -108,6 +118,17 @@
  */
 #define KICK_READER_LOOPS 10000
 
+/*
+ * Active attempts to check for reader Q.S. before calling sched_yield().
+ */
+#define RCU_QS_ACTIVE_ATTEMPTS 100
+
+#ifdef DEBUG_RCU
+#define rcu_assert(args...)    assert(args)
+#else
+#define rcu_assert(args...)
+#endif
+
 #ifdef DEBUG_YIELD
 #include <sched.h>
 #include <time.h>
@@ -118,10 +139,10 @@
 #define YIELD_WRITE    (1 << 1)
 
 /*
- * Updates without CONFIG_URCU_AVOID_SIGNALS are much slower. Account this in
+ * Updates without URCU_MB are much slower. Account this in
  * the delay.
  */
-#ifdef CONFIG_URCU_AVOID_SIGNALS
+#ifdef URCU_MB
 /* maximum sleep delay, in us */
 #define MAX_SLEEP 50
 #else
@@ -164,7 +185,7 @@ static inline void debug_yield_init(void)
 }
 #endif
 
-#ifdef CONFIG_URCU_AVOID_SIGNALS
+#ifdef URCU_MB
 static inline void reader_barrier()
 {
        smp_mb();
@@ -184,6 +205,7 @@ static inline void reader_barrier()
 /* Use the amount of bits equal to half of the architecture long size */
 #define RCU_GP_CTR_BIT         (1UL << (sizeof(long) << 2))
 #define RCU_GP_CTR_NEST_MASK   (RCU_GP_CTR_BIT - 1)
+#define RCU_GP_ONGOING         (RCU_GP_CTR_BIT << 1)
 
 /*
  * Global quiescent period counter with low-order bits unused.
@@ -192,7 +214,12 @@ static inline void reader_barrier()
  */
 extern long urcu_gp_ctr;
 
-extern long __thread urcu_active_readers;
+struct urcu_reader_status {
+       long active_readers;
+       long gp_waiting;
+};
+
+extern struct urcu_reader_status __thread urcu_reader_status;
 
 static inline int rcu_old_gp_ongoing(long *value)
 {
@@ -211,25 +238,35 @@ static inline int rcu_old_gp_ongoing(long *value)
 
 static inline void _rcu_read_lock(void)
 {
-       long tmp;
+       long tmp, gp_ctr;
 
-       tmp = urcu_active_readers;
+       tmp = urcu_reader_status.active_readers;
        /* urcu_gp_ctr = RCU_GP_COUNT | (~RCU_GP_CTR_BIT or RCU_GP_CTR_BIT) */
        if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
-               _STORE_SHARED(urcu_active_readers, _LOAD_SHARED(urcu_gp_ctr));
+               /*
+                * volatile accesses can be reordered and optimized when within
+                * the same statement.
+                */
+               if (unlikely((gp_ctr = _LOAD_SHARED(urcu_gp_ctr))
+                               & RCU_GP_ONGOING) &&
+                   unlikely(LOAD_SHARED(urcu_reader_status.gp_waiting))) {
+                       sched_yield();
+                       gp_ctr = _LOAD_SHARED(urcu_gp_ctr);
+               }
+               _STORE_SHARED(urcu_reader_status.active_readers, gp_ctr);
                /*
                 * Set active readers count for outermost nesting level before
                 * accessing the pointer. See force_mb_all_threads().
                 */
                reader_barrier();
        } else {
-               _STORE_SHARED(urcu_active_readers, tmp + RCU_GP_COUNT);
+               _STORE_SHARED(urcu_reader_status.active_readers,
+                             tmp + RCU_GP_COUNT);
        }
 }
 
 static inline void _rcu_read_unlock(void)
 {
-       reader_barrier();
        /*
         * Finish using rcu before decrementing the pointer.
         * See force_mb_all_threads().
@@ -237,7 +274,9 @@ static inline void _rcu_read_unlock(void)
         * in place for nested unlocks to remove a branch from the common case
         * (no nesting).
         */
-       _STORE_SHARED(urcu_active_readers, urcu_active_readers - RCU_GP_COUNT);
+       reader_barrier();
+       _STORE_SHARED(urcu_reader_status.active_readers,
+                     urcu_reader_status.active_readers - RCU_GP_COUNT);
 }
 
 /**
@@ -261,6 +300,22 @@ static inline void _rcu_read_unlock(void)
                STORE_SHARED(p, v);                     \
        })
 
+/**
+ * _rcu_cmpxchg_pointer - same as rcu_assign_pointer, but tests if the pointer
+ * is as expected by "old". If succeeds, returns the previous pointer to the
+ * data structure, which can be safely freed after waiting for a quiescent state
+ * using synchronize_rcu(). If fails (unexpected value), returns old (which
+ * should not be freed !).
+ */
+
+#define _rcu_cmpxchg_pointer(p, old, _new)             \
+       ({                                              \
+               if (!__builtin_constant_p(_new) ||      \
+                   ((_new) != NULL))                   \
+                       wmb();                          \
+               cmpxchg(p, old, _new);                  \
+       })
+
 /**
  * _rcu_xchg_pointer - same as rcu_assign_pointer, but returns the previous
  * pointer to the data structure, which can be safely freed after waiting for a