dual writer fix

[urcu.git] / urcu.h

diff --git a/urcu.h b/urcu.h
index 2adca2286fe8ba86c74b902456a833ecf60f09eb..539b1d5588adc64c40bf5affdcef1fca6825f4b1 100644 (file)
--- a/urcu.h
+++ b/urcu.h
@@ -180,22 +180,36 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
 #define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
 
 /*
- * Load a data from remote memory, doing a cache flush if required.
+ * Identify a shared load. A smp_rmc() or smp_mc() should come before the load.
  */
-#define LOAD_REMOTE(p)        ({ \
-                               smp_rmc(); \
-                               typeof(p) _________p1 = ACCESS_ONCE(p); \
-                               (_________p1); \
-                               })
+#define _LOAD_SHARED(p)               ACCESS_ONCE(p)
 
 /*
- * Store v into x, where x is located in remote memory. Performs the required
- * cache flush after writing.
+ * Load a data from shared memory, doing a cache flush if required.
+ */
+#define LOAD_SHARED(p) \
+       ({ \
+               smp_rmc(); \
+               _LOAD_SHARED(p); \
+       })
+
+
+/*
+ * Identify a shared store. A smp_wmc() or smp_mc() should follow the store.
  */
-#define STORE_REMOTE(x, v) \
+#define _STORE_SHARED(x, v) \
        do { \
                (x) = (v); \
-               smp_wmc; \
+       } while (0)
+
+/*
+ * Store v into x, where x is located in shared memory. Performs the required
+ * cache flush after writing.
+ */
+#define STORE_SHARED(x, v) \
+       do { \
+               _STORE_SHARED(x, v); \
+               smp_wmc(); \
        } while (0)
 
 /**
@@ -209,13 +223,11 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
  */
 
 #define rcu_dereference(p)     ({ \
-                               typeof(p) _________p1 = LOAD_REMOTE(p); \
+                               typeof(p) _________p1 = LOAD_SHARED(p); \
                                smp_read_barrier_depends(); \
                                (_________p1); \
                                })
 
-
-
 #define SIGURCU SIGUSR1
 
 /*
@@ -318,7 +330,7 @@ static inline int rcu_old_gp_ongoing(long *value)
         * Make sure both tests below are done on the same version of *value
         * to insure consistency.
         */
-       v = LOAD_REMOTE(*value);
+       v = LOAD_SHARED(*value);
        return (v & RCU_GP_CTR_NEST_MASK) &&
                 ((v ^ urcu_gp_ctr) & RCU_GP_CTR_BIT);
 }
@@ -331,15 +343,15 @@ static inline void rcu_read_lock(void)
        /* urcu_gp_ctr = RCU_GP_COUNT | (~RCU_GP_CTR_BIT or RCU_GP_CTR_BIT) */
        /*
         * The data dependency "read urcu_gp_ctr, write urcu_active_readers",
-        * serializes those two memory operations. We are not using STORE_REMOTE
-        * and LOAD_REMOTE here (although we should) because the writer will
-        * wake us up with a signal which does a flush in its handler to perform
-        * urcu_gp_ctr re-read and urcu_active_readers commit to main memory.
+        * serializes those two memory operations. The memory barrier in the
+        * signal handler ensures we receive the proper memory commit barriers
+        * required by _STORE_SHARED and _LOAD_SHARED whenever communication
+        * with the writer is needed.
         */
        if (likely(!(tmp & RCU_GP_CTR_NEST_MASK)))
-               urcu_active_readers = ACCESS_ONCE(urcu_gp_ctr);
+               _STORE_SHARED(urcu_active_readers, _LOAD_SHARED(urcu_gp_ctr));
        else
-               urcu_active_readers = tmp + RCU_GP_COUNT;
+               _STORE_SHARED(urcu_active_readers, tmp + RCU_GP_COUNT);
        /*
         * Increment active readers count before accessing the pointer.
         * See force_mb_all_threads().
@@ -354,7 +366,7 @@ static inline void rcu_read_unlock(void)
         * Finish using rcu before decrementing the pointer.
         * See force_mb_all_threads().
         */
-       urcu_active_readers -= RCU_GP_COUNT;
+       _STORE_SHARED(urcu_active_readers, urcu_active_readers - RCU_GP_COUNT);
 }
 
 /**
@@ -375,8 +387,7 @@ static inline void rcu_read_unlock(void)
                if (!__builtin_constant_p(v) || \
                    ((v) != NULL)) \
                        wmb(); \
-               (p) = (v); \
-               smp_wmc();
+               STORE_SHARED(p, v); \
        })
 
 #define rcu_xchg_pointer(p, v) \