X-Git-Url: http://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu.h;h=0ff0877ac4ac45d6966d674d100b3b47fb7ecbbd;hp=3eca5ea3c44ccc7135635229c41141d5050df1e2;hb=3db2d75b432e617014976239f694b91de2bc0d7d;hpb=40e140c9f237bbe8c50d4a5355561139f01afb70

diff --git a/urcu.h b/urcu.h
index 3eca5ea..0ff0877 100644
--- a/urcu.h
+++ b/urcu.h
@@ -26,74 +26,27 @@
 #define likely(x)       __builtin_expect(!!(x), 1)
 #define unlikely(x)     __builtin_expect(!!(x), 0)
 
-/* x86 32/64 specific */
-#define mb()    asm volatile("mfence":::"memory")
-#define rmb()   asm volatile("lfence":::"memory")
-#define wmb()   asm volatile("sfence" ::: "memory")
-
 /* Assume SMP machine, given we don't have this information */
 #define CONFIG_SMP 1
 
+
 #ifdef CONFIG_SMP
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()
 #define smp_wmb()	wmb()
+#define smp_mc()	mc()
+#define smp_rmc()	rmc()
+#define smp_wmc()	wmc()
 #else
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
+#define smp_mc()	barrier()
+#define smp_rmc()	barrier()
+#define smp_wmc()	barrier()
 #endif
 
-static inline void atomic_inc(int *v)
-{
-	asm volatile("lock; incl %0"
-		     : "+m" (*v));
-}
-
-#define xchg(ptr, v)							\
-	((__typeof__(*(ptr)))__xchg((unsigned long)(v), (ptr), sizeof(*(ptr))))
-
-struct __xchg_dummy {
-	unsigned long a[100];
-};
-#define __xg(x) ((struct __xchg_dummy *)(x))
-
-/*
- * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
- * Note 2: xchg has side effect, so that attribute volatile is necessary,
- *	  but generally the primitive is invalid, *ptr is output argument. --ANK
- */
-static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
-				   int size)
-{
-	switch (size) {
-	case 1:
-		asm volatile("xchgb %b0,%1"
-			     : "=q" (x)
-			     : "m" (*__xg(ptr)), "0" (x)
-			     : "memory");
-		break;
-	case 2:
-		asm volatile("xchgw %w0,%1"
-			     : "=r" (x)
-			     : "m" (*__xg(ptr)), "0" (x)
-			     : "memory");
-		break;
-	case 4:
-		asm volatile("xchgl %k0,%1"
-			     : "=r" (x)
-			     : "m" (*__xg(ptr)), "0" (x)
-			     : "memory");
-		break;
-	case 8:
-		asm volatile("xchgq %0,%1"
-			     : "=r" (x)
-			     : "m" (*__xg(ptr)), "0" (x)
-			     : "memory");
-		break;
-	}
-	return x;
-}
+#include "arch.h"
 
 /* Nop everywhere except on alpha. */
 #define smp_read_barrier_depends()
@@ -112,6 +65,39 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
  */
 #define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
 
+/*
+ * Identify a shared load. A smp_rmc() or smp_mc() should come before the load.
+ */
+#define _LOAD_SHARED(p)	       ACCESS_ONCE(p)
+
+/*
+ * Load a data from shared memory, doing a cache flush if required.
+ */
+#define LOAD_SHARED(p) \
+	({ \
+		smp_rmc(); \
+		_LOAD_SHARED(p); \
+	})
+
+
+/*
+ * Identify a shared store. A smp_wmc() or smp_mc() should follow the store.
+ */
+#define _STORE_SHARED(x, v) \
+	do { \
+		(x) = (v); \
+	} while (0)
+
+/*
+ * Store v into x, where x is located in shared memory. Performs the required
+ * cache flush after writing.
+ */
+#define STORE_SHARED(x, v) \
+	do { \
+		_STORE_SHARED(x, v); \
+		smp_wmc(); \
+	} while (0)
+
 /**
  * rcu_dereference - fetch an RCU-protected pointer in an
  * RCU read-side critical section.  This pointer may later
@@ -123,7 +109,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
  */
 
 #define rcu_dereference(p)     ({ \
-				typeof(p) _________p1 = ACCESS_ONCE(p); \
+				typeof(p) _________p1 = LOAD_SHARED(p); \
 				smp_read_barrier_depends(); \
 				(_________p1); \
 				})
@@ -191,12 +177,12 @@ static inline void debug_yield_init(void)
 #endif
 
 #ifdef DEBUG_FULL_MB
-static inline void read_barrier()
+static inline void reader_barrier()
 {
 	smp_mb();
 }
 #else
-static inline void read_barrier()
+static inline void reader_barrier()
 {
 	barrier();
 }
@@ -230,7 +216,7 @@ static inline int rcu_old_gp_ongoing(long *value)
 	 * Make sure both tests below are done on the same version of *value
 	 * to insure consistency.
 	 */
-	v = ACCESS_ONCE(*value);
+	v = LOAD_SHARED(*value);
 	return (v & RCU_GP_CTR_NEST_MASK) &&
 		 ((v ^ urcu_gp_ctr) & RCU_GP_CTR_BIT);
 }
@@ -241,27 +227,32 @@ static inline void rcu_read_lock(void)
 
 	tmp = urcu_active_readers;
 	/* urcu_gp_ctr = RCU_GP_COUNT | (~RCU_GP_CTR_BIT or RCU_GP_CTR_BIT) */
-	/* The data dependency "read urcu_gp_ctr, write urcu_active_readers",
-	 * serializes those two memory operations. */
+	/*
+	 * The data dependency "read urcu_gp_ctr, write urcu_active_readers",
+	 * serializes those two memory operations. The memory barrier in the
+	 * signal handler ensures we receive the proper memory commit barriers
+	 * required by _STORE_SHARED and _LOAD_SHARED whenever communication
+	 * with the writer is needed.
+	 */
 	if (likely(!(tmp & RCU_GP_CTR_NEST_MASK)))
-		urcu_active_readers = urcu_gp_ctr;
+		_STORE_SHARED(urcu_active_readers, _LOAD_SHARED(urcu_gp_ctr));
 	else
-		urcu_active_readers = tmp + RCU_GP_COUNT;
+		_STORE_SHARED(urcu_active_readers, tmp + RCU_GP_COUNT);
 	/*
 	 * Increment active readers count before accessing the pointer.
 	 * See force_mb_all_threads().
 	 */
-	read_barrier();
+	reader_barrier();
 }
 
 static inline void rcu_read_unlock(void)
 {
-	read_barrier();
+	reader_barrier();
 	/*
 	 * Finish using rcu before decrementing the pointer.
 	 * See force_mb_all_threads().
 	 */
-	urcu_active_readers -= RCU_GP_COUNT;
+	_STORE_SHARED(urcu_active_readers, urcu_active_readers - RCU_GP_COUNT);
 }
 
 /**
@@ -282,7 +273,7 @@ static inline void rcu_read_unlock(void)
 		if (!__builtin_constant_p(v) || \
 		    ((v) != NULL)) \
 			wmb(); \
-		(p) = (v); \
+		STORE_SHARED(p, v); \
 	})
 
 #define rcu_xchg_pointer(p, v) \