X-Git-Url: http://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu.h;h=0ff0877ac4ac45d6966d674d100b3b47fb7ecbbd;hp=fb8cedf3e15beab5ba5355b153d0bb999fa1c7fc;hb=3db2d75b432e617014976239f694b91de2bc0d7d;hpb=f4a486ac095bd844d0c18995737d589b8f085b69 diff --git a/urcu.h b/urcu.h index fb8cedf..0ff0877 100644 --- a/urcu.h +++ b/urcu.h @@ -18,59 +18,35 @@ */ #include +#include /* The "volatile" is due to gcc bugs */ #define barrier() __asm__ __volatile__("": : :"memory") -/* x86 32/64 specific */ -#define mb() asm volatile("mfence":::"memory") -#define rmb() asm volatile("lfence":::"memory") -#define wmb() asm volatile("sfence" ::: "memory") +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) -static inline void atomic_inc(int *v) -{ - asm volatile("lock; incl %0" - : "+m" (*v)); -} +/* Assume SMP machine, given we don't have this information */ +#define CONFIG_SMP 1 -#define xchg(ptr, v) \ - ((__typeof__(*(ptr)))__xchg((unsigned long)(v), (ptr), sizeof(*(ptr)))) -struct __xchg_dummy { - unsigned long a[100]; -}; -#define __xg(x) ((struct __xchg_dummy *)(x)) +#ifdef CONFIG_SMP +#define smp_mb() mb() +#define smp_rmb() rmb() +#define smp_wmb() wmb() +#define smp_mc() mc() +#define smp_rmc() rmc() +#define smp_wmc() wmc() +#else +#define smp_mb() barrier() +#define smp_rmb() barrier() +#define smp_wmb() barrier() +#define smp_mc() barrier() +#define smp_rmc() barrier() +#define smp_wmc() barrier() +#endif -/* - * Note: no "lock" prefix even on SMP: xchg always implies lock anyway - * Note 2: xchg has side effect, so that attribute volatile is necessary, - * but generally the primitive is invalid, *ptr is output argument. --ANK - */ -static inline unsigned long __xchg(unsigned long x, volatile void *ptr, - int size) -{ - switch (size) { - case 1: - asm volatile("xchgb %b0,%1" - : "=q" (x) - : "m" (*__xg(ptr)), "0" (x) - : "memory"); - break; - case 2: - asm volatile("xchgw %w0,%1" - : "=r" (x) - : "m" (*__xg(ptr)), "0" (x) - : "memory"); - break; - case 4: - asm volatile("xchgl %0,%1" - : "=r" (x) - : "m" (*__xg(ptr)), "0" (x) - : "memory"); - break; - } - return x; -} +#include "arch.h" /* Nop everywhere except on alpha. */ #define smp_read_barrier_depends() @@ -89,6 +65,39 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, */ #define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x)) +/* + * Identify a shared load. A smp_rmc() or smp_mc() should come before the load. + */ +#define _LOAD_SHARED(p) ACCESS_ONCE(p) + +/* + * Load a data from shared memory, doing a cache flush if required. + */ +#define LOAD_SHARED(p) \ + ({ \ + smp_rmc(); \ + _LOAD_SHARED(p); \ + }) + + +/* + * Identify a shared store. A smp_wmc() or smp_mc() should follow the store. + */ +#define _STORE_SHARED(x, v) \ + do { \ + (x) = (v); \ + } while (0) + +/* + * Store v into x, where x is located in shared memory. Performs the required + * cache flush after writing. + */ +#define STORE_SHARED(x, v) \ + do { \ + _STORE_SHARED(x, v); \ + smp_wmc(); \ + } while (0) + /** * rcu_dereference - fetch an RCU-protected pointer in an * RCU read-side critical section. This pointer may later @@ -100,19 +109,37 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, */ #define rcu_dereference(p) ({ \ - typeof(p) _________p1 = ACCESS_ONCE(p); \ + typeof(p) _________p1 = LOAD_SHARED(p); \ smp_read_barrier_depends(); \ (_________p1); \ }) #define SIGURCU SIGUSR1 +/* + * If a reader is really non-cooperative and refuses to commit its + * urcu_active_readers count to memory (there is no barrier in the reader + * per-se), kick it after a few loops waiting for it. + */ +#define KICK_READER_LOOPS 10000 + #ifdef DEBUG_YIELD #include +#include +#include +#include #define YIELD_READ (1 << 0) #define YIELD_WRITE (1 << 1) +/* Updates without DEBUG_FULL_MB are much slower. Account this in the delay */ +#ifdef DEBUG_FULL_MB +/* maximum sleep delay, in us */ +#define MAX_SLEEP 50 +#else +#define MAX_SLEEP 30000 +#endif + extern unsigned int yield_active; extern unsigned int __thread rand_yield; @@ -120,14 +147,14 @@ static inline void debug_yield_read(void) { if (yield_active & YIELD_READ) if (rand_r(&rand_yield) & 0x1) - sched_yield(); + usleep(rand_r(&rand_yield) % MAX_SLEEP); } static inline void debug_yield_write(void) { if (yield_active & YIELD_WRITE) if (rand_r(&rand_yield) & 0x1) - sched_yield(); + usleep(rand_r(&rand_yield) % MAX_SLEEP); } static inline void debug_yield_init(void) @@ -149,63 +176,83 @@ static inline void debug_yield_init(void) } #endif +#ifdef DEBUG_FULL_MB +static inline void reader_barrier() +{ + smp_mb(); +} +#else +static inline void reader_barrier() +{ + barrier(); +} +#endif + /* - * Limiting the nesting level to 256 to keep instructions small in the read - * fast-path. + * The trick here is that RCU_GP_CTR_BIT must be a multiple of 8 so we can use a + * full 8-bits, 16-bits or 32-bits bitmask for the lower order bits. */ -#define RCU_GP_COUNT (1U << 0) -#define RCU_GP_CTR_BIT (1U << 8) +#define RCU_GP_COUNT (1UL << 0) +/* Use the amount of bits equal to half of the architecture long size */ +#define RCU_GP_CTR_BIT (1UL << (sizeof(long) << 2)) #define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_BIT - 1) -/* Global quiescent period counter with low-order bits unused. */ -extern int urcu_gp_ctr; +/* + * Global quiescent period counter with low-order bits unused. + * Using a int rather than a char to eliminate false register dependencies + * causing stalls on some architectures. + */ +extern long urcu_gp_ctr; -extern int __thread urcu_active_readers; +extern long __thread urcu_active_readers; -static inline int rcu_old_gp_ongoing(int *value) +static inline int rcu_old_gp_ongoing(long *value) { - int v; + long v; if (value == NULL) return 0; - debug_yield_write(); - v = ACCESS_ONCE(*value); - debug_yield_write(); + /* + * Make sure both tests below are done on the same version of *value + * to insure consistency. + */ + v = LOAD_SHARED(*value); return (v & RCU_GP_CTR_NEST_MASK) && - ((v ^ ACCESS_ONCE(urcu_gp_ctr)) & RCU_GP_CTR_BIT); + ((v ^ urcu_gp_ctr) & RCU_GP_CTR_BIT); } static inline void rcu_read_lock(void) { - int tmp; + long tmp; - debug_yield_read(); tmp = urcu_active_readers; - debug_yield_read(); - if (!(tmp & RCU_GP_CTR_NEST_MASK)) - urcu_active_readers = urcu_gp_ctr + RCU_GP_COUNT; + /* urcu_gp_ctr = RCU_GP_COUNT | (~RCU_GP_CTR_BIT or RCU_GP_CTR_BIT) */ + /* + * The data dependency "read urcu_gp_ctr, write urcu_active_readers", + * serializes those two memory operations. The memory barrier in the + * signal handler ensures we receive the proper memory commit barriers + * required by _STORE_SHARED and _LOAD_SHARED whenever communication + * with the writer is needed. + */ + if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) + _STORE_SHARED(urcu_active_readers, _LOAD_SHARED(urcu_gp_ctr)); else - urcu_active_readers = tmp + RCU_GP_COUNT; - debug_yield_read(); + _STORE_SHARED(urcu_active_readers, tmp + RCU_GP_COUNT); /* * Increment active readers count before accessing the pointer. * See force_mb_all_threads(). */ - barrier(); - debug_yield_read(); + reader_barrier(); } static inline void rcu_read_unlock(void) { - debug_yield_read(); - barrier(); - debug_yield_read(); + reader_barrier(); /* * Finish using rcu before decrementing the pointer. * See force_mb_all_threads(). */ - urcu_active_readers -= RCU_GP_COUNT; - debug_yield_read(); + _STORE_SHARED(urcu_active_readers, urcu_active_readers - RCU_GP_COUNT); } /** @@ -226,7 +273,7 @@ static inline void rcu_read_unlock(void) if (!__builtin_constant_p(v) || \ ((v) != NULL)) \ wmb(); \ - (p) = (v); \ + STORE_SHARED(p, v); \ }) #define rcu_xchg_pointer(p, v) \ @@ -246,7 +293,6 @@ extern void synchronize_rcu(void); #define urcu_publish_content(p, v) \ ({ \ void *oldptr; \ - debug_yield_write(); \ oldptr = rcu_xchg_pointer(p, v); \ synchronize_rcu(); \ oldptr; \