X-Git-Url: http://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu.h;h=03764ab33dbe241b643b72ef9cbfd6757662fa3b;hp=01a4c6857d7e6fd31668dec57615be03e94ccc91;hb=bb48818526ec4317f9e6daeb0aa1cd64d528f754;hpb=1430ee0bdca4cb454d534ef7fc84af3e0692f26b diff --git a/urcu.h b/urcu.h index 01a4c68..03764ab 100644 --- a/urcu.h +++ b/urcu.h @@ -18,10 +18,14 @@ */ #include +#include /* The "volatile" is due to gcc bugs */ #define barrier() __asm__ __volatile__("": : :"memory") +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) + /* x86 32/64 specific */ #define mb() asm volatile("mfence":::"memory") #define rmb() asm volatile("lfence":::"memory") @@ -33,6 +37,51 @@ static inline void atomic_inc(int *v) : "+m" (*v)); } +#define xchg(ptr, v) \ + ((__typeof__(*(ptr)))__xchg((unsigned long)(v), (ptr), sizeof(*(ptr)))) + +struct __xchg_dummy { + unsigned long a[100]; +}; +#define __xg(x) ((struct __xchg_dummy *)(x)) + +/* + * Note: no "lock" prefix even on SMP: xchg always implies lock anyway + * Note 2: xchg has side effect, so that attribute volatile is necessary, + * but generally the primitive is invalid, *ptr is output argument. --ANK + */ +static inline unsigned long __xchg(unsigned long x, volatile void *ptr, + int size) +{ + switch (size) { + case 1: + asm volatile("xchgb %b0,%1" + : "=q" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; + case 2: + asm volatile("xchgw %w0,%1" + : "=r" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; + case 4: + asm volatile("xchgl %k0,%1" + : "=r" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; + case 8: + asm volatile("xchgq %0,%1" + : "=r" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; + } + return x; +} + /* Nop everywhere except on alpha. */ #define smp_read_barrier_depends() @@ -70,10 +119,21 @@ static inline void atomic_inc(int *v) #ifdef DEBUG_YIELD #include +#include +#include +#include #define YIELD_READ (1 << 0) #define YIELD_WRITE (1 << 1) +/* Updates without DEBUG_FULL_MB are much slower. Account this in the delay */ +#ifdef DEBUG_FULL_MB +/* maximum sleep delay, in us */ +#define MAX_SLEEP 50 +#else +#define MAX_SLEEP 30000 +#endif + extern unsigned int yield_active; extern unsigned int __thread rand_yield; @@ -81,14 +141,14 @@ static inline void debug_yield_read(void) { if (yield_active & YIELD_READ) if (rand_r(&rand_yield) & 0x1) - sched_yield(); + usleep(rand_r(&rand_yield) % MAX_SLEEP); } static inline void debug_yield_write(void) { if (yield_active & YIELD_WRITE) if (rand_r(&rand_yield) & 0x1) - sched_yield(); + usleep(rand_r(&rand_yield) % MAX_SLEEP); } static inline void debug_yield_init(void) @@ -110,22 +170,39 @@ static inline void debug_yield_init(void) } #endif +#ifdef DEBUG_FULL_MB +static inline void read_barrier() +{ + mb(); +} +#else +static inline void read_barrier() +{ + barrier(); +} +#endif + /* - * Limiting the nesting level to 256 to keep instructions small in the read - * fast-path. + * The trick here is that RCU_GP_CTR_BIT must be a multiple of 8 so we can use a + * full 8-bits, 16-bits or 32-bits bitmask for the lower order bits. */ -#define RCU_GP_COUNT (1U << 0) -#define RCU_GP_CTR_BIT (1U << 8) +#define RCU_GP_COUNT (1UL << 0) +/* Use the amount of bits equal to half of the architecture long size */ +#define RCU_GP_CTR_BIT (1UL << (sizeof(long) << 2)) #define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_BIT - 1) -/* Global quiescent period counter with low-order bits unused. */ -extern int urcu_gp_ctr; +/* + * Global quiescent period counter with low-order bits unused. + * Using a int rather than a char to eliminate false register dependencies + * causing stalls on some architectures. + */ +extern long urcu_gp_ctr; -extern int __thread urcu_active_readers; +extern long __thread urcu_active_readers; -static inline int rcu_old_gp_ongoing(int *value) +static inline int rcu_old_gp_ongoing(long *value) { - int v; + long v; if (value == NULL) return 0; @@ -138,13 +215,14 @@ static inline int rcu_old_gp_ongoing(int *value) static inline void rcu_read_lock(void) { - int tmp; + long tmp; debug_yield_read(); tmp = urcu_active_readers; debug_yield_read(); - if (!(tmp & RCU_GP_CTR_NEST_MASK)) - urcu_active_readers = urcu_gp_ctr + RCU_GP_COUNT; + /* urcu_gp_ctr = RCU_GP_COUNT | (~RCU_GP_CTR_BIT or RCU_GP_CTR_BIT) */ + if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) + urcu_active_readers = urcu_gp_ctr; else urcu_active_readers = tmp + RCU_GP_COUNT; debug_yield_read(); @@ -152,14 +230,14 @@ static inline void rcu_read_lock(void) * Increment active readers count before accessing the pointer. * See force_mb_all_threads(). */ - barrier(); + read_barrier(); debug_yield_read(); } static inline void rcu_read_unlock(void) { debug_yield_read(); - barrier(); + read_barrier(); debug_yield_read(); /* * Finish using rcu before decrementing the pointer. @@ -169,7 +247,49 @@ static inline void rcu_read_unlock(void) debug_yield_read(); } -extern void *urcu_publish_content(void **ptr, void *new); +/** + * rcu_assign_pointer - assign (publicize) a pointer to a newly + * initialized structure that will be dereferenced by RCU read-side + * critical sections. Returns the value assigned. + * + * Inserts memory barriers on architectures that require them + * (pretty much all of them other than x86), and also prevents + * the compiler from reordering the code that initializes the + * structure after the pointer assignment. More importantly, this + * call documents which pointers will be dereferenced by RCU read-side + * code. + */ + +#define rcu_assign_pointer(p, v) \ + ({ \ + if (!__builtin_constant_p(v) || \ + ((v) != NULL)) \ + wmb(); \ + (p) = (v); \ + }) + +#define rcu_xchg_pointer(p, v) \ + ({ \ + if (!__builtin_constant_p(v) || \ + ((v) != NULL)) \ + wmb(); \ + xchg(p, v); \ + }) + +extern void synchronize_rcu(void); + +/* + * Exchanges the pointer and waits for quiescent state. + * The pointer returned can be freed. + */ +#define urcu_publish_content(p, v) \ + ({ \ + void *oldptr; \ + debug_yield_write(); \ + oldptr = rcu_xchg_pointer(p, v); \ + synchronize_rcu(); \ + oldptr; \ + }) /* * Reader thread registration.