X-Git-Url: http://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu%2Fstatic%2Furcu.h;h=92185d7629a72f2ceb70d109f24bd959e062529d;hp=0295a3e5183cc57315e8488a03a70d0023286d51;hb=a4922ed92bea8e5552a3e66b7dd7b4721b754c90;hpb=9d2614f07691a813a3c560a6c0bcd0a7be854ed5 diff --git a/urcu/static/urcu.h b/urcu/static/urcu.h index 0295a3e..92185d7 100644 --- a/urcu/static/urcu.h +++ b/urcu/static/urcu.h @@ -6,8 +6,8 @@ * * Userspace RCU header. * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu.h for linking - * dynamically with the userspace rcu library. + * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU + * RELEASE. See urcu.h for linking dynamically with the userspace rcu library. * * Copyright (c) 2009 Mathieu Desnoyers * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. @@ -40,6 +40,9 @@ #include #include #include +#include +#include +#include #ifdef __cplusplus extern "C" { @@ -50,30 +53,6 @@ extern "C" { #define RCU_MEMBARRIER #endif -/* - * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB - * otherwise. - */ -#if !defined(__linux__) && defined(RCU_MEMBARRIER) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier(...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif - /* * This code section can only be included in LGPL 2.1 compatible source code. * See below for the function call wrappers which can be used in code meant to @@ -95,83 +74,13 @@ extern "C" { #define SIGRCU SIGUSR1 #endif -/* - * If a reader is really non-cooperative and refuses to commit its - * rcu_active_readers count to memory (there is no barrier in the reader - * per-se), kick it after a few loops waiting for it. - */ -#define KICK_READER_LOOPS 10000 - -/* - * Active attempts to check for reader Q.S. before calling futex(). - */ -#define RCU_QS_ACTIVE_ATTEMPTS 100 - -#ifdef DEBUG_RCU -#define rcu_assert(args...) assert(args) -#else -#define rcu_assert(args...) -#endif - -#ifdef DEBUG_YIELD -#include -#include -#include -#include - -#define YIELD_READ (1 << 0) -#define YIELD_WRITE (1 << 1) - -/* - * Updates with RCU_SIGNAL are much slower. Account this in the delay. - */ -#ifdef RCU_SIGNAL -/* maximum sleep delay, in us */ -#define MAX_SLEEP 30000 -#else -#define MAX_SLEEP 50 -#endif - -extern unsigned int yield_active; -extern unsigned int __thread rand_yield; - -static inline void debug_yield_read(void) -{ - if (yield_active & YIELD_READ) - if (rand_r(&rand_yield) & 0x1) - usleep(rand_r(&rand_yield) % MAX_SLEEP); -} - -static inline void debug_yield_write(void) -{ - if (yield_active & YIELD_WRITE) - if (rand_r(&rand_yield) & 0x1) - usleep(rand_r(&rand_yield) % MAX_SLEEP); -} - -static inline void debug_yield_init(void) -{ - rand_yield = time(NULL) ^ (unsigned long) pthread_self(); -} -#else -static inline void debug_yield_read(void) -{ -} - -static inline void debug_yield_write(void) -{ -} - -static inline void debug_yield_init(void) -{ - -} -#endif +enum rcu_state { + RCU_READER_ACTIVE_CURRENT, + RCU_READER_ACTIVE_OLD, + RCU_READER_INACTIVE, +}; /* - * RCU memory barrier broadcast group. Currently, only broadcast to all process - * threads is supported (group 0). - * * Slave barriers are only guaranteed to be ordered wrt master barriers. * * The pair ordering is detailed as (O: ordered, X: not ordered) : @@ -180,15 +89,12 @@ static inline void debug_yield_init(void) * master O O */ -#define MB_GROUP_ALL 0 -#define RCU_MB_GROUP MB_GROUP_ALL - #ifdef RCU_MEMBARRIER -extern int has_sys_membarrier; +extern int rcu_has_sys_membarrier; -static inline void smp_mb_slave(int group) +static inline void smp_mb_slave(void) { - if (likely(has_sys_membarrier)) + if (caa_likely(rcu_has_sys_membarrier)) cmm_barrier(); else cmm_smp_mb(); @@ -196,14 +102,14 @@ static inline void smp_mb_slave(int group) #endif #ifdef RCU_MB -static inline void smp_mb_slave(int group) +static inline void smp_mb_slave(void) { cmm_smp_mb(); } #endif #ifdef RCU_SIGNAL -static inline void smp_mb_slave(int group) +static inline void smp_mb_slave(void) { cmm_barrier(); } @@ -218,12 +124,20 @@ static inline void smp_mb_slave(int group) #define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2)) #define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) -/* - * Global quiescent period counter with low-order bits unused. - * Using a int rather than a char to eliminate false register dependencies - * causing stalls on some architectures. - */ -extern unsigned long rcu_gp_ctr; +struct rcu_gp { + /* + * Global grace period counter. + * Contains the current RCU_GP_CTR_PHASE. + * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. + * Written to only by writer with mutex taken. + * Read by both writer and readers. + */ + unsigned long ctr; + + int32_t futex; +} __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + +extern struct rcu_gp rcu_gp; struct rcu_reader { /* Data used by both reader and synchronize_rcu() */ @@ -232,25 +146,30 @@ struct rcu_reader { /* Data used for registry */ struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE))); pthread_t tid; + /* Reader registered flag, for internal checks. */ + unsigned int registered:1; }; -extern struct rcu_reader __thread rcu_reader; - -extern int32_t gp_futex; +extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); /* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. */ static inline void wake_up_gp(void) { - if (unlikely(uatomic_read(&gp_futex) == -1)) { - uatomic_set(&gp_futex, 0); - futex_async(&gp_futex, FUTEX_WAKE, 1, - NULL, NULL, 0); + if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) { + uatomic_set(&rcu_gp.futex, 0); + /* + * Ignoring return value until we can make this function + * return something (because urcu_die() is not publicly + * exposed). + */ + (void) futex_async(&rcu_gp.futex, FUTEX_WAKE, 1, + NULL, NULL, 0); } } -static inline int rcu_gp_ongoing(unsigned long *ctr) +static inline enum rcu_state rcu_reader_state(unsigned long *ctr) { unsigned long v; @@ -259,53 +178,97 @@ static inline int rcu_gp_ongoing(unsigned long *ctr) * to insure consistency. */ v = CMM_LOAD_SHARED(*ctr); - return (v & RCU_GP_CTR_NEST_MASK) && - ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE); + if (!(v & RCU_GP_CTR_NEST_MASK)) + return RCU_READER_INACTIVE; + if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE)) + return RCU_READER_ACTIVE_CURRENT; + return RCU_READER_ACTIVE_OLD; } +/* + * Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as + * the per-thread rcu_reader.ctr) has the upper bits containing a count of + * _rcu_read_lock() nesting, and a lower-order bit that contains either zero + * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in + * _rcu_read_lock() happen before the subsequent read-side critical section. + */ +static inline void _rcu_read_lock_update(unsigned long tmp) +{ + if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { + _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp.ctr)); + smp_mb_slave(); + } else + _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp + RCU_GP_COUNT); +} + +/* + * Enter an RCU read-side critical section. + * + * The first cmm_barrier() call ensures that the compiler does not reorder + * the body of _rcu_read_lock() with a mutex. + * + * This function and its helper are both less than 10 lines long. The + * intent is that this function meets the 10-line criterion in LGPL, + * allowing this function to be invoked directly from non-LGPL code. + */ static inline void _rcu_read_lock(void) { unsigned long tmp; - cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ - tmp = rcu_reader.ctr; - /* - * rcu_gp_ctr is - * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE) - */ - if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { - _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); - /* - * Set active readers count for outermost nesting level before - * accessing the pointer. See smp_mb_master(). - */ - smp_mb_slave(RCU_MB_GROUP); - } else { - _CMM_STORE_SHARED(rcu_reader.ctr, tmp + RCU_GP_COUNT); - } + urcu_assert(URCU_TLS(rcu_reader).registered); + cmm_barrier(); + tmp = URCU_TLS(rcu_reader).ctr; + urcu_assert((tmp & RCU_GP_CTR_NEST_MASK) != RCU_GP_CTR_NEST_MASK); + _rcu_read_lock_update(tmp); +} + +/* + * This is a helper function for _rcu_read_unlock(). + * + * The first smp_mb_slave() call ensures that the critical section is + * seen to precede the store to rcu_reader.ctr. + * The second smp_mb_slave() call ensures that we write to rcu_reader.ctr + * before reading the update-side futex. + */ +static inline void _rcu_read_unlock_update_and_wakeup(unsigned long tmp) +{ + if (caa_likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) { + smp_mb_slave(); + _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp - RCU_GP_COUNT); + smp_mb_slave(); + wake_up_gp(); + } else + _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp - RCU_GP_COUNT); } +/* + * Exit an RCU read-side crtical section. Both this function and its + * helper are smaller than 10 lines of code, and are intended to be + * usable by non-LGPL code, as called out in LGPL. + */ static inline void _rcu_read_unlock(void) { unsigned long tmp; - tmp = rcu_reader.ctr; - /* - * Finish using rcu before decrementing the pointer. - * See smp_mb_master(). - */ - if (likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) { - smp_mb_slave(RCU_MB_GROUP); - _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT); - /* write rcu_reader.ctr before read futex */ - smp_mb_slave(RCU_MB_GROUP); - wake_up_gp(); - } else { - _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT); - } + urcu_assert(URCU_TLS(rcu_reader).registered); + tmp = URCU_TLS(rcu_reader).ctr; + urcu_assert(tmp & RCU_GP_CTR_NEST_MASK); + _rcu_read_unlock_update_and_wakeup(tmp); cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ } +/* + * Returns whether within a RCU read-side critical section. + * + * This function is less than 10 lines long. The intent is that this + * function meets the 10-line criterion for LGPL, allowing this function + * to be invoked directly from non-LGPL code. + */ +static inline int _rcu_read_ongoing(void) +{ + return URCU_TLS(rcu_reader).ctr & RCU_GP_CTR_NEST_MASK; +} + #ifdef __cplusplus } #endif