X-Git-Url: https://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu%2Fstatic%2Furcu.h;h=9082af71a92c443dd0efe8be74d879c55dd56ddd;hp=bde1459f24100467a0fd42f90e82e54dc4e4815e;hb=67ecffc0f530a7b5c4dd5111ea7dd3213da8eb91;hpb=ed1b099ef8acd1396dbde7c7150d1a5eb16ce8a6 diff --git a/urcu/static/urcu.h b/urcu/static/urcu.h index bde1459..9082af7 100644 --- a/urcu/static/urcu.h +++ b/urcu/static/urcu.h @@ -41,40 +41,18 @@ #include #include #include +#include +#include #ifdef __cplusplus extern "C" { -#endif +#endif /* Default is RCU_MEMBARRIER */ #if !defined(RCU_MEMBARRIER) && !defined(RCU_MB) && !defined(RCU_SIGNAL) #define RCU_MEMBARRIER #endif -/* - * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB - * otherwise. - */ -#if !defined(__linux__) && defined(RCU_MEMBARRIER) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier(...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif - /* * This code section can only be included in LGPL 2.1 compatible source code. * See below for the function call wrappers which can be used in code meant to @@ -102,71 +80,7 @@ enum rcu_state { RCU_READER_INACTIVE, }; -#ifdef DEBUG_RCU -#define rcu_assert(args...) assert(args) -#else -#define rcu_assert(args...) -#endif - -#ifdef DEBUG_YIELD -#include -#include -#include -#include - -#define RCU_YIELD_READ (1 << 0) -#define RCU_YIELD_WRITE (1 << 1) - -/* - * Updates with RCU_SIGNAL are much slower. Account this in the delay. - */ -#ifdef RCU_SIGNAL -/* maximum sleep delay, in us */ -#define MAX_SLEEP 30000 -#else -#define MAX_SLEEP 50 -#endif - -extern unsigned int rcu_yield_active; -extern DECLARE_URCU_TLS(unsigned int, rcu_rand_yield); - -static inline void rcu_debug_yield_read(void) -{ - if (rcu_yield_active & RCU_YIELD_READ) - if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1) - usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP); -} - -static inline void rcu_debug_yield_write(void) -{ - if (rcu_yield_active & RCU_YIELD_WRITE) - if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1) - usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP); -} - -static inline void rcu_debug_yield_init(void) -{ - URCU_TLS(rcu_rand_yield) = time(NULL) ^ (unsigned long) pthread_self(); -} -#else -static inline void rcu_debug_yield_read(void) -{ -} - -static inline void rcu_debug_yield_write(void) -{ -} - -static inline void rcu_debug_yield_init(void) -{ - -} -#endif - /* - * RCU memory barrier broadcast group. Currently, only broadcast to all process - * threads is supported (group 0). - * * Slave barriers are only guaranteed to be ordered wrt master barriers. * * The pair ordering is detailed as (O: ordered, X: not ordered) : @@ -175,13 +89,10 @@ static inline void rcu_debug_yield_init(void) * master O O */ -#define MB_GROUP_ALL 0 -#define RCU_MB_GROUP MB_GROUP_ALL - #ifdef RCU_MEMBARRIER extern int rcu_has_sys_membarrier; -static inline void smp_mb_slave(int group) +static inline void smp_mb_slave(void) { if (caa_likely(rcu_has_sys_membarrier)) cmm_barrier(); @@ -191,14 +102,14 @@ static inline void smp_mb_slave(int group) #endif #ifdef RCU_MB -static inline void smp_mb_slave(int group) +static inline void smp_mb_slave(void) { cmm_smp_mb(); } #endif #ifdef RCU_SIGNAL -static inline void smp_mb_slave(int group) +static inline void smp_mb_slave(void) { cmm_barrier(); } @@ -213,7 +124,7 @@ static inline void smp_mb_slave(int group) #define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2)) #define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) -struct urcu_gp { +struct rcu_gp { /* * Global grace period counter. * Contains the current RCU_GP_CTR_PHASE. @@ -226,7 +137,7 @@ struct urcu_gp { int32_t futex; } __attribute__((aligned(CAA_CACHE_LINE_SIZE))); -extern struct urcu_gp rcu_gp; +extern struct rcu_gp rcu_gp; struct rcu_reader { /* Data used by both reader and synchronize_rcu() */ @@ -235,6 +146,8 @@ struct rcu_reader { /* Data used for registry */ struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE))); pthread_t tid; + /* Reader registered flag, for internal checks. */ + unsigned int registered:1; }; extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); @@ -246,8 +159,13 @@ static inline void wake_up_gp(void) { if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) { uatomic_set(&rcu_gp.futex, 0); - futex_async(&rcu_gp.futex, FUTEX_WAKE, 1, - NULL, NULL, 0); + /* + * Ignoring return value until we can make this function + * return something (because urcu_die() is not publicly + * exposed). + */ + (void) futex_async(&rcu_gp.futex, FUTEX_WAKE, 1, + NULL, NULL, 0); } } @@ -278,7 +196,7 @@ static inline void _rcu_read_lock_update(unsigned long tmp) { if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp.ctr)); - smp_mb_slave(RCU_MB_GROUP); + smp_mb_slave(); } else _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp + RCU_GP_COUNT); } @@ -297,8 +215,10 @@ static inline void _rcu_read_lock(void) { unsigned long tmp; + urcu_assert(URCU_TLS(rcu_reader).registered); cmm_barrier(); tmp = URCU_TLS(rcu_reader).ctr; + urcu_assert((tmp & RCU_GP_CTR_NEST_MASK) != RCU_GP_CTR_NEST_MASK); _rcu_read_lock_update(tmp); } @@ -313,12 +233,12 @@ static inline void _rcu_read_lock(void) static inline void _rcu_read_unlock_update_and_wakeup(unsigned long tmp) { if (caa_likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) { - smp_mb_slave(RCU_MB_GROUP); - _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, URCU_TLS(rcu_reader).ctr - RCU_GP_COUNT); - smp_mb_slave(RCU_MB_GROUP); + smp_mb_slave(); + _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp - RCU_GP_COUNT); + smp_mb_slave(); wake_up_gp(); } else - _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, URCU_TLS(rcu_reader).ctr - RCU_GP_COUNT); + _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp - RCU_GP_COUNT); } /* @@ -330,7 +250,9 @@ static inline void _rcu_read_unlock(void) { unsigned long tmp; + urcu_assert(URCU_TLS(rcu_reader).registered); tmp = URCU_TLS(rcu_reader).ctr; + urcu_assert(tmp & RCU_GP_CTR_NEST_MASK); _rcu_read_unlock_update_and_wakeup(tmp); cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ }