X-Git-Url: http://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu%2Fstatic%2Furcu.h;h=33545b71f68c1be9dc25ef78a59a9a052d16aca5;hp=f32f300e599d95b922cbf043cc3a839e234190d5;hb=e1c0b55cbcc602bdff8b9778d911c35515f48cc3;hpb=1de4df4b770e5e90440008becc5e14a15c75c6e0 diff --git a/urcu/static/urcu.h b/urcu/static/urcu.h index f32f300..33545b7 100644 --- a/urcu/static/urcu.h +++ b/urcu/static/urcu.h @@ -41,6 +41,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -51,30 +52,6 @@ extern "C" { #define RCU_MEMBARRIER #endif -/* - * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB - * otherwise. - */ -#if !defined(__linux__) && defined(RCU_MEMBARRIER) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier(...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif - /* * This code section can only be included in LGPL 2.1 compatible source code. * See below for the function call wrappers which can be used in code meant to @@ -96,6 +73,12 @@ extern "C" { #define SIGRCU SIGUSR1 #endif +enum rcu_state { + RCU_READER_ACTIVE_CURRENT, + RCU_READER_ACTIVE_OLD, + RCU_READER_INACTIVE, +}; + #ifdef DEBUG_RCU #define rcu_assert(args...) assert(args) #else @@ -207,12 +190,20 @@ static inline void smp_mb_slave(int group) #define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2)) #define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) -/* - * Global quiescent period counter with low-order bits unused. - * Using a int rather than a char to eliminate false register dependencies - * causing stalls on some architectures. - */ -extern unsigned long rcu_gp_ctr; +struct rcu_gp { + /* + * Global grace period counter. + * Contains the current RCU_GP_CTR_PHASE. + * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. + * Written to only by writer with mutex taken. + * Read by both writer and readers. + */ + unsigned long ctr; + + int32_t futex; +} __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + +extern struct rcu_gp rcu_gp; struct rcu_reader { /* Data used by both reader and synchronize_rcu() */ @@ -225,21 +216,19 @@ struct rcu_reader { extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); -extern int32_t rcu_gp_futex; - /* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. */ static inline void wake_up_gp(void) { - if (caa_unlikely(uatomic_read(&rcu_gp_futex) == -1)) { - uatomic_set(&rcu_gp_futex, 0); - futex_async(&rcu_gp_futex, FUTEX_WAKE, 1, + if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) { + uatomic_set(&rcu_gp.futex, 0); + futex_async(&rcu_gp.futex, FUTEX_WAKE, 1, NULL, NULL, 0); } } -static inline int rcu_gp_ongoing(unsigned long *ctr) +static inline enum rcu_state rcu_reader_state(unsigned long *ctr) { unsigned long v; @@ -248,12 +237,15 @@ static inline int rcu_gp_ongoing(unsigned long *ctr) * to insure consistency. */ v = CMM_LOAD_SHARED(*ctr); - return (v & RCU_GP_CTR_NEST_MASK) && - ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE); + if (!(v & RCU_GP_CTR_NEST_MASK)) + return RCU_READER_INACTIVE; + if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE)) + return RCU_READER_ACTIVE_CURRENT; + return RCU_READER_ACTIVE_OLD; } /* - * Helper for _rcu_read_lock(). The format of rcu_gp_ctr (as well as + * Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as * the per-thread rcu_reader.ctr) has the upper bits containing a count of * _rcu_read_lock() nesting, and a lower-order bit that contains either zero * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in @@ -262,7 +254,7 @@ static inline int rcu_gp_ongoing(unsigned long *ctr) static inline void _rcu_read_lock_update(unsigned long tmp) { if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { - _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); + _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp.ctr)); smp_mb_slave(RCU_MB_GROUP); } else _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp + RCU_GP_COUNT); @@ -320,6 +312,18 @@ static inline void _rcu_read_unlock(void) cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ } +/* + * Returns whether within a RCU read-side critical section. + * + * This function is less than 10 lines long. The intent is that this + * function meets the 10-line criterion for LGPL, allowing this function + * to be invoked directly from non-LGPL code. + */ +static inline int _rcu_read_ongoing(void) +{ + return URCU_TLS(rcu_reader).ctr & RCU_GP_CTR_NEST_MASK; +} + #ifdef __cplusplus } #endif