X-Git-Url: https://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu%2Fstatic%2Furcu-qsbr.h;h=8d5fd03107373b5cee0d51273624da5c05552bdb;hp=f189e31ad757fdf679dc9d5b941c988b813e9041;hb=67ecffc0f530a7b5c4dd5111ea7dd3213da8eb91;hpb=a2e7bf9ce5de5113c7f59c380b0087e291cd603d diff --git a/urcu/static/urcu-qsbr.h b/urcu/static/urcu-qsbr.h index f189e31..8d5fd03 100644 --- a/urcu/static/urcu-qsbr.h +++ b/urcu/static/urcu-qsbr.h @@ -6,8 +6,8 @@ * * Userspace RCU QSBR header. * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-qsbr.h for linking - * dynamically with the userspace rcu QSBR library. + * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU + * RELEASE. See urcu.h for linking dynamically with the userspace rcu library. * * Copyright (c) 2009 Mathieu Desnoyers * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. @@ -31,21 +31,22 @@ #include #include -#include #include -#include #include +#include #include #include #include #include #include -#include +#include +#include +#include #ifdef __cplusplus extern "C" { -#endif +#endif /* * This code section can only be included in LGPL 2.1 compatible source code. @@ -55,149 +56,185 @@ extern "C" { * This is required to permit relinking with newer versions of the library. */ -/* - * If a reader is really non-cooperative and refuses to commit its - * rcu_reader.ctr count to memory (there is no barrier in the reader - * per-se), kick it after a few loops waiting for it. - */ -#define KICK_READER_LOOPS 10000 - -/* - * Active attempts to check for reader Q.S. before calling futex(). - */ -#define RCU_QS_ACTIVE_ATTEMPTS 100 - -#ifdef DEBUG_RCU -#define rcu_assert(args...) assert(args) -#else -#define rcu_assert(args...) -#endif - -#ifdef DEBUG_YIELD -#include -#include -#include -#include - -#define YIELD_READ (1 << 0) -#define YIELD_WRITE (1 << 1) - -/* maximum sleep delay, in us */ -#define MAX_SLEEP 50 - -extern unsigned int yield_active; -extern unsigned int __thread rand_yield; - -static inline void debug_yield_read(void) -{ - if (yield_active & YIELD_READ) - if (rand_r(&rand_yield) & 0x1) - usleep(rand_r(&rand_yield) % MAX_SLEEP); -} - -static inline void debug_yield_write(void) -{ - if (yield_active & YIELD_WRITE) - if (rand_r(&rand_yield) & 0x1) - usleep(rand_r(&rand_yield) % MAX_SLEEP); -} - -static inline void debug_yield_init(void) -{ - rand_yield = time(NULL) ^ pthread_self(); -} -#else -static inline void debug_yield_read(void) -{ -} - -static inline void debug_yield_write(void) -{ -} - -static inline void debug_yield_init(void) -{ - -} -#endif +enum rcu_state { + RCU_READER_ACTIVE_CURRENT, + RCU_READER_ACTIVE_OLD, + RCU_READER_INACTIVE, +}; #define RCU_GP_ONLINE (1UL << 0) #define RCU_GP_CTR (1UL << 1) -/* - * Global quiescent period counter with low-order bits unused. - * Using a int rather than a char to eliminate false register dependencies - * causing stalls on some architectures. - */ -extern unsigned long rcu_gp_ctr; +struct rcu_gp { + /* + * Global quiescent period counter with low-order bits unused. + * Using a int rather than a char to eliminate false register + * dependencies causing stalls on some architectures. + */ + unsigned long ctr; + + int32_t futex; +} __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + +extern struct rcu_gp rcu_gp; struct rcu_reader { /* Data used by both reader and synchronize_rcu() */ unsigned long ctr; /* Data used for registry */ struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + int waiting; pthread_t tid; + /* Reader registered flag, for internal checks. */ + unsigned int registered:1; }; -extern struct rcu_reader __thread rcu_reader; - -extern int gp_futex; +extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); /* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. */ static inline void wake_up_gp(void) { - if (unlikely(uatomic_read(&gp_futex) == -1)) { - uatomic_set(&gp_futex, 0); - futex_noasync(&gp_futex, FUTEX_WAKE, 1, - NULL, NULL, 0); + if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(rcu_reader).waiting))) { + _CMM_STORE_SHARED(URCU_TLS(rcu_reader).waiting, 0); + cmm_smp_mb(); + if (uatomic_read(&rcu_gp.futex) != -1) + return; + uatomic_set(&rcu_gp.futex, 0); + /* + * Ignoring return value until we can make this function + * return something (because urcu_die() is not publicly + * exposed). + */ + (void) futex_noasync(&rcu_gp.futex, FUTEX_WAKE, 1, + NULL, NULL, 0); } } -static inline int rcu_gp_ongoing(unsigned long *ctr) +static inline enum rcu_state rcu_reader_state(unsigned long *ctr) { unsigned long v; v = CMM_LOAD_SHARED(*ctr); - return v && (v != rcu_gp_ctr); + if (!v) + return RCU_READER_INACTIVE; + if (v == rcu_gp.ctr) + return RCU_READER_ACTIVE_CURRENT; + return RCU_READER_ACTIVE_OLD; } +/* + * Enter an RCU read-side critical section. + * + * This function is less than 10 lines long. The intent is that this + * function meets the 10-line criterion for LGPL, allowing this function + * to be invoked directly from non-LGPL code. + */ static inline void _rcu_read_lock(void) { - rcu_assert(rcu_reader.ctr); + urcu_assert(URCU_TLS(rcu_reader).ctr); } +/* + * Exit an RCU read-side critical section. + * + * This function is less than 10 lines long. The intent is that this + * function meets the 10-line criterion for LGPL, allowing this function + * to be invoked directly from non-LGPL code. + */ static inline void _rcu_read_unlock(void) { + urcu_assert(URCU_TLS(rcu_reader).ctr); } -static inline void _rcu_quiescent_state(void) +/* + * Returns whether within a RCU read-side critical section. + * + * This function is less than 10 lines long. The intent is that this + * function meets the 10-line criterion for LGPL, allowing this function + * to be invoked directly from non-LGPL code. + */ +static inline int _rcu_read_ongoing(void) +{ + return URCU_TLS(rcu_reader).ctr; +} + +/* + * This is a helper function for _rcu_quiescent_state(). + * The first cmm_smp_mb() ensures memory accesses in the prior read-side + * critical sections are not reordered with store to + * URCU_TLS(rcu_reader).ctr, and ensures that mutexes held within an + * offline section that would happen to end with this + * rcu_quiescent_state() call are not reordered with + * store to URCU_TLS(rcu_reader).ctr. + */ +static inline void _rcu_quiescent_state_update_and_wakeup(unsigned long gp_ctr) { cmm_smp_mb(); - _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); - cmm_smp_mb(); /* write rcu_reader.ctr before read futex */ + _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, gp_ctr); + cmm_smp_mb(); /* write URCU_TLS(rcu_reader).ctr before read futex */ wake_up_gp(); cmm_smp_mb(); } +/* + * Inform RCU of a quiescent state. + * + * This function is less than 10 lines long. The intent is that this + * function meets the 10-line criterion for LGPL, allowing this function + * to be invoked directly from non-LGPL code. + * + * We skip the memory barriers and gp store if our local ctr already + * matches the global rcu_gp.ctr value: this is OK because a prior + * _rcu_quiescent_state() or _rcu_thread_online() already updated it + * within our thread, so we have no quiescent state to report. + */ +static inline void _rcu_quiescent_state(void) +{ + unsigned long gp_ctr; + + urcu_assert(URCU_TLS(rcu_reader).registered); + if ((gp_ctr = CMM_LOAD_SHARED(rcu_gp.ctr)) == URCU_TLS(rcu_reader).ctr) + return; + _rcu_quiescent_state_update_and_wakeup(gp_ctr); +} + +/* + * Take a thread offline, prohibiting it from entering further RCU + * read-side critical sections. + * + * This function is less than 10 lines long. The intent is that this + * function meets the 10-line criterion for LGPL, allowing this function + * to be invoked directly from non-LGPL code. + */ static inline void _rcu_thread_offline(void) { + urcu_assert(URCU_TLS(rcu_reader).registered); cmm_smp_mb(); - CMM_STORE_SHARED(rcu_reader.ctr, 0); - cmm_smp_mb(); /* write rcu_reader.ctr before read futex */ + CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, 0); + cmm_smp_mb(); /* write URCU_TLS(rcu_reader).ctr before read futex */ wake_up_gp(); cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ } +/* + * Bring a thread online, allowing it to once again enter RCU + * read-side critical sections. + * + * This function is less than 10 lines long. The intent is that this + * function meets the 10-line criterion for LGPL, allowing this function + * to be invoked directly from non-LGPL code. + */ static inline void _rcu_thread_online(void) { + urcu_assert(URCU_TLS(rcu_reader).registered); cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ - _CMM_STORE_SHARED(rcu_reader.ctr, CMM_LOAD_SHARED(rcu_gp_ctr)); + _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp.ctr)); cmm_smp_mb(); } -#ifdef __cplusplus +#ifdef __cplusplus } #endif