X-Git-Url: https://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu-bp.c;h=20684cc4d0972a61427f8ecae0e5fd7eb5072189;hp=1856d3fc76fa1ec31a644ea9fa326a4ab48182f8;hb=19d0e7ef8f7998fb256017ba2da414d78b3cb91b;hpb=9340c38dbff1b407f35008f7f585a238fbd4de1c diff --git a/urcu-bp.c b/urcu-bp.c index 1856d3f..20684cc 100644 --- a/urcu-bp.c +++ b/urcu-bp.c @@ -36,6 +36,7 @@ #include #include +#include "urcu/arch.h" #include "urcu/wfcqueue.h" #include "urcu/map/urcu-bp.h" #include "urcu/static/urcu-bp.h" @@ -94,23 +95,46 @@ void *mremap_wrapper(void *old_address, size_t old_size, static int rcu_bp_refcount; +/* If the headers do not support membarrier system call, fall back smp_mb. */ +#ifdef __NR_membarrier +# define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__) +#else +# define membarrier(...) -ENOSYS +#endif + +enum membarrier_cmd { + MEMBARRIER_CMD_QUERY = 0, + MEMBARRIER_CMD_SHARED = (1 << 0), +}; + static void __attribute__((constructor)) rcu_bp_init(void); static void __attribute__((destructor)) rcu_bp_exit(void); +int urcu_bp_has_sys_membarrier; + +/* + * rcu_gp_lock ensures mutual exclusion between threads calling + * synchronize_rcu(). + */ static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER; +/* + * rcu_registry_lock ensures mutual exclusion between threads + * registering and unregistering themselves to/from the registry, and + * with threads reading that registry from synchronize_rcu(). However, + * this lock is not held all the way through the completion of awaiting + * for the grace period. It is sporadically released between iterations + * on the registry. + * rcu_registry_lock may nest inside rcu_gp_lock. + */ +static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER; static int initialized; static pthread_key_t urcu_bp_key; -#ifdef DEBUG_YIELD -unsigned int rcu_yield_active; -DEFINE_URCU_TLS(unsigned int, rcu_rand_yield); -#endif - struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT }; /* @@ -165,6 +189,18 @@ static void mutex_unlock(pthread_mutex_t *mutex) urcu_die(ret); } +static void smp_mb_master(void) +{ + if (caa_likely(urcu_bp_has_sys_membarrier)) + (void) membarrier(MEMBARRIER_CMD_SHARED, 0); + else + cmm_smp_mb(); +} + +/* + * Always called with rcu_registry lock held. Releases this lock between + * iterations and grabs it again. Holds the lock when it returns. + */ static void wait_for_readers(struct cds_list_head *input_readers, struct cds_list_head *cur_snap_readers, struct cds_list_head *qsreaders) @@ -207,10 +243,14 @@ static void wait_for_readers(struct cds_list_head *input_readers, if (cds_list_empty(input_readers)) { break; } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS); else caa_cpu_relax(); + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } } } @@ -229,16 +269,20 @@ void synchronize_rcu(void) mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); + if (cds_list_empty(®istry)) goto out; /* All threads should read qparity before accessing data structure * where new ptr points to. */ /* Write new ptr before changing the qparity */ - cmm_smp_mb(); + smp_mb_master(); /* * Wait for readers to observe original parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(®istry, &cur_snap_readers, &qsreaders); @@ -268,6 +312,8 @@ void synchronize_rcu(void) /* * Wait for readers to observe new parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(&cur_snap_readers, NULL, &qsreaders); @@ -280,8 +326,9 @@ void synchronize_rcu(void) * Finish waiting for reader threads before letting the old ptr being * freed. */ - cmm_smp_mb(); + smp_mb_master(); out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); @@ -490,9 +537,9 @@ void rcu_bp_register(void) */ rcu_bp_init(); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); add_thread(); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); end: ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); if (ret) @@ -513,9 +560,9 @@ void rcu_bp_unregister(struct rcu_reader *rcu_reader_reg) if (ret) abort(); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); remove_thread(rcu_reader_reg); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); if (ret) abort(); @@ -543,6 +590,10 @@ void rcu_bp_init(void) urcu_bp_thread_exit_notifier); if (ret) abort(); + ret = membarrier(MEMBARRIER_CMD_QUERY, 0); + if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)) { + urcu_bp_has_sys_membarrier = 1; + } initialized = 1; } mutex_unlock(&init_lock); @@ -569,9 +620,10 @@ void rcu_bp_exit(void) } /* - * Holding the rcu_gp_lock across fork will make sure we fork() don't race with - * a concurrent thread executing with this same lock held. This ensures that the - * registry is in a coherent state in the child. + * Holding the rcu_gp_lock and rcu_registry_lock across fork will make + * sure we fork() don't race with a concurrent thread executing with + * any of those locks held. This ensures that the registry and data + * protected by rcu_gp_lock are in a coherent state in the child. */ void rcu_bp_before_fork(void) { @@ -583,6 +635,7 @@ void rcu_bp_before_fork(void) ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); assert(!ret); mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); saved_fork_signal_mask = oldmask; } @@ -592,6 +645,7 @@ void rcu_bp_after_fork_parent(void) int ret; oldmask = saved_fork_signal_mask; + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); @@ -599,7 +653,7 @@ void rcu_bp_after_fork_parent(void) /* * Prune all entries from registry except our own thread. Fits the Linux - * fork behavior. Called with rcu_gp_lock held. + * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held. */ static void urcu_bp_prune_registry(void) @@ -627,6 +681,7 @@ void rcu_bp_after_fork_child(void) urcu_bp_prune_registry(); oldmask = saved_fork_signal_mask; + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret);