X-Git-Url: http://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu-bp.c;h=cbdebf407a892c2d201d42d948025723702414e7;hp=1856d3fc76fa1ec31a644ea9fa326a4ab48182f8;hb=7be759d1adf95b168f09e513232ea26de15e1eaf;hpb=9340c38dbff1b407f35008f7f585a238fbd4de1c diff --git a/urcu-bp.c b/urcu-bp.c index 1856d3f..cbdebf4 100644 --- a/urcu-bp.c +++ b/urcu-bp.c @@ -36,6 +36,7 @@ #include #include +#include "urcu/arch.h" #include "urcu/wfcqueue.h" #include "urcu/map/urcu-bp.h" #include "urcu/static/urcu-bp.h" @@ -94,23 +95,46 @@ void *mremap_wrapper(void *old_address, size_t old_size, static int rcu_bp_refcount; +/* If the headers do not support membarrier system call, fall back smp_mb. */ +#ifdef __NR_membarrier +# define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__) +#else +# define membarrier(...) -ENOSYS +#endif + +enum membarrier_cmd { + MEMBARRIER_CMD_QUERY = 0, + MEMBARRIER_CMD_SHARED = (1 << 0), +}; + static void __attribute__((constructor)) rcu_bp_init(void); static void __attribute__((destructor)) rcu_bp_exit(void); +int urcu_bp_has_sys_membarrier; + +/* + * rcu_gp_lock ensures mutual exclusion between threads calling + * synchronize_rcu(). + */ static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER; +/* + * rcu_registry_lock ensures mutual exclusion between threads + * registering and unregistering themselves to/from the registry, and + * with threads reading that registry from synchronize_rcu(). However, + * this lock is not held all the way through the completion of awaiting + * for the grace period. It is sporadically released between iterations + * on the registry. + * rcu_registry_lock may nest inside rcu_gp_lock. + */ +static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER; static int initialized; static pthread_key_t urcu_bp_key; -#ifdef DEBUG_YIELD -unsigned int rcu_yield_active; -DEFINE_URCU_TLS(unsigned int, rcu_rand_yield); -#endif - struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT }; /* @@ -165,6 +189,18 @@ static void mutex_unlock(pthread_mutex_t *mutex) urcu_die(ret); } +static void smp_mb_master(void) +{ + if (caa_likely(urcu_bp_has_sys_membarrier)) + (void) membarrier(MEMBARRIER_CMD_SHARED, 0); + else + cmm_smp_mb(); +} + +/* + * Always called with rcu_registry lock held. Releases this lock between + * iterations and grabs it again. Holds the lock when it returns. + */ static void wait_for_readers(struct cds_list_head *input_readers, struct cds_list_head *cur_snap_readers, struct cds_list_head *qsreaders) @@ -207,10 +243,14 @@ static void wait_for_readers(struct cds_list_head *input_readers, if (cds_list_empty(input_readers)) { break; } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS); else caa_cpu_relax(); + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } } } @@ -229,16 +269,20 @@ void synchronize_rcu(void) mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); + if (cds_list_empty(®istry)) goto out; /* All threads should read qparity before accessing data structure * where new ptr points to. */ /* Write new ptr before changing the qparity */ - cmm_smp_mb(); + smp_mb_master(); /* * Wait for readers to observe original parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(®istry, &cur_snap_readers, &qsreaders); @@ -268,6 +312,8 @@ void synchronize_rcu(void) /* * Wait for readers to observe new parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(&cur_snap_readers, NULL, &qsreaders); @@ -280,8 +326,9 @@ void synchronize_rcu(void) * Finish waiting for reader threads before letting the old ptr being * freed. */ - cmm_smp_mb(); + smp_mb_master(); out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); @@ -332,7 +379,7 @@ void expand_arena(struct registry_arena *arena) -1, 0); if (new_chunk == MAP_FAILED) abort(); - bzero(new_chunk, new_chunk_len); + memset(new_chunk, 0, new_chunk_len); new_chunk->data_len = new_chunk_len - sizeof(struct registry_chunk); cds_list_add_tail(&new_chunk->node, &arena->chunk_list); @@ -352,7 +399,7 @@ void expand_arena(struct registry_arena *arena) if (new_chunk != MAP_FAILED) { /* Should not have moved. */ assert(new_chunk == last_chunk); - bzero((char *) last_chunk + old_chunk_len, + memset((char *) last_chunk + old_chunk_len, 0, new_chunk_len - old_chunk_len); last_chunk->data_len = new_chunk_len - sizeof(struct registry_chunk); @@ -366,7 +413,7 @@ void expand_arena(struct registry_arena *arena) -1, 0); if (new_chunk == MAP_FAILED) abort(); - bzero(new_chunk, new_chunk_len); + memset(new_chunk, 0, new_chunk_len); new_chunk->data_len = new_chunk_len - sizeof(struct registry_chunk); cds_list_add_tail(&new_chunk->node, &arena->chunk_list); @@ -490,9 +537,9 @@ void rcu_bp_register(void) */ rcu_bp_init(); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); add_thread(); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); end: ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); if (ret) @@ -513,9 +560,9 @@ void rcu_bp_unregister(struct rcu_reader *rcu_reader_reg) if (ret) abort(); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); remove_thread(rcu_reader_reg); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); if (ret) abort(); @@ -543,6 +590,10 @@ void rcu_bp_init(void) urcu_bp_thread_exit_notifier); if (ret) abort(); + ret = membarrier(MEMBARRIER_CMD_QUERY, 0); + if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)) { + urcu_bp_has_sys_membarrier = 1; + } initialized = 1; } mutex_unlock(&init_lock); @@ -561,6 +612,7 @@ void rcu_bp_exit(void) munmap(chunk, chunk->data_len + sizeof(struct registry_chunk)); } + CDS_INIT_LIST_HEAD(®istry_arena.chunk_list); ret = pthread_key_delete(urcu_bp_key); if (ret) abort(); @@ -569,9 +621,10 @@ void rcu_bp_exit(void) } /* - * Holding the rcu_gp_lock across fork will make sure we fork() don't race with - * a concurrent thread executing with this same lock held. This ensures that the - * registry is in a coherent state in the child. + * Holding the rcu_gp_lock and rcu_registry_lock across fork will make + * sure we fork() don't race with a concurrent thread executing with + * any of those locks held. This ensures that the registry and data + * protected by rcu_gp_lock are in a coherent state in the child. */ void rcu_bp_before_fork(void) { @@ -583,6 +636,7 @@ void rcu_bp_before_fork(void) ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); assert(!ret); mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); saved_fork_signal_mask = oldmask; } @@ -592,6 +646,7 @@ void rcu_bp_after_fork_parent(void) int ret; oldmask = saved_fork_signal_mask; + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); @@ -599,7 +654,7 @@ void rcu_bp_after_fork_parent(void) /* * Prune all entries from registry except our own thread. Fits the Linux - * fork behavior. Called with rcu_gp_lock held. + * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held. */ static void urcu_bp_prune_registry(void) @@ -627,6 +682,7 @@ void rcu_bp_after_fork_child(void) urcu_bp_prune_registry(); oldmask = saved_fork_signal_mask; + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret);