X-Git-Url: https://git.liburcu.org/?a=blobdiff_plain;f=urcu-bp.c;h=ac8c6806e1008e5314f61e013b31e6b454e3fb69;hb=refs%2Fheads%2Fstable-0.8;hp=7c573d6ec1f7624328018dfc19313a5d6e964a3f;hpb=e198fb6a2ebc22ceac8b10d953103b59452f24d4;p=urcu.git diff --git a/urcu-bp.c b/urcu-bp.c index 7c573d6..ac8c680 100644 --- a/urcu-bp.c +++ b/urcu-bp.c @@ -97,9 +97,23 @@ int rcu_bp_refcount; static void __attribute__((constructor)) rcu_bp_init(void); static -void __attribute__((destructor)) rcu_bp_exit(void); +void __attribute__((destructor)) _rcu_bp_exit(void); +/* + * rcu_gp_lock ensures mutual exclusion between threads calling + * synchronize_rcu(). + */ static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER; +/* + * rcu_registry_lock ensures mutual exclusion between threads + * registering and unregistering themselves to/from the registry, and + * with threads reading that registry from synchronize_rcu(). However, + * this lock is not held all the way through the completion of awaiting + * for the grace period. It is sporadically released between iterations + * on the registry. + * rcu_registry_lock may nest inside rcu_gp_lock. + */ +static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER; static int initialized; @@ -165,6 +179,10 @@ static void mutex_unlock(pthread_mutex_t *mutex) urcu_die(ret); } +/* + * Always called with rcu_registry lock held. Releases this lock between + * iterations and grabs it again. Holds the lock when it returns. + */ static void wait_for_readers(struct cds_list_head *input_readers, struct cds_list_head *cur_snap_readers, struct cds_list_head *qsreaders) @@ -207,10 +225,14 @@ static void wait_for_readers(struct cds_list_head *input_readers, if (cds_list_empty(input_readers)) { break; } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS); else caa_cpu_relax(); + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } } } @@ -229,6 +251,8 @@ void synchronize_rcu(void) mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); + if (cds_list_empty(®istry)) goto out; @@ -239,6 +263,8 @@ void synchronize_rcu(void) /* * Wait for readers to observe original parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(®istry, &cur_snap_readers, &qsreaders); @@ -268,6 +294,8 @@ void synchronize_rcu(void) /* * Wait for readers to observe new parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(&cur_snap_readers, NULL, &qsreaders); @@ -282,6 +310,7 @@ void synchronize_rcu(void) */ cmm_smp_mb(); out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); @@ -326,7 +355,8 @@ void expand_arena(struct registry_arena *arena) sizeof(struct registry_chunk) + sizeof(struct rcu_reader)); new_chunk_len = ARENA_INIT_ALLOC; - new_chunk = mmap(NULL, new_chunk_len, + new_chunk = (struct registry_chunk *) mmap(NULL, + new_chunk_len, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); @@ -360,7 +390,8 @@ void expand_arena(struct registry_arena *arena) } /* Remap did not succeed, we need to add a new chunk. */ - new_chunk = mmap(NULL, new_chunk_len, + new_chunk = (struct registry_chunk *) mmap(NULL, + new_chunk_len, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); @@ -490,9 +521,9 @@ void rcu_bp_register(void) */ rcu_bp_init(); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); add_thread(); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); end: ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); if (ret) @@ -513,13 +544,13 @@ void rcu_bp_unregister(struct rcu_reader *rcu_reader_reg) if (ret) abort(); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); remove_thread(rcu_reader_reg); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); if (ret) abort(); - rcu_bp_exit(); + _rcu_bp_exit(); } /* @@ -549,7 +580,7 @@ void rcu_bp_init(void) } static -void rcu_bp_exit(void) +void _rcu_bp_exit(void) { mutex_lock(&init_lock); if (!--rcu_bp_refcount) { @@ -558,9 +589,10 @@ void rcu_bp_exit(void) cds_list_for_each_entry_safe(chunk, tmp, ®istry_arena.chunk_list, node) { - munmap(chunk, chunk->data_len + munmap((void *) chunk, chunk->data_len + sizeof(struct registry_chunk)); } + CDS_INIT_LIST_HEAD(®istry_arena.chunk_list); ret = pthread_key_delete(urcu_bp_key); if (ret) abort(); @@ -569,9 +601,19 @@ void rcu_bp_exit(void) } /* - * Holding the rcu_gp_lock across fork will make sure we fork() don't race with - * a concurrent thread executing with this same lock held. This ensures that the - * registry is in a coherent state in the child. + * Keep ABI compability within stable versions. This has never been + * exposed through a header, but needs to stay in the .so until the + * soname is bumped. + */ +void rcu_bp_exit(void) +{ +} + +/* + * Holding the rcu_gp_lock and rcu_registry_lock across fork will make + * sure we fork() don't race with a concurrent thread executing with + * any of those locks held. This ensures that the registry and data + * protected by rcu_gp_lock are in a coherent state in the child. */ void rcu_bp_before_fork(void) { @@ -583,6 +625,7 @@ void rcu_bp_before_fork(void) ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); assert(!ret); mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); saved_fork_signal_mask = oldmask; } @@ -592,6 +635,7 @@ void rcu_bp_after_fork_parent(void) int ret; oldmask = saved_fork_signal_mask; + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); @@ -599,7 +643,7 @@ void rcu_bp_after_fork_parent(void) /* * Prune all entries from registry except our own thread. Fits the Linux - * fork behavior. Called with rcu_gp_lock held. + * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held. */ static void urcu_bp_prune_registry(void) @@ -627,6 +671,7 @@ void rcu_bp_after_fork_child(void) urcu_bp_prune_registry(); oldmask = saved_fork_signal_mask; + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret);