From 66bc4dcd6d823e527395bac6755c17718c3f8e71 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 23 Apr 2015 14:00:23 -0400 Subject: [PATCH] Fix: deadlock when thread join is issued in read-side C.S. The transitive dependency between: RCU read-side C.S. -> synchronize_rcu -> rcu_gp_lock -> rcu_register_thread and the dependency: pthread_join -> awaiting for thread completion Can block a thread on join, and thus have the side-effect of deadlocking a thread doing a pthread_join while within a RCU read-side critical section. This join would be awaiting for completion of register_thread or rcu_unregister_thread, which may never complete because the rcu_gp_lock is held by synchronize_rcu executed from another thread. One solution to fix this is to add a new lock, rcu_registry_lock. This lock now protects the thread registry. It is released between iterations on the registry by synchronize_rcu, thus allowing thread registration/unregistration to complete even though synchronize_rcu is awaiting for RCU read-side critical sections to complete. Signed-off-by: Mathieu Desnoyers Reviewed-by: Paul E. McKenney CC: Eugene Ivanov CC: Lai Jiangshan CC: Stephen Hemminger --- urcu-bp.c | 48 +++++++++++++++++++++++++++++++++------- urcu-qsbr.c | 42 +++++++++++++++++++++++++++++++---- urcu.c | 63 +++++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 127 insertions(+), 26 deletions(-) diff --git a/urcu-bp.c b/urcu-bp.c index 280dd0d..3e5be5f 100644 --- a/urcu-bp.c +++ b/urcu-bp.c @@ -99,7 +99,21 @@ void __attribute__((constructor)) rcu_bp_init(void); static void __attribute__((destructor)) _rcu_bp_exit(void); +/* + * rcu_gp_lock ensures mutual exclusion between threads calling + * synchronize_rcu(). + */ static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER; +/* + * rcu_registry_lock ensures mutual exclusion between threads + * registering and unregistering themselves to/from the registry, and + * with threads reading that registry from synchronize_rcu(). However, + * this lock is not held all the way through the completion of awaiting + * for the grace period. It is sporadically released between iterations + * on the registry. + * rcu_registry_lock may nest inside rcu_gp_lock. + */ +static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER; static int initialized; @@ -171,6 +185,10 @@ static void mutex_unlock(pthread_mutex_t *mutex) urcu_die(ret); } +/* + * Always called with rcu_registry lock held. Releases this lock between + * iterations and grabs it again. Holds the lock when it returns. + */ void update_counter_and_wait(void) { CDS_LIST_HEAD(qsreaders); @@ -209,10 +227,14 @@ void update_counter_and_wait(void) if (cds_list_empty(®istry)) { break; } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS); else caa_cpu_relax(); + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } } /* put back the reader list in the registry */ @@ -230,6 +252,7 @@ void synchronize_rcu(void) assert(!ret); mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); if (cds_list_empty(®istry)) goto out; @@ -241,6 +264,8 @@ void synchronize_rcu(void) /* * Wait for previous parity to be empty of readers. + * update_counter_and_wait() can release and grab again + * rcu_registry_lock interally. */ update_counter_and_wait(); /* 0 -> 1, wait readers in parity 0 */ @@ -253,6 +278,8 @@ void synchronize_rcu(void) /* * Wait for previous parity to be empty of readers. + * update_counter_and_wait() can release and grab again + * rcu_registry_lock interally. */ update_counter_and_wait(); /* 1 -> 0, wait readers in parity 1 */ @@ -262,6 +289,7 @@ void synchronize_rcu(void) */ cmm_smp_mb(); out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); @@ -465,9 +493,9 @@ void rcu_bp_register(void) */ rcu_bp_init(); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); add_thread(); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); end: ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); if (ret) @@ -488,9 +516,9 @@ void rcu_bp_unregister(struct rcu_reader *rcu_reader_reg) if (ret) abort(); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); remove_thread(rcu_reader_reg); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); if (ret) abort(); @@ -553,9 +581,10 @@ void rcu_bp_exit(void) } /* - * Holding the rcu_gp_lock across fork will make sure we fork() don't race with - * a concurrent thread executing with this same lock held. This ensures that the - * registry is in a coherent state in the child. + * Holding the rcu_gp_lock and rcu_registry_lock across fork will make + * sure we fork() don't race with a concurrent thread executing with + * any of those locks held. This ensures that the registry and data + * protected by rcu_gp_lock are in a coherent state in the child. */ void rcu_bp_before_fork(void) { @@ -567,6 +596,7 @@ void rcu_bp_before_fork(void) ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); assert(!ret); mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); saved_fork_signal_mask = oldmask; } @@ -576,6 +606,7 @@ void rcu_bp_after_fork_parent(void) int ret; oldmask = saved_fork_signal_mask; + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); @@ -583,7 +614,7 @@ void rcu_bp_after_fork_parent(void) /* * Prune all entries from registry except our own thread. Fits the Linux - * fork behavior. Called with rcu_gp_lock held. + * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held. */ static void urcu_bp_prune_registry(void) @@ -611,6 +642,7 @@ void rcu_bp_after_fork_child(void) urcu_bp_prune_registry(); oldmask = saved_fork_signal_mask; + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); diff --git a/urcu-qsbr.c b/urcu-qsbr.c index a2cabb4..1a94efa 100644 --- a/urcu-qsbr.c +++ b/urcu-qsbr.c @@ -51,7 +51,21 @@ void __attribute__((destructor)) rcu_exit(void); +/* + * rcu_gp_lock ensures mutual exclusion between threads calling + * synchronize_rcu(). + */ static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER; +/* + * rcu_registry_lock ensures mutual exclusion between threads + * registering and unregistering themselves to/from the registry, and + * with threads reading that registry from synchronize_rcu(). However, + * this lock is not held all the way through the completion of awaiting + * for the grace period. It is sporadically released between iterations + * on the registry. + * rcu_registry_lock may nest inside rcu_gp_lock. + */ +static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER; int32_t gp_futex; @@ -116,6 +130,10 @@ static void wait_gp(void) NULL, NULL, 0); } +/* + * Always called with rcu_registry lock held. Releases this lock between + * iterations and grabs it again. Holds the lock when it returns. + */ static void update_counter_and_wait(void) { CDS_LIST_HEAD(qsreaders); @@ -178,6 +196,8 @@ static void update_counter_and_wait(void) } break; } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { wait_gp(); } else { @@ -187,6 +207,8 @@ static void update_counter_and_wait(void) cmm_smp_mb(); #endif /* #else #ifndef HAS_INCOHERENT_CACHES */ } + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } } /* put back the reader list in the registry */ @@ -219,12 +241,15 @@ void synchronize_rcu(void) cmm_smp_mb(); mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); if (cds_list_empty(®istry)) goto out; /* * Wait for previous parity to be empty of readers. + * update_counter_and_wait() can release and grab again + * rcu_registry_lock interally. */ update_counter_and_wait(); /* 0 -> 1, wait readers in parity 0 */ @@ -247,9 +272,12 @@ void synchronize_rcu(void) /* * Wait for previous parity to be empty of readers. + * update_counter_and_wait() can release and grab again + * rcu_registry_lock interally. */ update_counter_and_wait(); /* 1 -> 0, wait readers in parity 1 */ out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); /* @@ -279,10 +307,16 @@ void synchronize_rcu(void) cmm_smp_mb(); mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); if (cds_list_empty(®istry)) goto out; + /* + * update_counter_and_wait() can release and grab again + * rcu_registry_lock interally. + */ update_counter_and_wait(); out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); if (was_online) @@ -326,9 +360,9 @@ void rcu_register_thread(void) URCU_TLS(rcu_reader).tid = pthread_self(); assert(URCU_TLS(rcu_reader).ctr == 0); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); cds_list_add(&URCU_TLS(rcu_reader).node, ®istry); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); _rcu_thread_online(); } @@ -339,9 +373,9 @@ void rcu_unregister_thread(void) * with a waiting writer. */ _rcu_thread_offline(); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); cds_list_del(&URCU_TLS(rcu_reader).node); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); } void rcu_exit(void) diff --git a/urcu.c b/urcu.c index 8420ee4..5cb470b 100644 --- a/urcu.c +++ b/urcu.c @@ -81,7 +81,21 @@ void __attribute__((constructor)) rcu_init(void); void __attribute__((destructor)) rcu_exit(void); #endif +/* + * rcu_gp_lock ensures mutual exclusion between threads calling + * synchronize_rcu(). + */ static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER; +/* + * rcu_registry_lock ensures mutual exclusion between threads + * registering and unregistering themselves to/from the registry, and + * with threads reading that registry from synchronize_rcu(). However, + * this lock is not held all the way through the completion of awaiting + * for the grace period. It is sporadically released between iterations + * on the registry. + * rcu_registry_lock may nest inside rcu_gp_lock. + */ +static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER; int32_t gp_futex; @@ -92,7 +106,6 @@ int32_t gp_futex; * Written to only by writer with mutex taken. Read by both writer and readers. */ unsigned long rcu_gp_ctr = RCU_GP_COUNT; - /* * Written to only by each individual reader. Read by both the reader and the * writers. @@ -215,6 +228,10 @@ static void wait_gp(void) NULL, NULL, 0); } +/* + * Always called with rcu_registry lock held. Releases this lock between + * iterations and grabs it again. Holds the lock when it returns. + */ void update_counter_and_wait(void) { CDS_LIST_HEAD(qsreaders); @@ -269,10 +286,14 @@ void update_counter_and_wait(void) } break; } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) wait_gp(); else caa_cpu_relax(); + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } #else /* #ifndef HAS_INCOHERENT_CACHES */ /* @@ -292,12 +313,16 @@ void update_counter_and_wait(void) smp_mb_master(RCU_MB_GROUP); wait_gp_loops = 0; } + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { wait_gp(); wait_gp_loops++; } else { caa_cpu_relax(); } + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } #endif /* #else #ifndef HAS_INCOHERENT_CACHES */ } @@ -308,18 +333,23 @@ void update_counter_and_wait(void) void synchronize_rcu(void) { mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); if (cds_list_empty(®istry)) goto out; - /* All threads should read qparity before accessing data structure - * where new ptr points to. Must be done within rcu_gp_lock because it - * iterates on reader threads.*/ + /* + * All threads should read qparity before accessing data structure + * where new ptr points to. Must be done within rcu_registry_lock + * because it iterates on reader threads. + */ /* Write new ptr before changing the qparity */ smp_mb_master(RCU_MB_GROUP); /* * Wait for previous parity to be empty of readers. + * update_counter_and_wait() can release and grab again + * rcu_registry_lock interally. */ update_counter_and_wait(); /* 0 -> 1, wait readers in parity 0 */ @@ -341,14 +371,19 @@ void synchronize_rcu(void) /* * Wait for previous parity to be empty of readers. + * update_counter_and_wait() can release and grab again + * rcu_registry_lock interally. */ update_counter_and_wait(); /* 1 -> 0, wait readers in parity 1 */ - /* Finish waiting for reader threads before letting the old ptr being - * freed. Must be done within rcu_gp_lock because it iterates on reader - * threads. */ + /* + * Finish waiting for reader threads before letting the old ptr + * being freed. Must be done within rcu_registry_lock because it + * iterates on reader threads. + */ smp_mb_master(RCU_MB_GROUP); out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); } @@ -372,17 +407,17 @@ void rcu_register_thread(void) assert(URCU_TLS(rcu_reader).need_mb == 0); assert(!(URCU_TLS(rcu_reader).ctr & RCU_GP_CTR_NEST_MASK)); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); rcu_init(); /* In case gcc does not support constructor attribute */ cds_list_add(&URCU_TLS(rcu_reader).node, ®istry); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); } void rcu_unregister_thread(void) { - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); cds_list_del(&URCU_TLS(rcu_reader).node); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); } #ifdef RCU_MEMBARRIER @@ -413,9 +448,9 @@ static void sigrcu_handler(int signo, siginfo_t *siginfo, void *context) * rcu_init constructor. Called when the library is linked, but also when * reader threads are calling rcu_register_thread(). * Should only be called by a single thread at a given time. This is ensured by - * holing the rcu_gp_lock from rcu_register_thread() or by running at library - * load time, which should not be executed by multiple threads nor concurrently - * with rcu_register_thread() anyway. + * holing the rcu_registry_lock from rcu_register_thread() or by running + * at library load time, which should not be executed by multiple + * threads nor concurrently with rcu_register_thread() anyway. */ void rcu_init(void) { -- 2.34.1