X-Git-Url: https://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu.c;h=a5568bdbd075ed1edc4539dec460c7cce40ee590;hp=5ffeb79553b6374f0fb1a607b3862ed78dda63e2;hb=1a186a881c195c0f240c0baa813056b0cc1eab91;hpb=64f469e628fe45059f091f2808188fb52fa109c9 diff --git a/urcu.c b/urcu.c index 5ffeb79..a5568bd 100644 --- a/urcu.c +++ b/urcu.c @@ -37,6 +37,7 @@ #include #include +#include "urcu/arch.h" #include "urcu/wfcqueue.h" #include "urcu/map/urcu.h" #include "urcu/static/urcu.h" @@ -63,16 +64,9 @@ */ #define RCU_QS_ACTIVE_ATTEMPTS 100 -/* - * RCU_MEMBARRIER is only possibly available on Linux. - */ -#if defined(RCU_MEMBARRIER) && defined(__linux__) -#include -#endif - -/* If the headers do not support SYS_membarrier, fall back on RCU_MB */ -#ifdef SYS_membarrier -# define membarrier(...) syscall(SYS_membarrier, __VA_ARGS__) +/* If the headers do not support membarrier system call, fall back on RCU_MB */ +#ifdef __NR_membarrier +# define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__) #else # define membarrier(...) -ENOSYS #endif @@ -165,7 +159,7 @@ static void mutex_unlock(pthread_mutex_t *mutex) } #ifdef RCU_MEMBARRIER -static void smp_mb_master(int group) +static void smp_mb_master(void) { if (caa_likely(rcu_has_sys_membarrier)) (void) membarrier(MEMBARRIER_CMD_SHARED, 0); @@ -175,7 +169,7 @@ static void smp_mb_master(int group) #endif #ifdef RCU_MB -static void smp_mb_master(int group) +static void smp_mb_master(void) { cmm_smp_mb(); } @@ -224,7 +218,7 @@ static void force_mb_all_readers(void) cmm_smp_mb(); /* read ->need_mb before ending the barrier */ } -static void smp_mb_master(int group) +static void smp_mb_master(void) { force_mb_all_readers(); } @@ -232,19 +226,27 @@ static void smp_mb_master(int group) /* * synchronize_rcu() waiting. Single thread. + * Always called with rcu_registry lock held. Releases this lock and + * grabs it again. Holds the lock when it returns. */ static void wait_gp(void) { - /* Read reader_gp before read futex */ - smp_mb_master(RCU_MB_GROUP); + /* + * Read reader_gp before read futex. smp_mb_master() needs to + * be called with the rcu registry lock held in RCU_SIGNAL + * flavor. + */ + smp_mb_master(); + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); if (uatomic_read(&rcu_gp.futex) != -1) - return; + goto end; while (futex_async(&rcu_gp.futex, FUTEX_WAIT, -1, NULL, NULL, 0)) { switch (errno) { case EWOULDBLOCK: /* Value already changed. */ - return; + goto end; case EINTR: /* Retry if interrupted by signal. */ break; /* Get out of switch. */ @@ -253,6 +255,11 @@ static void wait_gp(void) urcu_die(errno); } } +end: + /* + * Re-lock the registry lock before the next loop. + */ + mutex_lock(&rcu_registry_lock); } /* @@ -280,7 +287,7 @@ static void wait_for_readers(struct cds_list_head *input_readers, if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { uatomic_dec(&rcu_gp.futex); /* Write futex before read reader_gp */ - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); } cds_list_for_each_entry_safe(index, tmp, input_readers, node) { @@ -310,19 +317,24 @@ static void wait_for_readers(struct cds_list_head *input_readers, if (cds_list_empty(input_readers)) { if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { /* Read reader_gp before write futex */ - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); uatomic_set(&rcu_gp.futex, 0); } break; } else { - /* Temporarily unlock the registry lock. */ - mutex_unlock(&rcu_registry_lock); - if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { + /* wait_gp unlocks/locks registry lock. */ wait_gp(); - else + } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); caa_cpu_relax(); - /* Re-lock the registry lock before the next loop. */ - mutex_lock(&rcu_registry_lock); + /* + * Re-lock the registry lock before the + * next loop. + */ + mutex_lock(&rcu_registry_lock); + } } #else /* #ifndef HAS_INCOHERENT_CACHES */ /* @@ -333,25 +345,29 @@ static void wait_for_readers(struct cds_list_head *input_readers, if (cds_list_empty(input_readers)) { if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { /* Read reader_gp before write futex */ - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); uatomic_set(&rcu_gp.futex, 0); } break; } else { if (wait_gp_loops == KICK_READER_LOOPS) { - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); wait_gp_loops = 0; } - /* Temporarily unlock the registry lock. */ - mutex_unlock(&rcu_registry_lock); if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { + /* wait_gp unlocks/locks registry lock. */ wait_gp(); wait_gp_loops++; } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); caa_cpu_relax(); + /* + * Re-lock the registry lock before the + * next loop. + */ + mutex_lock(&rcu_registry_lock); } - /* Re-lock the registry lock before the next loop. */ - mutex_lock(&rcu_registry_lock); } #endif /* #else #ifndef HAS_INCOHERENT_CACHES */ } @@ -400,7 +416,7 @@ void synchronize_rcu(void) * because it iterates on reader threads. */ /* Write new ptr before changing the qparity */ - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); /* * Wait for readers to observe original parity or be quiescent. @@ -461,7 +477,7 @@ void synchronize_rcu(void) * being freed. Must be done within rcu_registry_lock because it * iterates on reader threads. */ - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); out: mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); @@ -500,6 +516,8 @@ void rcu_register_thread(void) assert(!(URCU_TLS(rcu_reader).ctr & RCU_GP_CTR_NEST_MASK)); mutex_lock(&rcu_registry_lock); + assert(!URCU_TLS(rcu_reader).registered); + URCU_TLS(rcu_reader).registered = 1; rcu_init(); /* In case gcc does not support constructor attribute */ cds_list_add(&URCU_TLS(rcu_reader).node, ®istry); mutex_unlock(&rcu_registry_lock); @@ -508,6 +526,8 @@ void rcu_register_thread(void) void rcu_unregister_thread(void) { mutex_lock(&rcu_registry_lock); + assert(URCU_TLS(rcu_reader).registered); + URCU_TLS(rcu_reader).registered = 0; cds_list_del(&URCU_TLS(rcu_reader).node); mutex_unlock(&rcu_registry_lock); }