X-Git-Url: https://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=urcu.c;h=a5568bdbd075ed1edc4539dec460c7cce40ee590;hp=1429f6dfbbd3f392b307a9139931bf24eeebd67a;hb=b83b3590caf8ddd0fb1393b0d8378f916c8bf831;hpb=731ccb963c80afd067e20acee2f9bd7cb4875ffb diff --git a/urcu.c b/urcu.c index 1429f6d..a5568bd 100644 --- a/urcu.c +++ b/urcu.c @@ -26,6 +26,7 @@ #define _BSD_SOURCE #define _GNU_SOURCE #define _LGPL_SOURCE +#define _DEFAULT_SOURCE #include #include #include @@ -36,6 +37,7 @@ #include #include +#include "urcu/arch.h" #include "urcu/wfcqueue.h" #include "urcu/map/urcu.h" #include "urcu/static/urcu.h" @@ -62,23 +64,17 @@ */ #define RCU_QS_ACTIVE_ATTEMPTS 100 -/* - * RCU_MEMBARRIER is only possibly available on Linux. - */ -#if defined(RCU_MEMBARRIER) && defined(__linux__) -#include -#endif - -/* If the headers do not support SYS_membarrier, fall back on RCU_MB */ -#ifdef SYS_membarrier -# define membarrier(...) syscall(SYS_membarrier, __VA_ARGS__) +/* If the headers do not support membarrier system call, fall back on RCU_MB */ +#ifdef __NR_membarrier +# define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__) #else # define membarrier(...) -ENOSYS #endif -#define MEMBARRIER_EXPEDITED (1 << 0) -#define MEMBARRIER_DELAYED (1 << 1) -#define MEMBARRIER_QUERY (1 << 16) +enum membarrier_cmd { + MEMBARRIER_CMD_QUERY = 0, + MEMBARRIER_CMD_SHARED = (1 << 0), +}; #ifdef RCU_MEMBARRIER static int init_done; @@ -148,7 +144,7 @@ static void mutex_lock(pthread_mutex_t *mutex) _CMM_STORE_SHARED(URCU_TLS(rcu_reader).need_mb, 0); cmm_smp_mb(); } - poll(NULL,0,10); + (void) poll(NULL, 0, 10); } #endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */ } @@ -163,17 +159,17 @@ static void mutex_unlock(pthread_mutex_t *mutex) } #ifdef RCU_MEMBARRIER -static void smp_mb_master(int group) +static void smp_mb_master(void) { if (caa_likely(rcu_has_sys_membarrier)) - (void) membarrier(MEMBARRIER_EXPEDITED); + (void) membarrier(MEMBARRIER_CMD_SHARED, 0); else cmm_smp_mb(); } #endif #ifdef RCU_MB -static void smp_mb_master(int group) +static void smp_mb_master(void) { cmm_smp_mb(); } @@ -216,13 +212,13 @@ static void force_mb_all_readers(void) cds_list_for_each_entry(index, ®istry, node) { while (CMM_LOAD_SHARED(index->need_mb)) { pthread_kill(index->tid, SIGRCU); - poll(NULL, 0, 1); + (void) poll(NULL, 0, 1); } } cmm_smp_mb(); /* read ->need_mb before ending the barrier */ } -static void smp_mb_master(int group) +static void smp_mb_master(void) { force_mb_all_readers(); } @@ -230,14 +226,40 @@ static void smp_mb_master(int group) /* * synchronize_rcu() waiting. Single thread. + * Always called with rcu_registry lock held. Releases this lock and + * grabs it again. Holds the lock when it returns. */ static void wait_gp(void) { - /* Read reader_gp before read futex */ - smp_mb_master(RCU_MB_GROUP); - if (uatomic_read(&rcu_gp.futex) == -1) - futex_async(&rcu_gp.futex, FUTEX_WAIT, -1, - NULL, NULL, 0); + /* + * Read reader_gp before read futex. smp_mb_master() needs to + * be called with the rcu registry lock held in RCU_SIGNAL + * flavor. + */ + smp_mb_master(); + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); + if (uatomic_read(&rcu_gp.futex) != -1) + goto end; + while (futex_async(&rcu_gp.futex, FUTEX_WAIT, -1, + NULL, NULL, 0)) { + switch (errno) { + case EWOULDBLOCK: + /* Value already changed. */ + goto end; + case EINTR: + /* Retry if interrupted by signal. */ + break; /* Get out of switch. */ + default: + /* Unexpected error. */ + urcu_die(errno); + } + } +end: + /* + * Re-lock the registry lock before the next loop. + */ + mutex_lock(&rcu_registry_lock); } /* @@ -265,7 +287,7 @@ static void wait_for_readers(struct cds_list_head *input_readers, if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { uatomic_dec(&rcu_gp.futex); /* Write futex before read reader_gp */ - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); } cds_list_for_each_entry_safe(index, tmp, input_readers, node) { @@ -295,19 +317,24 @@ static void wait_for_readers(struct cds_list_head *input_readers, if (cds_list_empty(input_readers)) { if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { /* Read reader_gp before write futex */ - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); uatomic_set(&rcu_gp.futex, 0); } break; } else { - /* Temporarily unlock the registry lock. */ - mutex_unlock(&rcu_registry_lock); - if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { + /* wait_gp unlocks/locks registry lock. */ wait_gp(); - else + } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); caa_cpu_relax(); - /* Re-lock the registry lock before the next loop. */ - mutex_lock(&rcu_registry_lock); + /* + * Re-lock the registry lock before the + * next loop. + */ + mutex_lock(&rcu_registry_lock); + } } #else /* #ifndef HAS_INCOHERENT_CACHES */ /* @@ -318,25 +345,29 @@ static void wait_for_readers(struct cds_list_head *input_readers, if (cds_list_empty(input_readers)) { if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { /* Read reader_gp before write futex */ - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); uatomic_set(&rcu_gp.futex, 0); } break; } else { if (wait_gp_loops == KICK_READER_LOOPS) { - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); wait_gp_loops = 0; } - /* Temporarily unlock the registry lock. */ - mutex_unlock(&rcu_registry_lock); if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { + /* wait_gp unlocks/locks registry lock. */ wait_gp(); wait_gp_loops++; } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); caa_cpu_relax(); + /* + * Re-lock the registry lock before the + * next loop. + */ + mutex_lock(&rcu_registry_lock); } - /* Re-lock the registry lock before the next loop. */ - mutex_lock(&rcu_registry_lock); } #endif /* #else #ifndef HAS_INCOHERENT_CACHES */ } @@ -385,7 +416,7 @@ void synchronize_rcu(void) * because it iterates on reader threads. */ /* Write new ptr before changing the qparity */ - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); /* * Wait for readers to observe original parity or be quiescent. @@ -446,7 +477,7 @@ void synchronize_rcu(void) * being freed. Must be done within rcu_registry_lock because it * iterates on reader threads. */ - smp_mb_master(RCU_MB_GROUP); + smp_mb_master(); out: mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); @@ -485,6 +516,8 @@ void rcu_register_thread(void) assert(!(URCU_TLS(rcu_reader).ctr & RCU_GP_CTR_NEST_MASK)); mutex_lock(&rcu_registry_lock); + assert(!URCU_TLS(rcu_reader).registered); + URCU_TLS(rcu_reader).registered = 1; rcu_init(); /* In case gcc does not support constructor attribute */ cds_list_add(&URCU_TLS(rcu_reader).node, ®istry); mutex_unlock(&rcu_registry_lock); @@ -493,6 +526,8 @@ void rcu_register_thread(void) void rcu_unregister_thread(void) { mutex_lock(&rcu_registry_lock); + assert(URCU_TLS(rcu_reader).registered); + URCU_TLS(rcu_reader).registered = 0; cds_list_del(&URCU_TLS(rcu_reader).node); mutex_unlock(&rcu_registry_lock); } @@ -500,11 +535,15 @@ void rcu_unregister_thread(void) #ifdef RCU_MEMBARRIER void rcu_init(void) { + int ret; + if (init_done) return; init_done = 1; - if (!membarrier(MEMBARRIER_EXPEDITED | MEMBARRIER_QUERY)) + ret = membarrier(MEMBARRIER_CMD_QUERY, 0); + if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)) { rcu_has_sys_membarrier = 1; + } } #endif