From: Mathieu Desnoyers Date: Thu, 21 Dec 2017 18:42:23 +0000 (-0500) Subject: liburcu: Use membarrier private expedited when available X-Git-Tag: v0.11.0~46 X-Git-Url: https://git.liburcu.org/?a=commitdiff_plain;h=c0bb9f693f926595a7cb8b4ce712cef08d9f5d49;hp=6ef909036441002b1f219a45d2ec453c60328163;p=urcu.git liburcu: Use membarrier private expedited when available For the liburcu flavor, use the membarrier private expedited command when available. It is faster than the shared expedited command, but has only been introduced in 4.14 Linux kernels. When configured with --disable-sys-membarrier-fallback, liburcu will abort if running on a kernel that provide neither the shared nor the private expedited membarrier commands. This is the case if running on a CONFIG_MEMBARRIER=n kernel, or a kernel version below 4.3. Signed-off-by: Mathieu Desnoyers --- diff --git a/src/urcu.c b/src/urcu.c index a81b129..c47f51b 100644 --- a/src/urcu.c +++ b/src/urcu.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include "urcu/arch.h" @@ -71,12 +72,18 @@ #endif enum membarrier_cmd { - MEMBARRIER_CMD_QUERY = 0, - MEMBARRIER_CMD_SHARED = (1 << 0), + MEMBARRIER_CMD_QUERY = 0, + MEMBARRIER_CMD_SHARED = (1 << 0), + /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */ + /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */ + MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3), + MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4), }; #ifdef RCU_MEMBARRIER static int init_done; +static int has_sys_membarrier_private_expedited; + #ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER int rcu_has_sys_membarrier_memb; #endif @@ -162,10 +169,14 @@ static void mutex_unlock(pthread_mutex_t *mutex) #ifdef RCU_MEMBARRIER static void smp_mb_master(void) { - if (caa_likely(rcu_has_sys_membarrier_memb)) - (void) membarrier(MEMBARRIER_CMD_SHARED, 0); - else + if (caa_likely(rcu_has_sys_membarrier_memb)) { + if (membarrier(has_sys_membarrier_private_expedited ? + MEMBARRIER_CMD_PRIVATE_EXPEDITED : + MEMBARRIER_CMD_SHARED, 0)) + urcu_die(errno); + } else { cmm_smp_mb(); + } } #endif @@ -537,29 +548,47 @@ void rcu_unregister_thread(void) #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER static -void rcu_sys_membarrier_status(int available) +void rcu_sys_membarrier_status(bool available) { if (!available) abort(); } #else static -void rcu_sys_membarrier_status(int available) +void rcu_sys_membarrier_status(bool available) { - if (available) - rcu_has_sys_membarrier_memb = 1; + if (!available) + return; + rcu_has_sys_membarrier_memb = 1; } #endif -void rcu_init(void) +static +void rcu_sys_membarrier_init(void) { - int ret; + bool available = false; + int mask; + + mask = membarrier(MEMBARRIER_CMD_QUERY, 0); + if (mask >= 0) { + if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) { + if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0)) + urcu_die(errno); + has_sys_membarrier_private_expedited = 1; + available = true; + } else if (mask & MEMBARRIER_CMD_SHARED) { + available = true; + } + } + rcu_sys_membarrier_status(available); +} +void rcu_init(void) +{ if (init_done) return; init_done = 1; - ret = membarrier(MEMBARRIER_CMD_QUERY, 0); - rcu_sys_membarrier_status(ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)); + rcu_sys_membarrier_init(); } #endif