summary |
shortlog |
log |
commit | commitdiff |
tree
raw |
patch |
inline | side by side (from parent 1:
6ef9090)
For the liburcu flavor, use the membarrier private expedited
command when available. It is faster than the shared expedited
command, but has only been introduced in 4.14 Linux kernels.
When configured with --disable-sys-membarrier-fallback, liburcu
will abort if running on a kernel that provide neither the shared
nor the private expedited membarrier commands. This is the case
if running on a CONFIG_MEMBARRIER=n kernel, or a kernel version
below 4.3.
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <poll.h>
#include "urcu/arch.h"
#include <poll.h>
#include "urcu/arch.h"
#endif
enum membarrier_cmd {
#endif
enum membarrier_cmd {
- MEMBARRIER_CMD_QUERY = 0,
- MEMBARRIER_CMD_SHARED = (1 << 0),
+ MEMBARRIER_CMD_QUERY = 0,
+ MEMBARRIER_CMD_SHARED = (1 << 0),
+ /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
+ /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
+ MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3),
+ MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4),
};
#ifdef RCU_MEMBARRIER
static int init_done;
};
#ifdef RCU_MEMBARRIER
static int init_done;
+static int has_sys_membarrier_private_expedited;
+
#ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER
int rcu_has_sys_membarrier_memb;
#endif
#ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER
int rcu_has_sys_membarrier_memb;
#endif
#ifdef RCU_MEMBARRIER
static void smp_mb_master(void)
{
#ifdef RCU_MEMBARRIER
static void smp_mb_master(void)
{
- if (caa_likely(rcu_has_sys_membarrier_memb))
- (void) membarrier(MEMBARRIER_CMD_SHARED, 0);
- else
+ if (caa_likely(rcu_has_sys_membarrier_memb)) {
+ if (membarrier(has_sys_membarrier_private_expedited ?
+ MEMBARRIER_CMD_PRIVATE_EXPEDITED :
+ MEMBARRIER_CMD_SHARED, 0))
+ urcu_die(errno);
+ } else {
#ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
static
#ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
static
-void rcu_sys_membarrier_status(int available)
+void rcu_sys_membarrier_status(bool available)
{
if (!available)
abort();
}
#else
static
{
if (!available)
abort();
}
#else
static
-void rcu_sys_membarrier_status(int available)
+void rcu_sys_membarrier_status(bool available)
- if (available)
- rcu_has_sys_membarrier_memb = 1;
+ if (!available)
+ return;
+ rcu_has_sys_membarrier_memb = 1;
+static
+void rcu_sys_membarrier_init(void)
+ bool available = false;
+ int mask;
+
+ mask = membarrier(MEMBARRIER_CMD_QUERY, 0);
+ if (mask >= 0) {
+ if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
+ if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0))
+ urcu_die(errno);
+ has_sys_membarrier_private_expedited = 1;
+ available = true;
+ } else if (mask & MEMBARRIER_CMD_SHARED) {
+ available = true;
+ }
+ }
+ rcu_sys_membarrier_status(available);
+}
if (init_done)
return;
init_done = 1;
if (init_done)
return;
init_done = 1;
- ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
- rcu_sys_membarrier_status(ret >= 0 && (ret & MEMBARRIER_CMD_SHARED));
+ rcu_sys_membarrier_init();