summary |
shortlog |
log |
commit | commitdiff |
tree
raw |
patch |
inline | side by side (from parent 1:
c0bb9f6)
For the liburcu-bp flavor, use the membarrier private expedited command
when available. It is faster than the shared expedited command, but has
only been introduced in 4.14 Linux kernels.
When configured with --disable-sys-membarrier-fallback, liburcu-bp
will abort if running on a kernel that do not provide the membarrier
private expedited command (e.g. CONFIG_MEMBARRIER=n or kernel version
below 4.14).
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
#include <errno.h>
#include <poll.h>
#include <unistd.h>
#include <errno.h>
#include <poll.h>
#include <unistd.h>
#include <sys/mman.h>
#include "urcu/arch.h"
#include <sys/mman.h>
#include "urcu/arch.h"
#endif
enum membarrier_cmd {
#endif
enum membarrier_cmd {
- MEMBARRIER_CMD_QUERY = 0,
- MEMBARRIER_CMD_SHARED = (1 << 0),
+ MEMBARRIER_CMD_QUERY = 0,
+ MEMBARRIER_CMD_SHARED = (1 << 0),
+ /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
+ /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
+ MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3),
+ MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4),
static void smp_mb_master(void)
{
static void smp_mb_master(void)
{
- if (caa_likely(urcu_bp_has_sys_membarrier))
- (void) membarrier(MEMBARRIER_CMD_SHARED, 0);
- else
+ if (caa_likely(urcu_bp_has_sys_membarrier)) {
+ if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0))
+ urcu_die(errno);
+ } else {
#ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
static
#ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
static
-void rcu_sys_membarrier_status(int available)
+void rcu_sys_membarrier_status(bool available)
{
if (!available)
abort();
}
#else
static
{
if (!available)
abort();
}
#else
static
-void rcu_sys_membarrier_status(int available)
+void rcu_sys_membarrier_status(bool available)
- /*
- * membarrier has blocking behavior, which changes the
- * application behavior too much compared to using barriers when
- * synchronize_rcu is used repeatedly (without using call_rcu).
- * Don't use membarrier for now, unless its use has been
- * explicitly forced when building liburcu.
- */
+ if (!available)
+ return;
+ urcu_bp_has_sys_membarrier = 1;
+static
+void rcu_sys_membarrier_init(void)
+{
+ bool available = false;
+ int mask;
+
+ mask = membarrier(MEMBARRIER_CMD_QUERY, 0);
+ if (mask >= 0) {
+ if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
+ if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0))
+ urcu_die(errno);
+ available = true;
+ }
+ }
+ rcu_sys_membarrier_status(available);
+}
+
static
void rcu_bp_init(void)
{
static
void rcu_bp_init(void)
{
urcu_bp_thread_exit_notifier);
if (ret)
abort();
urcu_bp_thread_exit_notifier);
if (ret)
abort();
- ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
- rcu_sys_membarrier_status(ret >= 0
- && (ret & MEMBARRIER_CMD_SHARED));
+ rcu_sys_membarrier_init();
initialized = 1;
}
mutex_unlock(&init_lock);
initialized = 1;
}
mutex_unlock(&init_lock);