+// SPDX-FileCopyrightText: 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+//
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
#ifndef _URCU_WAIT_H
#define _URCU_WAIT_H
/*
- * urcu-wait.h
- *
* Userspace RCU library wait/wakeup management
- *
- * Copyright (c) 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <urcu/assert.h>
#include <urcu/uatomic.h>
#include <urcu/wfstack.h>
+#include <urcu/futex.h>
#include "urcu-die.h"
/*
};
#define URCU_WAIT_NODE_INIT(name, _state) \
- { .state = _state }
+ { .node = { .next = NULL }, .state = _state }
#define DEFINE_URCU_WAIT_NODE(name, state) \
struct urcu_wait_node name = URCU_WAIT_NODE_INIT(name, state)
static inline
void urcu_adaptative_wake_up(struct urcu_wait_node *wait)
{
- cmm_smp_mb();
urcu_posix_assert(uatomic_read(&wait->state) == URCU_WAIT_WAITING);
- uatomic_set(&wait->state, URCU_WAIT_WAKEUP);
+ uatomic_store(&wait->state, URCU_WAIT_WAKEUP, CMM_RELEASE);
if (!(uatomic_read(&wait->state) & URCU_WAIT_RUNNING)) {
if (futex_noasync(&wait->state, FUTEX_WAKE, 1,
NULL, NULL, 0) < 0)
urcu_die(errno);
}
/* Allow teardown of struct urcu_wait memory. */
- uatomic_or(&wait->state, URCU_WAIT_TEARDOWN);
+ uatomic_or_mo(&wait->state, URCU_WAIT_TEARDOWN, CMM_RELEASE);
}
/*
/* Load and test condition before read state */
cmm_smp_rmb();
for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
- if (uatomic_read(&wait->state) != URCU_WAIT_WAITING)
+ if (uatomic_load(&wait->state, CMM_ACQUIRE) != URCU_WAIT_WAITING)
goto skip_futex_wait;
caa_cpu_relax();
}
- while (futex_noasync(&wait->state, FUTEX_WAIT, URCU_WAIT_WAITING,
- NULL, NULL, 0)) {
+ while (uatomic_load(&wait->state, CMM_ACQUIRE) == URCU_WAIT_WAITING) {
+ if (!futex_noasync(&wait->state, FUTEX_WAIT, URCU_WAIT_WAITING, NULL, NULL, 0)) {
+ /*
+ * Prior queued wakeups queued by unrelated code
+ * using the same address can cause futex wait to
+ * return 0 even through the futex value is still
+ * URCU_WAIT_WAITING (spurious wakeups). Check
+ * the value again in user-space to validate
+ * whether it really differs from
+ * URCU_WAIT_WAITING.
+ */
+ continue;
+ }
switch (errno) {
- case EWOULDBLOCK:
+ case EAGAIN:
/* Value already changed. */
goto skip_futex_wait;
case EINTR:
/* Retry if interrupted by signal. */
- break; /* Get out of switch. */
+ break; /* Get out of switch. Check again. */
default:
/* Unexpected error. */
urcu_die(errno);
* memory allocated for struct urcu_wait.
*/
for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
- if (uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN)
+ if (uatomic_load(&wait->state, CMM_RELAXED) & URCU_WAIT_TEARDOWN)
break;
caa_cpu_relax();
}
- while (!(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN))
+ while (!(uatomic_load(&wait->state, CMM_ACQUIRE) & URCU_WAIT_TEARDOWN))
poll(NULL, 0, 10);
urcu_posix_assert(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN);
}
caa_container_of(iter, struct urcu_wait_node, node);
/* Don't wake already running threads */
- if (wait_node->state & URCU_WAIT_RUNNING)
+ if (uatomic_load(&wait_node->state, CMM_RELAXED) & URCU_WAIT_RUNNING)
continue;
urcu_adaptative_wake_up(wait_node);
}