#include <stdio.h>
#include <pthread.h>
#include <signal.h>
-#include <assert.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <sched.h>
#include "compat-getcpu.h"
-#include "urcu/wfcqueue.h"
-#include "urcu-call-rcu.h"
-#include "urcu-pointer.h"
-#include "urcu/list.h"
-#include "urcu/futex.h"
-#include "urcu/tls-compat.h"
-#include "urcu/ref.h"
+#include <urcu/assert.h>
+#include <urcu/wfcqueue.h>
+#include <urcu/call-rcu.h>
+#include <urcu/pointer.h>
+#include <urcu/list.h>
+#include <urcu/futex.h>
+#include <urcu/tls-compat.h>
+#include <urcu/ref.h>
#include "urcu-die.h"
+#include "urcu-utils.h"
#define SET_AFFINITY_CHECK_PERIOD (1U << 8) /* 256 */
#define SET_AFFINITY_CHECK_PERIOD_MASK (SET_AFFINITY_CHECK_PERIOD - 1)
* Losing affinity can be caused by CPU hotunplug/hotplug, or by
* cpuset(7).
*/
-#if HAVE_SCHED_SETAFFINITY
+#ifdef HAVE_SCHED_SETAFFINITY
static
int set_thread_cpu_affinity(struct call_rcu_data *crdp)
{
CPU_ZERO(&mask);
CPU_SET(crdp->cpu_affinity, &mask);
-#if SCHED_SETAFFINITY_ARGS == 2
- ret = sched_setaffinity(0, &mask);
-#else
ret = sched_setaffinity(0, sizeof(mask), &mask);
-#endif
+
/*
* EINVAL is fine: can be caused by hotunplugged CPUs, or by
* cpuset(7). This is why we should always retry if we detect
}
#else
static
-int set_thread_cpu_affinity(struct call_rcu_data *crdp)
+int set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((unused)))
{
return 0;
}
{
/* Read call_rcu list before read futex */
cmm_smp_mb();
- if (uatomic_read(&crdp->futex) != -1)
- return;
- while (futex_async(&crdp->futex, FUTEX_WAIT, -1,
- NULL, NULL, 0)) {
+ while (uatomic_read(&crdp->futex) == -1) {
+ if (!futex_async(&crdp->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
+ /*
+ * Prior queued wakeups queued by unrelated code
+ * using the same address can cause futex wait to
+ * return 0 even through the futex value is still
+ * -1 (spurious wakeups). Check the value again
+ * in user-space to validate whether it really
+ * differs from -1.
+ */
+ continue;
+ }
switch (errno) {
- case EWOULDBLOCK:
+ case EAGAIN:
/* Value already changed. */
return;
case EINTR:
/* Retry if interrupted by signal. */
- break; /* Get out of switch. */
+ break; /* Get out of switch. Check again. */
default:
/* Unexpected error. */
urcu_die(errno);
{
/* Read completion barrier count before read futex */
cmm_smp_mb();
- if (uatomic_read(&completion->futex) != -1)
- return;
- while (futex_async(&completion->futex, FUTEX_WAIT, -1,
- NULL, NULL, 0)) {
+ while (uatomic_read(&completion->futex) == -1) {
+ if (!futex_async(&completion->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
+ /*
+ * Prior queued wakeups queued by unrelated code
+ * using the same address can cause futex wait to
+ * return 0 even through the futex value is still
+ * -1 (spurious wakeups). Check the value again
+ * in user-space to validate whether it really
+ * differs from -1.
+ */
+ continue;
+ }
switch (errno) {
- case EWOULDBLOCK:
+ case EAGAIN:
/* Value already changed. */
return;
case EINTR:
/* Retry if interrupted by signal. */
- break; /* Get out of switch. */
+ break; /* Get out of switch. Check again. */
default:
/* Unexpected error. */
urcu_die(errno);
cds_wfcq_init(&cbs_tmp_head, &cbs_tmp_tail);
splice_ret = __cds_wfcq_splice_blocking(&cbs_tmp_head,
&cbs_tmp_tail, &crdp->cbs_head, &crdp->cbs_tail);
- assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK);
- assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY);
+ urcu_posix_assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK);
+ urcu_posix_assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY);
if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) {
synchronize_rcu();
cbcount = 0;
while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOPPED) == 0)
(void) poll(NULL, 0, 1);
}
+ call_rcu_lock(&call_rcu_mutex);
if (!cds_wfcq_empty(&crdp->cbs_head, &crdp->cbs_tail)) {
- /* Create default call rcu data if need be */
+ call_rcu_unlock(&call_rcu_mutex);
+ /* Create default call rcu data if need be. */
+ /* CBs queued here will be handed to the default list. */
(void) get_default_call_rcu_data();
+ call_rcu_lock(&call_rcu_mutex);
__cds_wfcq_splice_blocking(&default_call_rcu_data->cbs_head,
&default_call_rcu_data->cbs_tail,
&crdp->cbs_head, &crdp->cbs_tail);
wake_call_rcu_thread(default_call_rcu_data);
}
- call_rcu_lock(&call_rcu_mutex);
cds_list_del(&crdp->list);
call_rcu_unlock(&call_rcu_mutex);
call_rcu_unlock(&call_rcu_mutex);
}
-void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork)
+void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork __attribute__((unused)))
{
call_rcu_lock(&call_rcu_mutex);
if (--registered_rculfhash_atfork_refcount)