#include <stdio.h>
#include <pthread.h>
#include <signal.h>
-#include <assert.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <sched.h>
#include "compat-getcpu.h"
+#include <urcu/assert.h>
#include <urcu/wfcqueue.h>
#include <urcu/call-rcu.h>
#include <urcu/pointer.h>
}
#else
static
-int set_thread_cpu_affinity(struct call_rcu_data *crdp)
+int set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((unused)))
{
return 0;
}
{
/* Read call_rcu list before read futex */
cmm_smp_mb();
- if (uatomic_read(&crdp->futex) != -1)
- return;
- while (futex_async(&crdp->futex, FUTEX_WAIT, -1,
- NULL, NULL, 0)) {
+ while (uatomic_read(&crdp->futex) == -1) {
+ if (!futex_async(&crdp->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
+ /*
+ * Prior queued wakeups queued by unrelated code
+ * using the same address can cause futex wait to
+ * return 0 even through the futex value is still
+ * -1 (spurious wakeups). Check the value again
+ * in user-space to validate whether it really
+ * differs from -1.
+ */
+ continue;
+ }
switch (errno) {
- case EWOULDBLOCK:
+ case EAGAIN:
/* Value already changed. */
return;
case EINTR:
/* Retry if interrupted by signal. */
- break; /* Get out of switch. */
+ break; /* Get out of switch. Check again. */
default:
/* Unexpected error. */
urcu_die(errno);
{
/* Read completion barrier count before read futex */
cmm_smp_mb();
- if (uatomic_read(&completion->futex) != -1)
- return;
- while (futex_async(&completion->futex, FUTEX_WAIT, -1,
- NULL, NULL, 0)) {
+ while (uatomic_read(&completion->futex) == -1) {
+ if (!futex_async(&completion->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
+ /*
+ * Prior queued wakeups queued by unrelated code
+ * using the same address can cause futex wait to
+ * return 0 even through the futex value is still
+ * -1 (spurious wakeups). Check the value again
+ * in user-space to validate whether it really
+ * differs from -1.
+ */
+ continue;
+ }
switch (errno) {
- case EWOULDBLOCK:
+ case EAGAIN:
/* Value already changed. */
return;
case EINTR:
/* Retry if interrupted by signal. */
- break; /* Get out of switch. */
+ break; /* Get out of switch. Check again. */
default:
/* Unexpected error. */
urcu_die(errno);
cds_wfcq_init(&cbs_tmp_head, &cbs_tmp_tail);
splice_ret = __cds_wfcq_splice_blocking(&cbs_tmp_head,
&cbs_tmp_tail, &crdp->cbs_head, &crdp->cbs_tail);
- assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK);
- assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY);
+ urcu_posix_assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK);
+ urcu_posix_assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY);
if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) {
synchronize_rcu();
cbcount = 0;
return NULL;
return rcu_dereference(pcpu_crdp[cpu]);
}
-URCU_ATTR_ALIAS(urcu_stringify(get_cpu_call_rcu_data))
-struct call_rcu_data *alias_get_cpu_call_rcu_data();
/*
* Return the tid corresponding to the call_rcu thread whose
{
return crdp->tid;
}
-URCU_ATTR_ALIAS(urcu_stringify(get_call_rcu_thread))
-pthread_t alias_get_call_rcu_thread();
/*
* Create a call_rcu_data structure (with thread) and return a pointer.
return crdp;
}
-URCU_ATTR_ALIAS(urcu_stringify(create_call_rcu_data))
-struct call_rcu_data *alias_create_call_rcu_data();
struct call_rcu_data *create_call_rcu_data(unsigned long flags,
int cpu_affinity)
{
call_rcu_unlock(&call_rcu_mutex);
return 0;
}
-URCU_ATTR_ALIAS(urcu_stringify(set_cpu_call_rcu_data))
-int alias_set_cpu_call_rcu_data();
/*
* Return a pointer to the default call_rcu_data structure, creating
call_rcu_unlock(&call_rcu_mutex);
return default_call_rcu_data;
}
-URCU_ATTR_ALIAS(urcu_stringify(get_default_call_rcu_data))
-struct call_rcu_data *alias_get_default_call_rcu_data();
/*
* Return the call_rcu_data structure that applies to the currently
return get_default_call_rcu_data();
}
-URCU_ATTR_ALIAS(urcu_stringify(get_call_rcu_data))
-struct call_rcu_data *alias_get_call_rcu_data();
/*
* Return a pointer to this task's call_rcu_data if there is one.
{
return URCU_TLS(thread_call_rcu_data);
}
-URCU_ATTR_ALIAS(urcu_stringify(get_thread_call_rcu_data))
-struct call_rcu_data *alias_get_thread_call_rcu_data();
/*
* Set this task's call_rcu_data structure as specified, regardless
{
URCU_TLS(thread_call_rcu_data) = crdp;
}
-URCU_ATTR_ALIAS(urcu_stringify(set_thread_call_rcu_data))
-void alias_set_thread_call_rcu_data();
/*
* Create a separate call_rcu thread for each CPU. This does not
}
return 0;
}
-URCU_ATTR_ALIAS(urcu_stringify(create_all_cpu_call_rcu_data))
-int alias_create_all_cpu_call_rcu_data();
/*
* Wake up the call_rcu thread corresponding to the specified
_call_rcu(head, func, crdp);
_rcu_read_unlock();
}
-URCU_ATTR_ALIAS(urcu_stringify(call_rcu)) void alias_call_rcu();
/*
* Free up the specified call_rcu_data structure, terminating the
free(crdp);
}
-URCU_ATTR_ALIAS(urcu_stringify(call_rcu_data_free))
-void alias_call_rcu_data_free();
/*
* Clean up all the per-CPU call_rcu threads.
}
free(crdp);
}
-#ifdef RCU_QSBR
-/* ABI6 has a non-namespaced free_all_cpu_call_rcu_data for qsbr */
-#undef free_all_cpu_call_rcu_data
-URCU_ATTR_ALIAS("urcu_qsbr_free_all_cpu_call_rcu_data")
-void free_all_cpu_call_rcu_data();
-#define free_all_cpu_call_rcu_data urcu_qsbr_free_all_cpu_call_rcu_data
-#else
-URCU_ATTR_ALIAS(urcu_stringify(free_all_cpu_call_rcu_data))
-void alias_free_all_cpu_call_rcu_data();
-#endif
static
void free_completion(struct urcu_ref *ref)
if (was_online)
rcu_thread_online();
}
-URCU_ATTR_ALIAS(urcu_stringify(rcu_barrier))
-void alias_rcu_barrier();
/*
* Acquire the call_rcu_mutex in order to ensure that the child sees
(void) poll(NULL, 0, 1);
}
}
-URCU_ATTR_ALIAS(urcu_stringify(call_rcu_before_fork))
-void alias_call_rcu_before_fork();
/*
* Clean up call_rcu data structures in the parent of a successful fork()
atfork->after_fork_parent(atfork->priv);
call_rcu_unlock(&call_rcu_mutex);
}
-URCU_ATTR_ALIAS(urcu_stringify(call_rcu_after_fork_parent))
-void alias_call_rcu_after_fork_parent();
/*
* Clean up call_rcu data structures in the child of a successful fork()
call_rcu_data_free(crdp);
}
}
-URCU_ATTR_ALIAS(urcu_stringify(call_rcu_after_fork_child))
-void alias_call_rcu_after_fork_child();
void urcu_register_rculfhash_atfork(struct urcu_atfork *atfork)
{
end:
call_rcu_unlock(&call_rcu_mutex);
}
-URCU_ATTR_ALIAS(urcu_stringify(urcu_register_rculfhash_atfork))
-void alias_urcu_register_rculfhash_atfork();
void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork __attribute__((unused)))
{
end:
call_rcu_unlock(&call_rcu_mutex);
}
-URCU_ATTR_ALIAS(urcu_stringify(urcu_unregister_rculfhash_atfork))
-void alias_urcu_unregister_rculfhash_atfork();