X-Git-Url: https://git.liburcu.org/?a=blobdiff_plain;f=src%2Furcu-call-rcu-impl.h;h=2ad02eb1b796614bd5ddfc1930b580277c0d09c7;hb=98d705dd32be11b3f134b5aed87f9c201c028c10;hp=18fd65a0ec45b04de012dff83838f260cbe07bcb;hpb=51b98c3cb988eca7538238e22268bdcf316d4541;p=urcu.git diff --git a/src/urcu-call-rcu-impl.h b/src/urcu-call-rcu-impl.h index 18fd65a..2ad02eb 100644 --- a/src/urcu-call-rcu-impl.h +++ b/src/urcu-call-rcu-impl.h @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -35,6 +34,7 @@ #include #include "compat-getcpu.h" +#include #include #include #include @@ -199,7 +199,7 @@ static void call_rcu_unlock(pthread_mutex_t *pmp) * Losing affinity can be caused by CPU hotunplug/hotplug, or by * cpuset(7). */ -#if HAVE_SCHED_SETAFFINITY +#ifdef HAVE_SCHED_SETAFFINITY static int set_thread_cpu_affinity(struct call_rcu_data *crdp) { @@ -215,11 +215,8 @@ int set_thread_cpu_affinity(struct call_rcu_data *crdp) CPU_ZERO(&mask); CPU_SET(crdp->cpu_affinity, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - ret = sched_setaffinity(0, &mask); -#else ret = sched_setaffinity(0, sizeof(mask), &mask); -#endif + /* * EINVAL is fine: can be caused by hotunplugged CPUs, or by * cpuset(7). This is why we should always retry if we detect @@ -233,7 +230,7 @@ int set_thread_cpu_affinity(struct call_rcu_data *crdp) } #else static -int set_thread_cpu_affinity(struct call_rcu_data *crdp) +int set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((unused))) { return 0; } @@ -243,17 +240,25 @@ static void call_rcu_wait(struct call_rcu_data *crdp) { /* Read call_rcu list before read futex */ cmm_smp_mb(); - if (uatomic_read(&crdp->futex) != -1) - return; - while (futex_async(&crdp->futex, FUTEX_WAIT, -1, - NULL, NULL, 0)) { + while (uatomic_read(&crdp->futex) == -1) { + if (!futex_async(&crdp->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) { + /* + * Prior queued wakeups queued by unrelated code + * using the same address can cause futex wait to + * return 0 even through the futex value is still + * -1 (spurious wakeups). Check the value again + * in user-space to validate whether it really + * differs from -1. + */ + continue; + } switch (errno) { - case EWOULDBLOCK: + case EAGAIN: /* Value already changed. */ return; case EINTR: /* Retry if interrupted by signal. */ - break; /* Get out of switch. */ + break; /* Get out of switch. Check again. */ default: /* Unexpected error. */ urcu_die(errno); @@ -277,17 +282,25 @@ static void call_rcu_completion_wait(struct call_rcu_completion *completion) { /* Read completion barrier count before read futex */ cmm_smp_mb(); - if (uatomic_read(&completion->futex) != -1) - return; - while (futex_async(&completion->futex, FUTEX_WAIT, -1, - NULL, NULL, 0)) { + while (uatomic_read(&completion->futex) == -1) { + if (!futex_async(&completion->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) { + /* + * Prior queued wakeups queued by unrelated code + * using the same address can cause futex wait to + * return 0 even through the futex value is still + * -1 (spurious wakeups). Check the value again + * in user-space to validate whether it really + * differs from -1. + */ + continue; + } switch (errno) { - case EWOULDBLOCK: + case EAGAIN: /* Value already changed. */ return; case EINTR: /* Retry if interrupted by signal. */ - break; /* Get out of switch. */ + break; /* Get out of switch. Check again. */ default: /* Unexpected error. */ urcu_die(errno); @@ -358,8 +371,8 @@ static void *call_rcu_thread(void *arg) cds_wfcq_init(&cbs_tmp_head, &cbs_tmp_tail); splice_ret = __cds_wfcq_splice_blocking(&cbs_tmp_head, &cbs_tmp_tail, &crdp->cbs_head, &crdp->cbs_tail); - assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK); - assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY); + urcu_posix_assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK); + urcu_posix_assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY); if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) { synchronize_rcu(); cbcount = 0; @@ -465,8 +478,6 @@ struct call_rcu_data *get_cpu_call_rcu_data(int cpu) return NULL; return rcu_dereference(pcpu_crdp[cpu]); } -URCU_ATTR_ALIAS(urcu_stringify(get_cpu_call_rcu_data)) -struct call_rcu_data *alias_get_cpu_call_rcu_data(); /* * Return the tid corresponding to the call_rcu thread whose @@ -477,8 +488,6 @@ pthread_t get_call_rcu_thread(struct call_rcu_data *crdp) { return crdp->tid; } -URCU_ATTR_ALIAS(urcu_stringify(get_call_rcu_thread)) -pthread_t alias_get_call_rcu_thread(); /* * Create a call_rcu_data structure (with thread) and return a pointer. @@ -493,8 +502,6 @@ static struct call_rcu_data *__create_call_rcu_data(unsigned long flags, return crdp; } -URCU_ATTR_ALIAS(urcu_stringify(create_call_rcu_data)) -struct call_rcu_data *alias_create_call_rcu_data(); struct call_rcu_data *create_call_rcu_data(unsigned long flags, int cpu_affinity) { @@ -551,8 +558,6 @@ int set_cpu_call_rcu_data(int cpu, struct call_rcu_data *crdp) call_rcu_unlock(&call_rcu_mutex); return 0; } -URCU_ATTR_ALIAS(urcu_stringify(set_cpu_call_rcu_data)) -int alias_set_cpu_call_rcu_data(); /* * Return a pointer to the default call_rcu_data structure, creating @@ -573,8 +578,6 @@ struct call_rcu_data *get_default_call_rcu_data(void) call_rcu_unlock(&call_rcu_mutex); return default_call_rcu_data; } -URCU_ATTR_ALIAS(urcu_stringify(get_default_call_rcu_data)) -struct call_rcu_data *alias_get_default_call_rcu_data(); /* * Return the call_rcu_data structure that applies to the currently @@ -602,8 +605,6 @@ struct call_rcu_data *get_call_rcu_data(void) return get_default_call_rcu_data(); } -URCU_ATTR_ALIAS(urcu_stringify(get_call_rcu_data)) -struct call_rcu_data *alias_get_call_rcu_data(); /* * Return a pointer to this task's call_rcu_data if there is one. @@ -613,8 +614,6 @@ struct call_rcu_data *get_thread_call_rcu_data(void) { return URCU_TLS(thread_call_rcu_data); } -URCU_ATTR_ALIAS(urcu_stringify(get_thread_call_rcu_data)) -struct call_rcu_data *alias_get_thread_call_rcu_data(); /* * Set this task's call_rcu_data structure as specified, regardless @@ -631,8 +630,6 @@ void set_thread_call_rcu_data(struct call_rcu_data *crdp) { URCU_TLS(thread_call_rcu_data) = crdp; } -URCU_ATTR_ALIAS(urcu_stringify(set_thread_call_rcu_data)) -void alias_set_thread_call_rcu_data(); /* * Create a separate call_rcu thread for each CPU. This does not @@ -684,8 +681,6 @@ int create_all_cpu_call_rcu_data(unsigned long flags) } return 0; } -URCU_ATTR_ALIAS(urcu_stringify(create_all_cpu_call_rcu_data)) -int alias_create_all_cpu_call_rcu_data(); /* * Wake up the call_rcu thread corresponding to the specified @@ -733,7 +728,6 @@ void call_rcu(struct rcu_head *head, _call_rcu(head, func, crdp); _rcu_read_unlock(); } -URCU_ATTR_ALIAS(urcu_stringify(call_rcu)) void alias_call_rcu(); /* * Free up the specified call_rcu_data structure, terminating the @@ -792,8 +786,6 @@ void call_rcu_data_free(struct call_rcu_data *crdp) free(crdp); } -URCU_ATTR_ALIAS(urcu_stringify(call_rcu_data_free)) -void alias_call_rcu_data_free(); /* * Clean up all the per-CPU call_rcu threads. @@ -834,16 +826,6 @@ void free_all_cpu_call_rcu_data(void) } free(crdp); } -#ifdef RCU_QSBR -/* ABI6 has a non-namespaced free_all_cpu_call_rcu_data for qsbr */ -#undef free_all_cpu_call_rcu_data -URCU_ATTR_ALIAS("urcu_qsbr_free_all_cpu_call_rcu_data") -void free_all_cpu_call_rcu_data(); -#define free_all_cpu_call_rcu_data urcu_qsbr_free_all_cpu_call_rcu_data -#else -URCU_ATTR_ALIAS(urcu_stringify(free_all_cpu_call_rcu_data)) -void alias_free_all_cpu_call_rcu_data(); -#endif static void free_completion(struct urcu_ref *ref) @@ -935,8 +917,6 @@ online: if (was_online) rcu_thread_online(); } -URCU_ATTR_ALIAS(urcu_stringify(rcu_barrier)) -void alias_rcu_barrier(); /* * Acquire the call_rcu_mutex in order to ensure that the child sees @@ -965,8 +945,6 @@ void call_rcu_before_fork(void) (void) poll(NULL, 0, 1); } } -URCU_ATTR_ALIAS(urcu_stringify(call_rcu_before_fork)) -void alias_call_rcu_before_fork(); /* * Clean up call_rcu data structures in the parent of a successful fork() @@ -989,8 +967,6 @@ void call_rcu_after_fork_parent(void) atfork->after_fork_parent(atfork->priv); call_rcu_unlock(&call_rcu_mutex); } -URCU_ATTR_ALIAS(urcu_stringify(call_rcu_after_fork_parent)) -void alias_call_rcu_after_fork_parent(); /* * Clean up call_rcu data structures in the child of a successful fork() @@ -1038,8 +1014,6 @@ void call_rcu_after_fork_child(void) call_rcu_data_free(crdp); } } -URCU_ATTR_ALIAS(urcu_stringify(call_rcu_after_fork_child)) -void alias_call_rcu_after_fork_child(); void urcu_register_rculfhash_atfork(struct urcu_atfork *atfork) { @@ -1050,10 +1024,8 @@ void urcu_register_rculfhash_atfork(struct urcu_atfork *atfork) end: call_rcu_unlock(&call_rcu_mutex); } -URCU_ATTR_ALIAS(urcu_stringify(urcu_register_rculfhash_atfork)) -void alias_urcu_register_rculfhash_atfork(); -void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork) +void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork __attribute__((unused))) { call_rcu_lock(&call_rcu_mutex); if (--registered_rculfhash_atfork_refcount) @@ -1062,5 +1034,3 @@ void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork) end: call_rcu_unlock(&call_rcu_mutex); } -URCU_ATTR_ALIAS(urcu_stringify(urcu_unregister_rculfhash_atfork)) -void alias_urcu_unregister_rculfhash_atfork();