X-Git-Url: https://git.liburcu.org/?a=blobdiff_plain;f=src%2Furcu-call-rcu-impl.h;h=6301d135930a0da30f71e1ec21aa40585db40210;hb=19ec56ca7d502f12dfb7f7d4299b506ba5abecce;hp=f7844cc17f6dc49f774650cba20bbdc8ecf825b0;hpb=4477a87021ffbfbfdb2a2084d05a084171343d36;p=urcu.git

diff --git a/src/urcu-call-rcu-impl.h b/src/urcu-call-rcu-impl.h
index f7844cc..6301d13 100644
--- a/src/urcu-call-rcu-impl.h
+++ b/src/urcu-call-rcu-impl.h
@@ -44,6 +44,7 @@
 #include <urcu/ref.h>
 #include "urcu-die.h"
 #include "urcu-utils.h"
+#include "compat-smp.h"
 
 #define SET_AFFINITY_CHECK_PERIOD		(1U << 8)	/* 256 */
 #define SET_AFFINITY_CHECK_PERIOD_MASK		(SET_AFFINITY_CHECK_PERIOD - 1)
@@ -101,7 +102,6 @@ static pthread_mutex_t call_rcu_mutex = PTHREAD_MUTEX_INITIALIZER;
 static struct call_rcu_data *default_call_rcu_data;
 
 static struct urcu_atfork *registered_rculfhash_atfork;
-static unsigned long registered_rculfhash_atfork_refcount;
 
 /*
  * If the sched_getcpu() and sysconf(_SC_NPROCESSORS_CONF) calls are
@@ -120,11 +120,11 @@ static unsigned long registered_rculfhash_atfork_refcount;
  */
 
 static struct call_rcu_data **per_cpu_call_rcu_data;
-static long maxcpus;
+static long cpus_array_len;
 
-static void maxcpus_reset(void)
+static void cpus_array_len_reset(void)
 {
-	maxcpus = 0;
+	cpus_array_len = 0;
 }
 
 /* Allocate the array if it has not already been allocated. */
@@ -134,15 +134,15 @@ static void alloc_cpu_call_rcu_data(void)
 	struct call_rcu_data **p;
 	static int warned = 0;
 
-	if (maxcpus != 0)
+	if (cpus_array_len != 0)
 		return;
-	maxcpus = sysconf(_SC_NPROCESSORS_CONF);
-	if (maxcpus <= 0) {
+	cpus_array_len = get_possible_cpus_array_len();
+	if (cpus_array_len <= 0) {
 		return;
 	}
-	p = malloc(maxcpus * sizeof(*per_cpu_call_rcu_data));
+	p = malloc(cpus_array_len * sizeof(*per_cpu_call_rcu_data));
 	if (p != NULL) {
-		memset(p, '\0', maxcpus * sizeof(*per_cpu_call_rcu_data));
+		memset(p, '\0', cpus_array_len * sizeof(*per_cpu_call_rcu_data));
 		rcu_set_pointer(&per_cpu_call_rcu_data, p);
 	} else {
 		if (!warned) {
@@ -160,9 +160,9 @@ static void alloc_cpu_call_rcu_data(void)
  * constant.
  */
 static struct call_rcu_data **per_cpu_call_rcu_data = NULL;
-static const long maxcpus = -1;
+static const long cpus_array_len = -1;
 
-static void maxcpus_reset(void)
+static void cpus_array_len_reset(void)
 {
 }
 
@@ -199,7 +199,7 @@ static void call_rcu_unlock(pthread_mutex_t *pmp)
  * Losing affinity can be caused by CPU hotunplug/hotplug, or by
  * cpuset(7).
  */
-#if HAVE_SCHED_SETAFFINITY
+#ifdef HAVE_SCHED_SETAFFINITY
 static
 int set_thread_cpu_affinity(struct call_rcu_data *crdp)
 {
@@ -233,7 +233,7 @@ int set_thread_cpu_affinity(struct call_rcu_data *crdp)
 }
 #else
 static
-int set_thread_cpu_affinity(struct call_rcu_data *crdp)
+int set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((unused)))
 {
 	return 0;
 }
@@ -243,17 +243,25 @@ static void call_rcu_wait(struct call_rcu_data *crdp)
 {
 	/* Read call_rcu list before read futex */
 	cmm_smp_mb();
-	if (uatomic_read(&crdp->futex) != -1)
-		return;
-	while (futex_async(&crdp->futex, FUTEX_WAIT, -1,
-			NULL, NULL, 0)) {
+	while (uatomic_read(&crdp->futex) == -1) {
+		if (!futex_async(&crdp->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
+			/*
+			 * Prior queued wakeups queued by unrelated code
+			 * using the same address can cause futex wait to
+			 * return 0 even through the futex value is still
+			 * -1 (spurious wakeups). Check the value again
+			 * in user-space to validate whether it really
+			 * differs from -1.
+			 */
+			continue;
+		}
 		switch (errno) {
-		case EWOULDBLOCK:
+		case EAGAIN:
 			/* Value already changed. */
 			return;
 		case EINTR:
 			/* Retry if interrupted by signal. */
-			break;	/* Get out of switch. */
+			break;	/* Get out of switch. Check again. */
 		default:
 			/* Unexpected error. */
 			urcu_die(errno);
@@ -277,17 +285,25 @@ static void call_rcu_completion_wait(struct call_rcu_completion *completion)
 {
 	/* Read completion barrier count before read futex */
 	cmm_smp_mb();
-	if (uatomic_read(&completion->futex) != -1)
-		return;
-	while (futex_async(&completion->futex, FUTEX_WAIT, -1,
-			NULL, NULL, 0)) {
+	while (uatomic_read(&completion->futex) == -1) {
+		if (!futex_async(&completion->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
+			/*
+			 * Prior queued wakeups queued by unrelated code
+			 * using the same address can cause futex wait to
+			 * return 0 even through the futex value is still
+			 * -1 (spurious wakeups). Check the value again
+			 * in user-space to validate whether it really
+			 * differs from -1.
+			 */
+			continue;
+		}
 		switch (errno) {
-		case EWOULDBLOCK:
+		case EAGAIN:
 			/* Value already changed. */
 			return;
 		case EINTR:
 			/* Retry if interrupted by signal. */
-			break;	/* Get out of switch. */
+			break;	/* Get out of switch. Check again. */
 		default:
 			/* Unexpected error. */
 			urcu_die(errno);
@@ -457,15 +473,15 @@ struct call_rcu_data *get_cpu_call_rcu_data(int cpu)
 	pcpu_crdp = rcu_dereference(per_cpu_call_rcu_data);
 	if (pcpu_crdp == NULL)
 		return NULL;
-	if (!warned && maxcpus > 0 && (cpu < 0 || maxcpus <= cpu)) {
+	if (!warned && cpus_array_len > 0 && (cpu < 0 || cpus_array_len <= cpu)) {
 		fprintf(stderr, "[error] liburcu: get CPU # out of range\n");
 		warned = 1;
 	}
-	if (cpu < 0 || maxcpus <= cpu)
+	if (cpu < 0 || cpus_array_len <= cpu)
 		return NULL;
 	return rcu_dereference(pcpu_crdp[cpu]);
 }
-__attribute__((alias(urcu_stringify(get_cpu_call_rcu_data))))
+URCU_ATTR_ALIAS(urcu_stringify(get_cpu_call_rcu_data))
 struct call_rcu_data *alias_get_cpu_call_rcu_data();
 
 /*
@@ -477,7 +493,7 @@ pthread_t get_call_rcu_thread(struct call_rcu_data *crdp)
 {
 	return crdp->tid;
 }
-__attribute__((alias(urcu_stringify(get_call_rcu_thread))))
+URCU_ATTR_ALIAS(urcu_stringify(get_call_rcu_thread))
 pthread_t alias_get_call_rcu_thread();
 
 /*
@@ -493,7 +509,7 @@ static struct call_rcu_data *__create_call_rcu_data(unsigned long flags,
 	return crdp;
 }
 
-__attribute__((alias(urcu_stringify(create_call_rcu_data))))
+URCU_ATTR_ALIAS(urcu_stringify(create_call_rcu_data))
 struct call_rcu_data *alias_create_call_rcu_data();
 struct call_rcu_data *create_call_rcu_data(unsigned long flags,
 					   int cpu_affinity)
@@ -525,7 +541,7 @@ int set_cpu_call_rcu_data(int cpu, struct call_rcu_data *crdp)
 
 	call_rcu_lock(&call_rcu_mutex);
 	alloc_cpu_call_rcu_data();
-	if (cpu < 0 || maxcpus <= cpu) {
+	if (cpu < 0 || cpus_array_len <= cpu) {
 		if (!warned) {
 			fprintf(stderr, "[error] liburcu: set CPU # out of range\n");
 			warned = 1;
@@ -551,7 +567,7 @@ int set_cpu_call_rcu_data(int cpu, struct call_rcu_data *crdp)
 	call_rcu_unlock(&call_rcu_mutex);
 	return 0;
 }
-__attribute__((alias(urcu_stringify(set_cpu_call_rcu_data))))
+URCU_ATTR_ALIAS(urcu_stringify(set_cpu_call_rcu_data))
 int alias_set_cpu_call_rcu_data();
 
 /*
@@ -573,7 +589,7 @@ struct call_rcu_data *get_default_call_rcu_data(void)
 	call_rcu_unlock(&call_rcu_mutex);
 	return default_call_rcu_data;
 }
-__attribute__((alias(urcu_stringify(get_default_call_rcu_data))))
+URCU_ATTR_ALIAS(urcu_stringify(get_default_call_rcu_data))
 struct call_rcu_data *alias_get_default_call_rcu_data();
 
 /*
@@ -594,7 +610,7 @@ struct call_rcu_data *get_call_rcu_data(void)
 	if (URCU_TLS(thread_call_rcu_data) != NULL)
 		return URCU_TLS(thread_call_rcu_data);
 
-	if (maxcpus > 0) {
+	if (cpus_array_len > 0) {
 		crd = get_cpu_call_rcu_data(urcu_sched_getcpu());
 		if (crd)
 			return crd;
@@ -602,7 +618,7 @@ struct call_rcu_data *get_call_rcu_data(void)
 
 	return get_default_call_rcu_data();
 }
-__attribute__((alias(urcu_stringify(get_call_rcu_data))))
+URCU_ATTR_ALIAS(urcu_stringify(get_call_rcu_data))
 struct call_rcu_data *alias_get_call_rcu_data();
 
 /*
@@ -613,7 +629,7 @@ struct call_rcu_data *get_thread_call_rcu_data(void)
 {
 	return URCU_TLS(thread_call_rcu_data);
 }
-__attribute__((alias(urcu_stringify(get_thread_call_rcu_data))))
+URCU_ATTR_ALIAS(urcu_stringify(get_thread_call_rcu_data))
 struct call_rcu_data *alias_get_thread_call_rcu_data();
 
 /*
@@ -631,7 +647,7 @@ void set_thread_call_rcu_data(struct call_rcu_data *crdp)
 {
 	URCU_TLS(thread_call_rcu_data) = crdp;
 }
-__attribute__((alias(urcu_stringify(set_thread_call_rcu_data))))
+URCU_ATTR_ALIAS(urcu_stringify(set_thread_call_rcu_data))
 void alias_set_thread_call_rcu_data();
 
 /*
@@ -651,7 +667,7 @@ int create_all_cpu_call_rcu_data(unsigned long flags)
 	call_rcu_lock(&call_rcu_mutex);
 	alloc_cpu_call_rcu_data();
 	call_rcu_unlock(&call_rcu_mutex);
-	if (maxcpus <= 0) {
+	if (cpus_array_len <= 0) {
 		errno = EINVAL;
 		return -EINVAL;
 	}
@@ -659,7 +675,7 @@ int create_all_cpu_call_rcu_data(unsigned long flags)
 		errno = ENOMEM;
 		return -ENOMEM;
 	}
-	for (i = 0; i < maxcpus; i++) {
+	for (i = 0; i < cpus_array_len; i++) {
 		call_rcu_lock(&call_rcu_mutex);
 		if (get_cpu_call_rcu_data(i)) {
 			call_rcu_unlock(&call_rcu_mutex);
@@ -684,7 +700,7 @@ int create_all_cpu_call_rcu_data(unsigned long flags)
 	}
 	return 0;
 }
-__attribute__((alias(urcu_stringify(create_all_cpu_call_rcu_data))))
+URCU_ATTR_ALIAS(urcu_stringify(create_all_cpu_call_rcu_data))
 int alias_create_all_cpu_call_rcu_data();
 
 /*
@@ -733,7 +749,7 @@ void call_rcu(struct rcu_head *head,
 	_call_rcu(head, func, crdp);
 	_rcu_read_unlock();
 }
-__attribute__((alias(urcu_stringify(call_rcu)))) void alias_call_rcu();
+URCU_ATTR_ALIAS(urcu_stringify(call_rcu)) void alias_call_rcu();
 
 /*
  * Free up the specified call_rcu_data structure, terminating the
@@ -772,9 +788,13 @@ void call_rcu_data_free(struct call_rcu_data *crdp)
 		while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOPPED) == 0)
 			(void) poll(NULL, 0, 1);
 	}
+	call_rcu_lock(&call_rcu_mutex);
 	if (!cds_wfcq_empty(&crdp->cbs_head, &crdp->cbs_tail)) {
-		/* Create default call rcu data if need be */
+		call_rcu_unlock(&call_rcu_mutex);
+		/* Create default call rcu data if need be. */
+		/* CBs queued here will be handed to the default list. */
 		(void) get_default_call_rcu_data();
+		call_rcu_lock(&call_rcu_mutex);
 		__cds_wfcq_splice_blocking(&default_call_rcu_data->cbs_head,
 			&default_call_rcu_data->cbs_tail,
 			&crdp->cbs_head, &crdp->cbs_tail);
@@ -783,13 +803,12 @@ void call_rcu_data_free(struct call_rcu_data *crdp)
 		wake_call_rcu_thread(default_call_rcu_data);
 	}
 
-	call_rcu_lock(&call_rcu_mutex);
 	cds_list_del(&crdp->list);
 	call_rcu_unlock(&call_rcu_mutex);
 
 	free(crdp);
 }
-__attribute__((alias(urcu_stringify(call_rcu_data_free))))
+URCU_ATTR_ALIAS(urcu_stringify(call_rcu_data_free))
 void alias_call_rcu_data_free();
 
 /*
@@ -801,10 +820,10 @@ void free_all_cpu_call_rcu_data(void)
 	struct call_rcu_data **crdp;
 	static int warned = 0;
 
-	if (maxcpus <= 0)
+	if (cpus_array_len <= 0)
 		return;
 
-	crdp = malloc(sizeof(*crdp) * maxcpus);
+	crdp = malloc(sizeof(*crdp) * cpus_array_len);
 	if (!crdp) {
 		if (!warned) {
 			fprintf(stderr, "[error] liburcu: unable to allocate per-CPU pointer array\n");
@@ -813,7 +832,7 @@ void free_all_cpu_call_rcu_data(void)
 		return;
 	}
 
-	for (cpu = 0; cpu < maxcpus; cpu++) {
+	for (cpu = 0; cpu < cpus_array_len; cpu++) {
 		crdp[cpu] = get_cpu_call_rcu_data(cpu);
 		if (crdp[cpu] == NULL)
 			continue;
@@ -824,7 +843,7 @@ void free_all_cpu_call_rcu_data(void)
 	 * call_rcu_data to become quiescent.
 	 */
 	synchronize_rcu();
-	for (cpu = 0; cpu < maxcpus; cpu++) {
+	for (cpu = 0; cpu < cpus_array_len; cpu++) {
 		if (crdp[cpu] == NULL)
 			continue;
 		call_rcu_data_free(crdp[cpu]);
@@ -834,11 +853,11 @@ void free_all_cpu_call_rcu_data(void)
 #ifdef RCU_QSBR
 /* ABI6 has a non-namespaced free_all_cpu_call_rcu_data for qsbr */
 #undef free_all_cpu_call_rcu_data
-__attribute__((alias("urcu_qsbr_free_all_cpu_call_rcu_data")))
+URCU_ATTR_ALIAS("urcu_qsbr_free_all_cpu_call_rcu_data")
 void free_all_cpu_call_rcu_data();
 #define free_all_cpu_call_rcu_data urcu_qsbr_free_all_cpu_call_rcu_data
 #else
-__attribute__((alias(urcu_stringify(free_all_cpu_call_rcu_data))))
+URCU_ATTR_ALIAS(urcu_stringify(free_all_cpu_call_rcu_data))
 void alias_free_all_cpu_call_rcu_data();
 #endif
 
@@ -932,7 +951,7 @@ online:
 	if (was_online)
 		rcu_thread_online();
 }
-__attribute__((alias(urcu_stringify(rcu_barrier))))
+URCU_ATTR_ALIAS(urcu_stringify(rcu_barrier))
 void alias_rcu_barrier();
 
 /*
@@ -962,7 +981,7 @@ void call_rcu_before_fork(void)
 			(void) poll(NULL, 0, 1);
 	}
 }
-__attribute__((alias(urcu_stringify(call_rcu_before_fork))))
+URCU_ATTR_ALIAS(urcu_stringify(call_rcu_before_fork))
 void alias_call_rcu_before_fork();
 
 /*
@@ -986,7 +1005,7 @@ void call_rcu_after_fork_parent(void)
 		atfork->after_fork_parent(atfork->priv);
 	call_rcu_unlock(&call_rcu_mutex);
 }
-__attribute__((alias(urcu_stringify(call_rcu_after_fork_parent))))
+URCU_ATTR_ALIAS(urcu_stringify(call_rcu_after_fork_parent))
 void alias_call_rcu_after_fork_parent();
 
 /*
@@ -1018,7 +1037,7 @@ void call_rcu_after_fork_child(void)
 	(void)get_default_call_rcu_data();
 
 	/* Cleanup call_rcu_data pointers before use */
-	maxcpus_reset();
+	cpus_array_len_reset();
 	free(per_cpu_call_rcu_data);
 	rcu_set_pointer(&per_cpu_call_rcu_data, NULL);
 	URCU_TLS(thread_call_rcu_data) = NULL;
@@ -1035,29 +1054,30 @@ void call_rcu_after_fork_child(void)
 		call_rcu_data_free(crdp);
 	}
 }
-__attribute__((alias(urcu_stringify(call_rcu_after_fork_child))))
+URCU_ATTR_ALIAS(urcu_stringify(call_rcu_after_fork_child))
 void alias_call_rcu_after_fork_child();
 
 void urcu_register_rculfhash_atfork(struct urcu_atfork *atfork)
 {
+	if (CMM_LOAD_SHARED(registered_rculfhash_atfork))
+		return;
 	call_rcu_lock(&call_rcu_mutex);
-	if (registered_rculfhash_atfork_refcount++)
-		goto end;
-	registered_rculfhash_atfork = atfork;
-end:
+	if (!registered_rculfhash_atfork)
+		registered_rculfhash_atfork = atfork;
 	call_rcu_unlock(&call_rcu_mutex);
 }
-__attribute__((alias(urcu_stringify(urcu_register_rculfhash_atfork))))
+URCU_ATTR_ALIAS(urcu_stringify(urcu_register_rculfhash_atfork))
 void alias_urcu_register_rculfhash_atfork();
 
-void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork)
+/*
+ * This unregistration function is deprecated, meant only for internal
+ * use by rculfhash.
+ */
+__attribute__((noreturn))
+void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork __attribute__((unused)))
 {
-	call_rcu_lock(&call_rcu_mutex);
-	if (--registered_rculfhash_atfork_refcount)
-		goto end;
-	registered_rculfhash_atfork = NULL;
-end:
-	call_rcu_unlock(&call_rcu_mutex);
+	urcu_die(EPERM);
 }
-__attribute__((alias(urcu_stringify(urcu_unregister_rculfhash_atfork))))
+URCU_ATTR_ALIAS(urcu_stringify(urcu_unregister_rculfhash_atfork))
+__attribute__((noreturn))
 void alias_urcu_unregister_rculfhash_atfork();