Fix: update ax_pthread macro to handle newer clang

[urcu.git] / urcu-call-rcu-impl.h

diff --git a/urcu-call-rcu-impl.h b/urcu-call-rcu-impl.h
index 5cc02d97704514df5e45a7dd459744d7f79eb597..65f63ee51670ae73b9c8ead9869554e4f8f35a82 100644 (file)
--- a/urcu-call-rcu-impl.h
+++ b/urcu-call-rcu-impl.h
@@ -36,6 +36,7 @@
 #include <sched.h>
 
 #include "config.h"
+#include "compat-getcpu.h"
 #include "urcu/wfcqueue.h"
 #include "urcu-call-rcu.h"
 #include "urcu-pointer.h"
@@ -45,6 +46,9 @@
 #include "urcu/ref.h"
 #include "urcu-die.h"
 
+#define SET_AFFINITY_CHECK_PERIOD              (1U << 8)       /* 256 */
+#define SET_AFFINITY_CHECK_PERIOD_MASK         (SET_AFFINITY_CHECK_PERIOD - 1)
+
 /* Data structure that identifies a call_rcu thread. */
 
 struct call_rcu_data {
@@ -62,6 +66,7 @@ struct call_rcu_data {
        unsigned long qlen; /* maintained for debugging. */
        pthread_t tid;
        int cpu_affinity;
+       unsigned long gp_count;
        struct cds_list_head list;
 } __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
 
@@ -102,23 +107,7 @@ static struct call_rcu_data *default_call_rcu_data;
  * CPUs rather than only to specific threads.
  */
 
-#ifdef HAVE_SCHED_GETCPU
-
-static int urcu_sched_getcpu(void)
-{
-       return sched_getcpu();
-}
-
-#else /* #ifdef HAVE_SCHED_GETCPU */
-
-static int urcu_sched_getcpu(void)
-{
-       return -1;
-}
-
-#endif /* #else #ifdef HAVE_SCHED_GETCPU */
-
-#if defined(HAVE_SYSCONF) && defined(HAVE_SCHED_GETCPU)
+#if defined(HAVE_SYSCONF) && (defined(HAVE_SCHED_GETCPU) || defined(HAVE_GETCPUID))
 
 /*
  * Pointer to array of pointers to per-CPU call_rcu_data structures
@@ -203,22 +192,42 @@ static void call_rcu_unlock(pthread_mutex_t *pmp)
                urcu_die(ret);
 }
 
+/*
+ * Periodically retry setting CPU affinity if we migrate.
+ * Losing affinity can be caused by CPU hotunplug/hotplug, or by
+ * cpuset(7).
+ */
 #if HAVE_SCHED_SETAFFINITY
 static
 int set_thread_cpu_affinity(struct call_rcu_data *crdp)
 {
        cpu_set_t mask;
+       int ret;
 
        if (crdp->cpu_affinity < 0)
                return 0;
+       if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK)
+               return 0;
+       if (urcu_sched_getcpu() == crdp->cpu_affinity)
+               return 0;
 
        CPU_ZERO(&mask);
        CPU_SET(crdp->cpu_affinity, &mask);
 #if SCHED_SETAFFINITY_ARGS == 2
-       return sched_setaffinity(0, &mask);
+       ret = sched_setaffinity(0, &mask);
 #else
-       return sched_setaffinity(0, sizeof(mask), &mask);
+       ret = sched_setaffinity(0, sizeof(mask), &mask);
 #endif
+       /*
+        * EINVAL is fine: can be caused by hotunplugged CPUs, or by
+        * cpuset(7). This is why we should always retry if we detect
+        * migration.
+        */
+       if (ret && errno == EINVAL) {
+               ret = 0;
+               errno = 0;
+       }
+       return ret;
 }
 #else
 static
@@ -232,9 +241,22 @@ static void call_rcu_wait(struct call_rcu_data *crdp)
 {
        /* Read call_rcu list before read futex */
        cmm_smp_mb();
-       if (uatomic_read(&crdp->futex) == -1)
-               futex_async(&crdp->futex, FUTEX_WAIT, -1,
-                     NULL, NULL, 0);
+       if (uatomic_read(&crdp->futex) != -1)
+               return;
+       while (futex_async(&crdp->futex, FUTEX_WAIT, -1,
+                       NULL, NULL, 0)) {
+               switch (errno) {
+               case EWOULDBLOCK:
+                       /* Value already changed. */
+                       return;
+               case EINTR:
+                       /* Retry if interrupted by signal. */
+                       break;  /* Get out of switch. */
+               default:
+                       /* Unexpected error. */
+                       urcu_die(errno);
+               }
+       }
 }
 
 static void call_rcu_wake_up(struct call_rcu_data *crdp)
@@ -243,8 +265,9 @@ static void call_rcu_wake_up(struct call_rcu_data *crdp)
        cmm_smp_mb();
        if (caa_unlikely(uatomic_read(&crdp->futex) == -1)) {
                uatomic_set(&crdp->futex, 0);
-               futex_async(&crdp->futex, FUTEX_WAKE, 1,
-                     NULL, NULL, 0);
+               if (futex_async(&crdp->futex, FUTEX_WAKE, 1,
+                               NULL, NULL, 0) < 0)
+                       urcu_die(errno);
        }
 }
 
@@ -252,9 +275,22 @@ static void call_rcu_completion_wait(struct call_rcu_completion *completion)
 {
        /* Read completion barrier count before read futex */
        cmm_smp_mb();
-       if (uatomic_read(&completion->futex) == -1)
-               futex_async(&completion->futex, FUTEX_WAIT, -1,
-                     NULL, NULL, 0);
+       if (uatomic_read(&completion->futex) != -1)
+               return;
+       while (futex_async(&completion->futex, FUTEX_WAIT, -1,
+                       NULL, NULL, 0)) {
+               switch (errno) {
+               case EWOULDBLOCK:
+                       /* Value already changed. */
+                       return;
+               case EINTR:
+                       /* Retry if interrupted by signal. */
+                       break;  /* Get out of switch. */
+               default:
+                       /* Unexpected error. */
+                       urcu_die(errno);
+               }
+       }
 }
 
 static void call_rcu_completion_wake_up(struct call_rcu_completion *completion)
@@ -263,8 +299,9 @@ static void call_rcu_completion_wake_up(struct call_rcu_completion *completion)
        cmm_smp_mb();
        if (caa_unlikely(uatomic_read(&completion->futex) == -1)) {
                uatomic_set(&completion->futex, 0);
-               futex_async(&completion->futex, FUTEX_WAKE, 1,
-                     NULL, NULL, 0);
+               if (futex_async(&completion->futex, FUTEX_WAKE, 1,
+                               NULL, NULL, 0) < 0)
+                       urcu_die(errno);
        }
 }
 
@@ -275,10 +312,8 @@ static void *call_rcu_thread(void *arg)
        unsigned long cbcount;
        struct call_rcu_data *crdp = (struct call_rcu_data *) arg;
        int rt = !!(uatomic_read(&crdp->flags) & URCU_CALL_RCU_RT);
-       int ret;
 
-       ret = set_thread_cpu_affinity(crdp);
-       if (ret)
+       if (set_thread_cpu_affinity(crdp))
                urcu_die(errno);
 
        /*
@@ -298,6 +333,9 @@ static void *call_rcu_thread(void *arg)
                struct cds_wfcq_node *cbs, *cbs_tmp_n;
                enum cds_wfcq_ret splice_ret;
 
+               if (set_thread_cpu_affinity(crdp))
+                       urcu_die(errno);
+
                if (uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSE) {
                        /*
                         * Pause requested. Become quiescent: remove
@@ -391,6 +429,7 @@ static void call_rcu_data_init(struct call_rcu_data **crdpp,
        crdp->flags = flags;
        cds_list_add(&crdp->list, &call_rcu_data_list);
        crdp->cpu_affinity = cpu_affinity;
+       crdp->gp_count = 0;
        cmm_smp_mb();  /* Structure initialized before pointer is planted. */
        *crdpp = crdp;
        ret = pthread_create(&crdp->tid, NULL, call_rcu_thread, crdp);