Fix: compat_futex.c: *uaddr should be read as volatile

[urcu.git] / urcu-call-rcu-impl.h

diff --git a/urcu-call-rcu-impl.h b/urcu-call-rcu-impl.h
index c2e175b7524982ab341f73865248404e8e303542..1d2065796088dc974604edd9eda3f19bb25b48bb 100644 (file)
--- a/urcu-call-rcu-impl.h
+++ b/urcu-call-rcu-impl.h
@@ -42,8 +42,12 @@
 #include "urcu/list.h"
 #include "urcu/futex.h"
 #include "urcu/tls-compat.h"
+#include "urcu/ref.h"
 #include "urcu-die.h"
 
+#define SET_AFFINITY_CHECK_PERIOD              (1U << 8)       /* 256 */
+#define SET_AFFINITY_CHECK_PERIOD_MASK         (SET_AFFINITY_CHECK_PERIOD - 1)
+
 /* Data structure that identifies a call_rcu thread. */
 
 struct call_rcu_data {
@@ -61,12 +65,14 @@ struct call_rcu_data {
        unsigned long qlen; /* maintained for debugging. */
        pthread_t tid;
        int cpu_affinity;
+       unsigned long gp_count;
        struct cds_list_head list;
 } __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
 
 struct call_rcu_completion {
        int barrier_count;
        int32_t futex;
+       struct urcu_ref ref;
 };
 
 struct call_rcu_completion_work {
@@ -201,22 +207,42 @@ static void call_rcu_unlock(pthread_mutex_t *pmp)
                urcu_die(ret);
 }
 
+/*
+ * Periodically retry setting CPU affinity if we migrate.
+ * Losing affinity can be caused by CPU hotunplug/hotplug, or by
+ * cpuset(7).
+ */
 #if HAVE_SCHED_SETAFFINITY
 static
 int set_thread_cpu_affinity(struct call_rcu_data *crdp)
 {
        cpu_set_t mask;
+       int ret;
 
        if (crdp->cpu_affinity < 0)
                return 0;
+       if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK)
+               return 0;
+       if (urcu_sched_getcpu() == crdp->cpu_affinity)
+               return 0;
 
        CPU_ZERO(&mask);
        CPU_SET(crdp->cpu_affinity, &mask);
 #if SCHED_SETAFFINITY_ARGS == 2
-       return sched_setaffinity(0, &mask);
+       ret = sched_setaffinity(0, &mask);
 #else
-       return sched_setaffinity(0, sizeof(mask), &mask);
+       ret = sched_setaffinity(0, sizeof(mask), &mask);
 #endif
+       /*
+        * EINVAL is fine: can be caused by hotunplugged CPUs, or by
+        * cpuset(7). This is why we should always retry if we detect
+        * migration.
+        */
+       if (ret && errno == EINVAL) {
+               ret = 0;
+               errno = 0;
+       }
+       return ret;
 }
 #else
 static
@@ -273,10 +299,8 @@ static void *call_rcu_thread(void *arg)
        unsigned long cbcount;
        struct call_rcu_data *crdp = (struct call_rcu_data *) arg;
        int rt = !!(uatomic_read(&crdp->flags) & URCU_CALL_RCU_RT);
-       int ret;
 
-       ret = set_thread_cpu_affinity(crdp);
-       if (ret)
+       if (set_thread_cpu_affinity(crdp))
                urcu_die(errno);
 
        /*
@@ -296,6 +320,9 @@ static void *call_rcu_thread(void *arg)
                struct cds_wfcq_node *cbs, *cbs_tmp_n;
                enum cds_wfcq_ret splice_ret;
 
+               if (set_thread_cpu_affinity(crdp))
+                       urcu_die(errno);
+
                if (uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSE) {
                        /*
                         * Pause requested. Become quiescent: remove
@@ -389,6 +416,7 @@ static void call_rcu_data_init(struct call_rcu_data **crdpp,
        crdp->flags = flags;
        cds_list_add(&crdp->list, &call_rcu_data_list);
        crdp->cpu_affinity = cpu_affinity;
+       crdp->gp_count = 0;
        cmm_smp_mb();  /* Structure initialized before pointer is planted. */
        *crdpp = crdp;
        ret = pthread_create(&crdp->tid, NULL, call_rcu_thread, crdp);
@@ -667,10 +695,10 @@ void call_rcu(struct rcu_head *head,
        struct call_rcu_data *crdp;
 
        /* Holding rcu read-side lock across use of per-cpu crdp */
-       rcu_read_lock();
+       _rcu_read_lock();
        crdp = get_call_rcu_data();
        _call_rcu(head, func, crdp);
-       rcu_read_unlock();
+       _rcu_read_unlock();
 }
 
 /*
@@ -768,6 +796,15 @@ void free_all_cpu_call_rcu_data(void)
        free(crdp);
 }
 
+static
+void free_completion(struct urcu_ref *ref)
+{
+       struct call_rcu_completion *completion;
+
+       completion = caa_container_of(ref, struct call_rcu_completion, ref);
+       free(completion);
+}
+
 static
 void _rcu_barrier_complete(struct rcu_head *head)
 {
@@ -776,8 +813,9 @@ void _rcu_barrier_complete(struct rcu_head *head)
 
        work = caa_container_of(head, struct call_rcu_completion_work, head);
        completion = work->completion;
-       uatomic_dec(&completion->barrier_count);
-       call_rcu_completion_wake_up(completion);
+       if (!uatomic_sub_return(&completion->barrier_count, 1))
+               call_rcu_completion_wake_up(completion);
+       urcu_ref_put(&completion->ref, free_completion);
        free(work);
 }
 
@@ -787,19 +825,19 @@ void _rcu_barrier_complete(struct rcu_head *head)
 void rcu_barrier(void)
 {
        struct call_rcu_data *crdp;
-       struct call_rcu_completion completion;
+       struct call_rcu_completion *completion;
        int count = 0;
        int was_online;
 
        /* Put in offline state in QSBR. */
-       was_online = rcu_read_ongoing();
+       was_online = _rcu_read_ongoing();
        if (was_online)
                rcu_thread_offline();
        /*
         * Calling a rcu_barrier() within a RCU read-side critical
         * section is an error.
         */
-       if (rcu_read_ongoing()) {
+       if (_rcu_read_ongoing()) {
                static int warned = 0;
 
                if (!warned) {
@@ -809,11 +847,17 @@ void rcu_barrier(void)
                goto online;
        }
 
+       completion = calloc(sizeof(*completion), 1);
+       if (!completion)
+               urcu_die(errno);
+
        call_rcu_lock(&call_rcu_mutex);
        cds_list_for_each_entry(crdp, &call_rcu_data_list, list)
                count++;
 
-       completion.barrier_count = count;
+       /* Referenced by rcu_barrier() and each call_rcu thread. */
+       urcu_ref_set(&completion->ref, count + 1);
+       completion->barrier_count = count;
 
        cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
                struct call_rcu_completion_work *work;
@@ -821,20 +865,23 @@ void rcu_barrier(void)
                work = calloc(sizeof(*work), 1);
                if (!work)
                        urcu_die(errno);
-               work->completion = &completion;
+               work->completion = completion;
                _call_rcu(&work->head, _rcu_barrier_complete, crdp);
        }
        call_rcu_unlock(&call_rcu_mutex);
 
        /* Wait for them */
        for (;;) {
-               uatomic_dec(&completion.futex);
+               uatomic_dec(&completion->futex);
                /* Decrement futex before reading barrier_count */
                cmm_smp_mb();
-               if (!uatomic_read(&completion.barrier_count))
+               if (!uatomic_read(&completion->barrier_count))
                        break;
-               call_rcu_completion_wait(&completion);
+               call_rcu_completion_wait(completion);
        }
+
+       urcu_ref_put(&completion->ref, free_completion);
+
 online:
        if (was_online)
                rcu_thread_online();