Test fix: 0.6 branch does not have tls-compat.h

[userspace-rcu.git] / urcu-call-rcu-impl.h

diff --git a/urcu-call-rcu-impl.h b/urcu-call-rcu-impl.h
index 182e9b15bd96552ed5e4fb8125bee02da04a8715..6ea7b9a05d74bdfb0c2760d426306d663cb0ade8 100644 (file)
--- a/urcu-call-rcu-impl.h
+++ b/urcu-call-rcu-impl.h
@@ -64,8 +64,9 @@ CDS_LIST_HEAD(call_rcu_data_list);
 
 static __thread struct call_rcu_data *thread_call_rcu_data;
 
-/* Guard call_rcu thread creation. */
-
+/*
+ * Guard call_rcu thread creation and atfork handlers.
+ */
 static pthread_mutex_t call_rcu_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 /* If a given thread does not have its own call_rcu thread, this is default. */
@@ -204,7 +205,7 @@ static void call_rcu_wake_up(struct call_rcu_data *crdp)
 {
        /* Write to call_rcu list before reading/writing futex */
        cmm_smp_mb();
-       if (unlikely(uatomic_read(&crdp->futex) == -1)) {
+       if (caa_unlikely(uatomic_read(&crdp->futex) == -1)) {
                uatomic_set(&crdp->futex, 0);
                futex_async(&crdp->futex, FUTEX_WAKE, 1,
                      NULL, NULL, 0);
@@ -239,6 +240,21 @@ static void *call_rcu_thread(void *arg)
                cmm_smp_mb();
        }
        for (;;) {
+               if (uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSE) {
+                       /*
+                        * Pause requested. Become quiescent: remove
+                        * ourself from all global lists, and don't
+                        * process any callback. The callback lists may
+                        * still be non-empty though.
+                        */
+                       rcu_unregister_thread();
+                       cmm_smp_mb__before_uatomic_or();
+                       uatomic_or(&crdp->flags, URCU_CALL_RCU_PAUSED);
+                       while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSE) != 0)
+                               poll(NULL, 0, 1);
+                       rcu_register_thread();
+               }
+
                if (&crdp->cbs.head != _CMM_LOAD_SHARED(crdp->cbs.tail)) {
                        while ((cbs = _CMM_LOAD_SHARED(crdp->cbs.head)) == NULL)
                                poll(NULL, 0, 1);
@@ -645,8 +661,9 @@ void call_rcu_data_free(struct call_rcu_data *crdp)
                /* Create default call rcu data if need be */
                (void) get_default_call_rcu_data();
                cbs_endprev = (struct cds_wfq_node **)
-                       uatomic_xchg(&default_call_rcu_data, cbs_tail);
-               *cbs_endprev = cbs;
+                       uatomic_xchg(&default_call_rcu_data->cbs.tail,
+                                       cbs_tail);
+               _CMM_STORE_SHARED(*cbs_endprev, cbs);
                uatomic_add(&default_call_rcu_data->qlen,
                            uatomic_read(&crdp->qlen));
                wake_call_rcu_thread(default_call_rcu_data);
@@ -701,12 +718,25 @@ void free_all_cpu_call_rcu_data(void)
 
 /*
  * Acquire the call_rcu_mutex in order to ensure that the child sees
- * all of the call_rcu() data structures in a consistent state.
+ * all of the call_rcu() data structures in a consistent state. Ensure
+ * that all call_rcu threads are in a quiescent state across fork.
  * Suitable for pthread_atfork() and friends.
  */
 void call_rcu_before_fork(void)
 {
+       struct call_rcu_data *crdp;
+
        call_rcu_lock(&call_rcu_mutex);
+
+       cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
+               uatomic_or(&crdp->flags, URCU_CALL_RCU_PAUSE);
+               cmm_smp_mb__after_uatomic_or();
+               wake_call_rcu_thread(crdp);
+       }
+       cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
+               while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSED) == 0)
+                       poll(NULL, 0, 1);
+       }
 }
 
 /*
@@ -716,6 +746,10 @@ void call_rcu_before_fork(void)
  */
 void call_rcu_after_fork_parent(void)
 {
+       struct call_rcu_data *crdp;
+
+       cds_list_for_each_entry(crdp, &call_rcu_data_list, list)
+               uatomic_and(&crdp->flags, ~URCU_CALL_RCU_PAUSE);
        call_rcu_unlock(&call_rcu_mutex);
 }
 
@@ -748,7 +782,11 @@ void call_rcu_after_fork_child(void)
        rcu_set_pointer(&per_cpu_call_rcu_data, NULL);
        thread_call_rcu_data = NULL;
 
-       /* Dispose of all of the rest of the call_rcu_data structures. */
+       /*
+        * Dispose of all of the rest of the call_rcu_data structures.
+        * Leftover call_rcu callbacks will be merged into the new
+        * default call_rcu thread queue.
+        */
        cds_list_for_each_entry_safe(crdp, next, &call_rcu_data_list, list) {
                if (crdp == default_call_rcu_data)
                        continue;