test: fork handling

[urcu.git] / urcu-call-rcu-impl.h

diff --git a/urcu-call-rcu-impl.h b/urcu-call-rcu-impl.h
index dca98e4ae073d45f93d8665235b715661f16c0ad..6580397b1ee42b04a8e48b5c29a76220ea39b6d0 100644 (file)
--- a/urcu-call-rcu-impl.h
+++ b/urcu-call-rcu-impl.h
@@ -48,15 +48,14 @@
 
 struct call_rcu_data {
        /*
-        * Align the tail on cache line size to eliminate false-sharing
-        * with head. Small note, however: the "qlen" field, kept for
-        * debugging, will cause false-sharing between enqueue and
-        * dequeue.
+        * We do not align head on a different cache-line than tail
+        * mainly because call_rcu callback-invocation threads use
+        * batching ("splice") to get an entire list of callbacks, which
+        * effectively empties the queue, and requires to touch the tail
+        * anyway.
         */
        struct cds_wfcq_tail cbs_tail;
-       /* Alignment on cache line size will add padding here */
-
-       struct cds_wfcq_head __attribute__((aligned(CAA_CACHE_LINE_SIZE))) cbs_head;
+       struct cds_wfcq_head cbs_head;
        unsigned long flags;
        int32_t futex;
        unsigned long qlen; /* maintained for debugging. */
@@ -70,7 +69,7 @@ struct call_rcu_data {
  * Protected by call_rcu_mutex.
  */
 
-CDS_LIST_HEAD(call_rcu_data_list);
+static CDS_LIST_HEAD(call_rcu_data_list);
 
 /* Link a thread using call_rcu() to its call_rcu thread. */
 
@@ -253,11 +252,14 @@ static void *call_rcu_thread(void *arg)
                struct cds_wfcq_head cbs_tmp_head;
                struct cds_wfcq_tail cbs_tmp_tail;
                struct cds_wfcq_node *cbs, *cbs_tmp_n;
+               enum cds_wfcq_ret splice_ret;
 
                cds_wfcq_init(&cbs_tmp_head, &cbs_tmp_tail);
-               __cds_wfcq_splice_blocking(&cbs_tmp_head, &cbs_tmp_tail,
-                       &crdp->cbs_head, &crdp->cbs_tail);
-               if (!cds_wfcq_empty(&cbs_tmp_head, &cbs_tmp_tail)) {
+               splice_ret = __cds_wfcq_splice_blocking(&cbs_tmp_head,
+                       &cbs_tmp_tail, &crdp->cbs_head, &crdp->cbs_tail);
+               assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK);
+               assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY);
+               if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) {
                        synchronize_rcu();
                        cbcount = 0;
                        __cds_wfcq_for_each_blocking_safe(&cbs_tmp_head,
@@ -627,6 +629,10 @@ void call_rcu(struct rcu_head *head,
  * The caller must wait for a grace-period to pass between return from
  * set_cpu_call_rcu_data() and call to call_rcu_data_free() passing the
  * previous call rcu data as argument.
+ *
+ * Note: introducing __cds_wfcq_splice_blocking() in this function fixed
+ * a list corruption bug in the 0.7.x series. The equivalent fix
+ * appeared in 0.6.8 for the stable-0.6 branch.
  */
 void call_rcu_data_free(struct call_rcu_data *crdp)
 {