Add missing rcu_cmpxchg_pointer define

[urcu.git] / urcu-defer.c

diff --git a/urcu-defer.c b/urcu-defer.c
index 3efe5ba2a5a47ccaff0a2f956efa957fb71bde01..ccf35116d2b73e63634730a475d09feeb650ae38 100644 (file)
--- a/urcu-defer.c
+++ b/urcu-defer.c
@@ -28,7 +28,6 @@
 #include <string.h>
 #include <errno.h>
 #include <poll.h>
-#include <linux/futex.h>
 #include <sys/time.h>
 #include <syscall.h>
 #include <unistd.h>
@@ -38,6 +37,8 @@
 #include "urcu-defer.h"
 
 #define futex(...)     syscall(__NR_futex, __VA_ARGS__)
+#define FUTEX_WAIT             0
+#define FUTEX_WAKE             1
 
 void __attribute__((destructor)) urcu_defer_exit(void);
 
@@ -71,29 +72,6 @@ static int num_deferers, alloc_deferers;
 
 static pthread_t tid_defer;
 
-/*
- * Wake-up any waiting defer thread. Called from many concurrent threads.
- */
-static void wake_up_defer(void)
-{
-       if (unlikely(atomic_read(&defer_thread_futex) == -1)) {
-               atomic_set(&defer_thread_futex, 0);
-               futex(&defer_thread_futex, FUTEX_WAKE, 0,
-                     NULL, NULL, 0);
-       }
-}
-
-/*
- * Defer thread waiting. Single thread.
- */
-static void wait_defer(void)
-{
-       atomic_dec(&defer_thread_futex);
-       if (atomic_read(&defer_thread_futex) == -1)
-               futex(&defer_thread_futex, FUTEX_WAIT, -1,
-                     NULL, NULL, 0);
-}
-
 static void internal_urcu_lock(pthread_mutex_t *mutex)
 {
        int ret;
@@ -128,6 +106,51 @@ static void internal_urcu_unlock(pthread_mutex_t *mutex)
        }
 }
 
+/*
+ * Wake-up any waiting defer thread. Called from many concurrent threads.
+ */
+static void wake_up_defer(void)
+{
+       if (unlikely(atomic_read(&defer_thread_futex) == -1)) {
+               atomic_set(&defer_thread_futex, 0);
+               futex(&defer_thread_futex, FUTEX_WAKE, 1,
+                     NULL, NULL, 0);
+       }
+}
+
+static unsigned long rcu_defer_num_callbacks(void)
+{
+       unsigned long num_items = 0, head;
+       struct deferer_registry *index;
+
+       internal_urcu_lock(&urcu_defer_mutex);
+       for (index = registry; index < registry + num_deferers; index++) {
+               head = LOAD_SHARED(index->defer_queue->head);
+               num_items += head - index->defer_queue->tail;
+       }
+       internal_urcu_unlock(&urcu_defer_mutex);
+       return num_items;
+}
+
+/*
+ * Defer thread waiting. Single thread.
+ */
+static void wait_defer(void)
+{
+       atomic_dec(&defer_thread_futex);
+       smp_mb();       /* Write futex before read queue */
+       if (rcu_defer_num_callbacks()) {
+               smp_mb();       /* Read queue before write futex */
+               /* Callbacks are queued, don't wait. */
+               atomic_set(&defer_thread_futex, 0);
+       } else {
+               smp_rmb();      /* Read queue before read futex */
+               if (atomic_read(&defer_thread_futex) == -1)
+                       futex(&defer_thread_futex, FUTEX_WAIT, -1,
+                             NULL, NULL, 0);
+       }
+}
+
 /*
  * Must be called after Q.S. is reached.
  */
@@ -280,6 +303,7 @@ void _rcu_defer_queue(void (*fct)(void *p), void *p)
        smp_wmb();      /* Publish new pointer before head */
                        /* Write q[] before head. */
        STORE_SHARED(defer_queue.head, head);
+       smp_mb();       /* Write queue head before read futex */
        /*
         * Wake-up any waiting defer thread.
         */