#include <stdlib.h>
#include <pthread.h>
#include <assert.h>
+#include <limits.h>
#include <compiler.h>
#include <arch.h>
smp_mb();
}
+#define RCU_GP_ONLINE (1UL << 0)
+#define RCU_GP_CTR (1UL << 1)
+
/*
* Global quiescent period counter with low-order bits unused.
* Using a int rather than a char to eliminate false register dependencies
* causing stalls on some architectures.
*/
-extern long urcu_gp_ctr;
+extern unsigned long urcu_gp_ctr;
-extern long __thread rcu_reader_qs_gp;
+extern unsigned long __thread rcu_reader_qs_gp;
-static inline int rcu_gp_ongoing(long *value)
+#if (BITS_PER_LONG < 64)
+static inline int rcu_gp_ongoing(unsigned long *value)
{
+ unsigned long reader_gp;
+
if (value == NULL)
return 0;
+ reader_gp = LOAD_SHARED(*value);
+ return reader_gp && ((reader_gp ^ urcu_gp_ctr) & RCU_GP_CTR);
+}
+#else /* !(BITS_PER_LONG < 64) */
+static inline int rcu_gp_ongoing(unsigned long *value)
+{
+ unsigned long reader_gp;
- return LOAD_SHARED(*value) & 1;
+ if (value == NULL)
+ return 0;
+ reader_gp = LOAD_SHARED(*value);
+ return reader_gp && (reader_gp - urcu_gp_ctr > ULONG_MAX / 2);
}
+#endif /* !(BITS_PER_LONG < 64) */
static inline void _rcu_read_lock(void)
{
- rcu_assert(rcu_reader_qs_gp & 1);
+ rcu_assert(rcu_reader_qs_gp);
}
static inline void _rcu_read_unlock(void)
static inline void _rcu_quiescent_state(void)
{
smp_mb();
- rcu_reader_qs_gp = ACCESS_ONCE(urcu_gp_ctr) + 1;
+ _STORE_SHARED(rcu_reader_qs_gp, _LOAD_SHARED(urcu_gp_ctr));
smp_mb();
}
static inline void _rcu_thread_offline(void)
{
smp_mb();
- rcu_reader_qs_gp = 0;
+ STORE_SHARED(rcu_reader_qs_gp, 0);
}
static inline void _rcu_thread_online(void)
{
- rcu_reader_qs_gp = ACCESS_ONCE(urcu_gp_ctr) + 1;
+ _STORE_SHARED(rcu_reader_qs_gp, LOAD_SHARED(urcu_gp_ctr));
smp_mb();
}
STORE_SHARED(p, v); \
})
+/**
+ * _rcu_cmpxchg_pointer - same as rcu_assign_pointer, but tests if the pointer
+ * is as expected by "old". If succeeds, returns the previous pointer to the
+ * data structure, which can be safely freed after waiting for a quiescent state
+ * using synchronize_rcu(). If fails (unexpected value), returns old (which
+ * should not be freed !).
+ */
+
+#define _rcu_cmpxchg_pointer(p, old, _new) \
+ ({ \
+ if (!__builtin_constant_p(_new) || \
+ ((_new) != NULL)) \
+ wmb(); \
+ cmpxchg(p, old, _new); \
+ })
+
/**
* _rcu_xchg_pointer - same as rcu_assign_pointer, but returns the previous
* pointer to the data structure, which can be safely freed after waiting for a