[urcu.git] / include / urcu / uatomic / generic.h

// SPDX-FileCopyrightText: 1991-1994 by Xerox Corporation.  All rights reserved.
// SPDX-FileCopyrightText: 1996-1999 by Silicon Graphics.  All rights reserved.
// SPDX-FileCopyrightText: 1999-2004 Hewlett-Packard Development Company, L.P.
// SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
// SPDX-FileCopyrightText: 2010 Paolo Bonzini
//
// SPDX-License-Identifier: LicenseRef-Boehm-GC

#ifndef _URCU_UATOMIC_GENERIC_H
#define _URCU_UATOMIC_GENERIC_H

/*
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <stdint.h>
#include <urcu/compiler.h>
#include <urcu/system.h>

#ifdef __cplusplus
extern "C" {
#endif

#ifndef uatomic_set
#define uatomic_set(addr, v)	((void) CMM_STORE_SHARED(*(addr), (v)))
#endif

/*
 * Can be defined for the architecture.
 *
 * What needs to be emitted _before_ the `operation' with memory ordering `mo'.
 */
#ifndef _cmm_compat_c11_smp_mb__before_mo
# define _cmm_compat_c11_smp_mb__before_mo(operation, mo) cmm_smp_mb()
#endif

/*
 * Can be defined for the architecture.
 *
 * What needs to be emitted _after_ the `operation' with memory ordering `mo'.
 */
#ifndef _cmm_compat_c11_smp_mb__after_mo
# define _cmm_compat_c11_smp_mb__after_mo(operation, mo) cmm_smp_mb()
#endif

#define uatomic_load_store_return_op(op, addr, v, mo)		\
	__extension__						\
	({							\
		_cmm_compat_c11_smp_mb__before_mo(op, mo);	\
		__typeof__((*addr)) _value = op(addr, v);	\
		_cmm_compat_c11_smp_mb__after_mo(op, mo);	\
								\
		_value;						\
	})

#define uatomic_load_store_op(op, addr, v, mo)			\
	do {							\
		_cmm_compat_c11_smp_mb__before_mo(op, mo);	\
		op(addr, v);					\
		_cmm_compat_c11_smp_mb__after_mo(op, mo);	\
	} while (0)

#define uatomic_store(addr, v, mo)					\
	do {								\
		_cmm_compat_c11_smp_mb__before_mo(uatomic_set, mo);	\
		uatomic_set(addr, v);					\
		_cmm_compat_c11_smp_mb__after_mo(uatomic_set, mo);	\
	} while (0)

#define uatomic_and_mo(addr, v, mo)			\
	uatomic_load_store_op(uatomic_and, addr, v, mo)

#define uatomic_or_mo(addr, v, mo)			\
	uatomic_load_store_op(uatomic_or, addr, v, mo)

#define uatomic_add_mo(addr, v, mo)			\
	uatomic_load_store_op(uatomic_add, addr, v, mo)

#define uatomic_sub_mo(addr, v, mo)			\
	uatomic_load_store_op(uatomic_sub, addr, v, mo)

#define uatomic_inc_mo(addr, mo)			\
	uatomic_load_store_op(uatomic_add, addr, 1, mo)

#define uatomic_dec_mo(addr, mo)				\
	uatomic_load_store_op(uatomic_add, addr, -1, mo)
/*
 * NOTE: We can not just do switch (_value == (old) ? mos : mof) otherwise the
 * compiler emit a -Wduplicated-cond warning.
 */
#define uatomic_cmpxchg_mo(addr, old, new, mos, mof)			\
	__extension__							\
	({								\
		_cmm_compat_c11_smp_mb__before_mo(uatomic_cmpxchg, mos); \
		__typeof__(*(addr)) _value = uatomic_cmpxchg(addr, old,	\
							new);		\
									\
		if (_value == (old)) {					\
			_cmm_compat_c11_smp_mb__after_mo(uatomic_cmpxchg, mos);	\
		} else {						\
			_cmm_compat_c11_smp_mb__after_mo(uatomic_cmpxchg, mof);	\
		}							\
		_value;							\
	})

#define uatomic_xchg_mo(addr, v, mo)				\
	uatomic_load_store_return_op(uatomic_xchg, addr, v, mo)

#define uatomic_add_return_mo(addr, v, mo)				\
	uatomic_load_store_return_op(uatomic_add_return, addr, v)

#define uatomic_sub_return_mo(addr, v, mo)				\
	uatomic_load_store_return_op(uatomic_sub_return, addr, v)

#ifndef uatomic_read
#define uatomic_read(addr)	CMM_LOAD_SHARED(*(addr))
#endif

#define uatomic_load(addr, mo)						\
	__extension__							\
	({								\
		_cmm_compat_c11_smp_mb__before_mo(uatomic_read, mo);	\
		__typeof__(*(addr)) _rcu_value = uatomic_read(addr);	\
		_cmm_compat_c11_smp_mb__after_mo(uatomic_read, mo);	\
									\
		_rcu_value;						\
	})

#if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR
#ifdef ILLEGAL_INSTR
static inline __attribute__((always_inline))
void _uatomic_link_error(void)
{
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__(ILLEGAL_INSTR);
}
#else
static inline __attribute__((always_inline, __noreturn__))
void _uatomic_link_error(void)
{
	__builtin_trap();
}
#endif

#else /* #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */
extern void _uatomic_link_error(void);
#endif /* #else #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */

/* cmpxchg */

#ifndef uatomic_cmpxchg
static inline __attribute__((always_inline))
unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
				_new);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
				_new);
#endif
	case 4:
		return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
				_new);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
				_new);
#endif
	}
	_uatomic_link_error();
	return 0;
}


#define uatomic_cmpxchg(addr, old, _new)				      \
	((__typeof__(*(addr))) _uatomic_cmpxchg((addr),			      \
						caa_cast_long_keep_sign(old), \
						caa_cast_long_keep_sign(_new),\
						sizeof(*(addr))))


/* uatomic_and */

#ifndef uatomic_and
static inline __attribute__((always_inline))
void _uatomic_and(void *addr, unsigned long val,
		  int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		__sync_and_and_fetch_1((uint8_t *) addr, val);
		return;
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		__sync_and_and_fetch_2((uint16_t *) addr, val);
		return;
#endif
	case 4:
		__sync_and_and_fetch_4((uint32_t *) addr, val);
		return;
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		__sync_and_and_fetch_8((uint64_t *) addr, val);
		return;
#endif
	}
	_uatomic_link_error();
}

#define uatomic_and(addr, v)			\
	(_uatomic_and((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_and()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_and()		cmm_barrier()

#endif

/* uatomic_or */

#ifndef uatomic_or
static inline __attribute__((always_inline))
void _uatomic_or(void *addr, unsigned long val,
		 int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		__sync_or_and_fetch_1((uint8_t *) addr, val);
		return;
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		__sync_or_and_fetch_2((uint16_t *) addr, val);
		return;
#endif
	case 4:
		__sync_or_and_fetch_4((uint32_t *) addr, val);
		return;
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		__sync_or_and_fetch_8((uint64_t *) addr, val);
		return;
#endif
	}
	_uatomic_link_error();
	return;
}

#define uatomic_or(addr, v)			\
	(_uatomic_or((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_or()		cmm_barrier()
#define cmm_smp_mb__after_uatomic_or()		cmm_barrier()

#endif


/* uatomic_add_return */

#ifndef uatomic_add_return
static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		return __sync_add_and_fetch_1((uint8_t *) addr, val);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		return __sync_add_and_fetch_2((uint16_t *) addr, val);
#endif
	case 4:
		return __sync_add_and_fetch_4((uint32_t *) addr, val);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		return __sync_add_and_fetch_8((uint64_t *) addr, val);
#endif
	}
	_uatomic_link_error();
	return 0;
}


#define uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) _uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_add_return */

#ifndef uatomic_xchg
/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old;

		do {
			old = uatomic_read((uint8_t *) addr);
		} while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
				old, val));

		return old;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old;

		do {
			old = uatomic_read((uint16_t *) addr);
		} while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
				old, val));

		return old;
	}
#endif
	case 4:
	{
		uint32_t old;

		do {
			old = uatomic_read((uint32_t *) addr);
		} while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
				old, val));

		return old;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old;

		do {
			old = uatomic_read((uint64_t *) addr);
		} while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
				old, val));

		return old;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_xchg */

#else /* #ifndef uatomic_cmpxchg */

#ifndef uatomic_and
/* uatomic_and */

static inline __attribute__((always_inline))
void _uatomic_and(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
		} while (oldt != old);

		return;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
		} while (oldt != old);
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
		} while (oldt != old);

		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
		} while (oldt != old);

		return;
	}
#endif
	}
	_uatomic_link_error();
}

#define uatomic_and(addr, v)			\
	(_uatomic_and((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_and()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_and()		cmm_barrier()

#endif /* #ifndef uatomic_and */

#ifndef uatomic_or
/* uatomic_or */

static inline __attribute__((always_inline))
void _uatomic_or(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
		} while (oldt != old);

		return;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
		} while (oldt != old);

		return;
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
		} while (oldt != old);

		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
		} while (oldt != old);

		return;
	}
#endif
	}
	_uatomic_link_error();
}

#define uatomic_or(addr, v)			\
	(_uatomic_or((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_or()		cmm_barrier()
#define cmm_smp_mb__after_uatomic_or()		cmm_barrier()

#endif /* #ifndef uatomic_or */

#ifndef uatomic_add_return
/* uatomic_add_return */

static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint8_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint16_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint32_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint64_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) _uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_add_return */

#ifndef uatomic_xchg
/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint8_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint16_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint32_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint64_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_xchg */

#endif /* #else #ifndef uatomic_cmpxchg */

/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */

#ifndef uatomic_add
#define uatomic_add(addr, v)		(void)uatomic_add_return((addr), (v))
#define cmm_smp_mb__before_uatomic_add()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_add()		cmm_barrier()
#endif

#define uatomic_sub_return(addr, v)	\
	uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
#define uatomic_sub(addr, v)		\
	uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
#define cmm_smp_mb__before_uatomic_sub()	cmm_smp_mb__before_uatomic_add()
#define cmm_smp_mb__after_uatomic_sub()		cmm_smp_mb__after_uatomic_add()

#ifndef uatomic_inc
#define uatomic_inc(addr)		uatomic_add((addr), 1)
#define cmm_smp_mb__before_uatomic_inc()	cmm_smp_mb__before_uatomic_add()
#define cmm_smp_mb__after_uatomic_inc()		cmm_smp_mb__after_uatomic_add()
#endif

#ifndef uatomic_dec
#define uatomic_dec(addr)		uatomic_add((addr), -1)
#define cmm_smp_mb__before_uatomic_dec()	cmm_smp_mb__before_uatomic_add()
#define cmm_smp_mb__after_uatomic_dec()		cmm_smp_mb__after_uatomic_add()
#endif

#ifdef __cplusplus
}
#endif

#endif /* _URCU_UATOMIC_GENERIC_H */
Commit	Line	Data
	1	// SPDX-FileCopyrightText: 1991-1994 by Xerox Corporation. All rights reserved.
	2	// SPDX-FileCopyrightText: 1996-1999 by Silicon Graphics. All rights reserved.
	3	// SPDX-FileCopyrightText: 1999-2004 Hewlett-Packard Development Company, L.P.
	4	// SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	5	// SPDX-FileCopyrightText: 2010 Paolo Bonzini
	6	//
	7	// SPDX-License-Identifier: LicenseRef-Boehm-GC
	8
	9	#ifndef _URCU_UATOMIC_GENERIC_H
	10	#define _URCU_UATOMIC_GENERIC_H
	11
	12	/*
	13	* Code inspired from libuatomic_ops-1.2, inherited in part from the
	14	* Boehm-Demers-Weiser conservative garbage collector.
	15	*/
	16
	17	#include <stdint.h>
	18	#include <urcu/compiler.h>
	19	#include <urcu/system.h>
	20
	21	#ifdef __cplusplus
	22	extern "C" {
	23	#endif
	24
	25	#ifndef uatomic_set
	26	#define uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
	27	#endif
	28
	29	/*
	30	* Can be defined for the architecture.
	31	*
	32	* What needs to be emitted _before_ the `operation' with memory ordering `mo'.
	33	*/
	34	#ifndef _cmm_compat_c11_smp_mb__before_mo
	35	# define _cmm_compat_c11_smp_mb__before_mo(operation, mo) cmm_smp_mb()
	36	#endif
	37
	38	/*
	39	* Can be defined for the architecture.
	40	*
	41	* What needs to be emitted _after_ the `operation' with memory ordering `mo'.
	42	*/
	43	#ifndef _cmm_compat_c11_smp_mb__after_mo
	44	# define _cmm_compat_c11_smp_mb__after_mo(operation, mo) cmm_smp_mb()
	45	#endif
	46
	47	#define uatomic_load_store_return_op(op, addr, v, mo) \
	48	__extension__ \
	49	({ \
	50	_cmm_compat_c11_smp_mb__before_mo(op, mo); \
	51	__typeof__((*addr)) _value = op(addr, v); \
	52	_cmm_compat_c11_smp_mb__after_mo(op, mo); \
	53	\
	54	_value; \
	55	})
	56
	57	#define uatomic_load_store_op(op, addr, v, mo) \
	58	do { \
	59	_cmm_compat_c11_smp_mb__before_mo(op, mo); \
	60	op(addr, v); \
	61	_cmm_compat_c11_smp_mb__after_mo(op, mo); \
	62	} while (0)
	63
	64	#define uatomic_store(addr, v, mo) \
	65	do { \
	66	_cmm_compat_c11_smp_mb__before_mo(uatomic_set, mo); \
	67	uatomic_set(addr, v); \
	68	_cmm_compat_c11_smp_mb__after_mo(uatomic_set, mo); \
	69	} while (0)
	70
	71	#define uatomic_and_mo(addr, v, mo) \
	72	uatomic_load_store_op(uatomic_and, addr, v, mo)
	73
	74	#define uatomic_or_mo(addr, v, mo) \
	75	uatomic_load_store_op(uatomic_or, addr, v, mo)
	76
	77	#define uatomic_add_mo(addr, v, mo) \
	78	uatomic_load_store_op(uatomic_add, addr, v, mo)
	79
	80	#define uatomic_sub_mo(addr, v, mo) \
	81	uatomic_load_store_op(uatomic_sub, addr, v, mo)
	82
	83	#define uatomic_inc_mo(addr, mo) \
	84	uatomic_load_store_op(uatomic_add, addr, 1, mo)
	85
	86	#define uatomic_dec_mo(addr, mo) \
	87	uatomic_load_store_op(uatomic_add, addr, -1, mo)
	88	/*
	89	* NOTE: We can not just do switch (_value == (old) ? mos : mof) otherwise the
	90	* compiler emit a -Wduplicated-cond warning.
	91	*/
	92	#define uatomic_cmpxchg_mo(addr, old, new, mos, mof) \
	93	__extension__ \
	94	({ \
	95	_cmm_compat_c11_smp_mb__before_mo(uatomic_cmpxchg, mos); \
	96	__typeof__(*(addr)) _value = uatomic_cmpxchg(addr, old, \
	97	new); \
	98	\
	99	if (_value == (old)) { \
	100	_cmm_compat_c11_smp_mb__after_mo(uatomic_cmpxchg, mos); \
	101	} else { \
	102	_cmm_compat_c11_smp_mb__after_mo(uatomic_cmpxchg, mof); \
	103	} \
	104	_value; \
	105	})
	106
	107	#define uatomic_xchg_mo(addr, v, mo) \
	108	uatomic_load_store_return_op(uatomic_xchg, addr, v, mo)
	109
	110	#define uatomic_add_return_mo(addr, v, mo) \
	111	uatomic_load_store_return_op(uatomic_add_return, addr, v)
	112
	113	#define uatomic_sub_return_mo(addr, v, mo) \
	114	uatomic_load_store_return_op(uatomic_sub_return, addr, v)
	115
	116	#ifndef uatomic_read
	117	#define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
	118	#endif
	119
	120	#define uatomic_load(addr, mo) \
	121	__extension__ \
	122	({ \
	123	_cmm_compat_c11_smp_mb__before_mo(uatomic_read, mo); \
	124	__typeof__(*(addr)) _rcu_value = uatomic_read(addr); \
	125	_cmm_compat_c11_smp_mb__after_mo(uatomic_read, mo); \
	126	\
	127	_rcu_value; \
	128	})
	129
	130	#if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR
	131	#ifdef ILLEGAL_INSTR
	132	static inline __attribute__((always_inline))
	133	void _uatomic_link_error(void)
	134	{
	135	/*
	136	* generate an illegal instruction. Cannot catch this with
	137	* linker tricks when optimizations are disabled.
	138	*/
	139	__asm__ __volatile__(ILLEGAL_INSTR);
	140	}
	141	#else
	142	static inline __attribute__((always_inline, __noreturn__))
	143	void _uatomic_link_error(void)
	144	{
	145	__builtin_trap();
	146	}
	147	#endif
	148
	149	#else /* #if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR */
	150	extern void _uatomic_link_error(void);
	151	#endif /* #else #if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR */
	152
	153	/* cmpxchg */
	154
	155	#ifndef uatomic_cmpxchg
	156	static inline __attribute__((always_inline))
	157	unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
	158	unsigned long _new, int len)
	159	{
	160	switch (len) {
	161	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	162	case 1:
	163	return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
	164	_new);
	165	#endif
	166	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	167	case 2:
	168	return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
	169	_new);
	170	#endif
	171	case 4:
	172	return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
	173	_new);
	174	#if (CAA_BITS_PER_LONG == 64)
	175	case 8:
	176	return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
	177	_new);
	178	#endif
	179	}
	180	_uatomic_link_error();
	181	return 0;
	182	}
	183
	184
	185	#define uatomic_cmpxchg(addr, old, _new) \
	186	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
	187	caa_cast_long_keep_sign(old), \
	188	caa_cast_long_keep_sign(_new),\
	189	sizeof(*(addr))))
	190
	191
	192	/* uatomic_and */
	193
	194	#ifndef uatomic_and
	195	static inline __attribute__((always_inline))
	196	void _uatomic_and(void *addr, unsigned long val,
	197	int len)
	198	{
	199	switch (len) {
	200	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	201	case 1:
	202	__sync_and_and_fetch_1((uint8_t *) addr, val);
	203	return;
	204	#endif
	205	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	206	case 2:
	207	__sync_and_and_fetch_2((uint16_t *) addr, val);
	208	return;
	209	#endif
	210	case 4:
	211	__sync_and_and_fetch_4((uint32_t *) addr, val);
	212	return;
	213	#if (CAA_BITS_PER_LONG == 64)
	214	case 8:
	215	__sync_and_and_fetch_8((uint64_t *) addr, val);
	216	return;
	217	#endif
	218	}
	219	_uatomic_link_error();
	220	}
	221
	222	#define uatomic_and(addr, v) \
	223	(_uatomic_and((addr), \
	224	caa_cast_long_keep_sign(v), \
	225	sizeof(*(addr))))
	226	#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
	227	#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
	228
	229	#endif
	230
	231	/* uatomic_or */
	232
	233	#ifndef uatomic_or
	234	static inline __attribute__((always_inline))
	235	void _uatomic_or(void *addr, unsigned long val,
	236	int len)
	237	{
	238	switch (len) {
	239	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	240	case 1:
	241	__sync_or_and_fetch_1((uint8_t *) addr, val);
	242	return;
	243	#endif
	244	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	245	case 2:
	246	__sync_or_and_fetch_2((uint16_t *) addr, val);
	247	return;
	248	#endif
	249	case 4:
	250	__sync_or_and_fetch_4((uint32_t *) addr, val);
	251	return;
	252	#if (CAA_BITS_PER_LONG == 64)
	253	case 8:
	254	__sync_or_and_fetch_8((uint64_t *) addr, val);
	255	return;
	256	#endif
	257	}
	258	_uatomic_link_error();
	259	return;
	260	}
	261
	262	#define uatomic_or(addr, v) \
	263	(_uatomic_or((addr), \
	264	caa_cast_long_keep_sign(v), \
	265	sizeof(*(addr))))
	266	#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
	267	#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
	268
	269	#endif
	270
	271
	272	/* uatomic_add_return */
	273
	274	#ifndef uatomic_add_return
	275	static inline __attribute__((always_inline))
	276	unsigned long _uatomic_add_return(void *addr, unsigned long val,
	277	int len)
	278	{
	279	switch (len) {
	280	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	281	case 1:
	282	return __sync_add_and_fetch_1((uint8_t *) addr, val);
	283	#endif
	284	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	285	case 2:
	286	return __sync_add_and_fetch_2((uint16_t *) addr, val);
	287	#endif
	288	case 4:
	289	return __sync_add_and_fetch_4((uint32_t *) addr, val);
	290	#if (CAA_BITS_PER_LONG == 64)
	291	case 8:
	292	return __sync_add_and_fetch_8((uint64_t *) addr, val);
	293	#endif
	294	}
	295	_uatomic_link_error();
	296	return 0;
	297	}
	298
	299
	300	#define uatomic_add_return(addr, v) \
	301	((__typeof__(*(addr))) _uatomic_add_return((addr), \
	302	caa_cast_long_keep_sign(v), \
	303	sizeof(*(addr))))
	304	#endif /* #ifndef uatomic_add_return */
	305
	306	#ifndef uatomic_xchg
	307	/* xchg */
	308
	309	static inline __attribute__((always_inline))
	310	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
	311	{
	312	switch (len) {
	313	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	314	case 1:
	315	{
	316	uint8_t old;
	317
	318	do {
	319	old = uatomic_read((uint8_t *) addr);
	320	} while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
	321	old, val));
	322
	323	return old;
	324	}
	325	#endif
	326	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	327	case 2:
	328	{
	329	uint16_t old;
	330
	331	do {
	332	old = uatomic_read((uint16_t *) addr);
	333	} while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
	334	old, val));
	335
	336	return old;
	337	}
	338	#endif
	339	case 4:
	340	{
	341	uint32_t old;
	342
	343	do {
	344	old = uatomic_read((uint32_t *) addr);
	345	} while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
	346	old, val));
	347
	348	return old;
	349	}
	350	#if (CAA_BITS_PER_LONG == 64)
	351	case 8:
	352	{
	353	uint64_t old;
	354
	355	do {
	356	old = uatomic_read((uint64_t *) addr);
	357	} while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
	358	old, val));
	359
	360	return old;
	361	}
	362	#endif
	363	}
	364	_uatomic_link_error();
	365	return 0;
	366	}
	367
	368	#define uatomic_xchg(addr, v) \
	369	((__typeof__(*(addr))) _uatomic_exchange((addr), \
	370	caa_cast_long_keep_sign(v), \
	371	sizeof(*(addr))))
	372	#endif /* #ifndef uatomic_xchg */
	373
	374	#else /* #ifndef uatomic_cmpxchg */
	375
	376	#ifndef uatomic_and
	377	/* uatomic_and */
	378
	379	static inline __attribute__((always_inline))
	380	void _uatomic_and(void *addr, unsigned long val, int len)
	381	{
	382	switch (len) {
	383	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	384	case 1:
	385	{
	386	uint8_t old, oldt;
	387
	388	oldt = uatomic_read((uint8_t *) addr);
	389	do {
	390	old = oldt;
	391	oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
	392	} while (oldt != old);
	393
	394	return;
	395	}
	396	#endif
	397	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	398	case 2:
	399	{
	400	uint16_t old, oldt;
	401
	402	oldt = uatomic_read((uint16_t *) addr);
	403	do {
	404	old = oldt;
	405	oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
	406	} while (oldt != old);
	407	}
	408	#endif
	409	case 4:
	410	{
	411	uint32_t old, oldt;
	412
	413	oldt = uatomic_read((uint32_t *) addr);
	414	do {
	415	old = oldt;
	416	oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
	417	} while (oldt != old);
	418
	419	return;
	420	}
	421	#if (CAA_BITS_PER_LONG == 64)
	422	case 8:
	423	{
	424	uint64_t old, oldt;
	425
	426	oldt = uatomic_read((uint64_t *) addr);
	427	do {
	428	old = oldt;
	429	oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
	430	} while (oldt != old);
	431
	432	return;
	433	}
	434	#endif
	435	}
	436	_uatomic_link_error();
	437	}
	438
	439	#define uatomic_and(addr, v) \
	440	(_uatomic_and((addr), \
	441	caa_cast_long_keep_sign(v), \
	442	sizeof(*(addr))))
	443	#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
	444	#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
	445
	446	#endif /* #ifndef uatomic_and */
	447
	448	#ifndef uatomic_or
	449	/* uatomic_or */
	450
	451	static inline __attribute__((always_inline))
	452	void _uatomic_or(void *addr, unsigned long val, int len)
	453	{
	454	switch (len) {
	455	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	456	case 1:
	457	{
	458	uint8_t old, oldt;
	459
	460	oldt = uatomic_read((uint8_t *) addr);
	461	do {
	462	old = oldt;
	463	oldt = _uatomic_cmpxchg(addr, old, old \| val, 1);
	464	} while (oldt != old);
	465
	466	return;
	467	}
	468	#endif
	469	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	470	case 2:
	471	{
	472	uint16_t old, oldt;
	473
	474	oldt = uatomic_read((uint16_t *) addr);
	475	do {
	476	old = oldt;
	477	oldt = _uatomic_cmpxchg(addr, old, old \| val, 2);
	478	} while (oldt != old);
	479
	480	return;
	481	}
	482	#endif
	483	case 4:
	484	{
	485	uint32_t old, oldt;
	486
	487	oldt = uatomic_read((uint32_t *) addr);
	488	do {
	489	old = oldt;
	490	oldt = _uatomic_cmpxchg(addr, old, old \| val, 4);
	491	} while (oldt != old);
	492
	493	return;
	494	}
	495	#if (CAA_BITS_PER_LONG == 64)
	496	case 8:
	497	{
	498	uint64_t old, oldt;
	499
	500	oldt = uatomic_read((uint64_t *) addr);
	501	do {
	502	old = oldt;
	503	oldt = _uatomic_cmpxchg(addr, old, old \| val, 8);
	504	} while (oldt != old);
	505
	506	return;
	507	}
	508	#endif
	509	}
	510	_uatomic_link_error();
	511	}
	512
	513	#define uatomic_or(addr, v) \
	514	(_uatomic_or((addr), \
	515	caa_cast_long_keep_sign(v), \
	516	sizeof(*(addr))))
	517	#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
	518	#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
	519
	520	#endif /* #ifndef uatomic_or */
	521
	522	#ifndef uatomic_add_return
	523	/* uatomic_add_return */
	524
	525	static inline __attribute__((always_inline))
	526	unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
	527	{
	528	switch (len) {
	529	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	530	case 1:
	531	{
	532	uint8_t old, oldt;
	533
	534	oldt = uatomic_read((uint8_t *) addr);
	535	do {
	536	old = oldt;
	537	oldt = uatomic_cmpxchg((uint8_t *) addr,
	538	old, old + val);
	539	} while (oldt != old);
	540
	541	return old + val;
	542	}
	543	#endif
	544	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	545	case 2:
	546	{
	547	uint16_t old, oldt;
	548
	549	oldt = uatomic_read((uint16_t *) addr);
	550	do {
	551	old = oldt;
	552	oldt = uatomic_cmpxchg((uint16_t *) addr,
	553	old, old + val);
	554	} while (oldt != old);
	555
	556	return old + val;
	557	}
	558	#endif
	559	case 4:
	560	{
	561	uint32_t old, oldt;
	562
	563	oldt = uatomic_read((uint32_t *) addr);
	564	do {
	565	old = oldt;
	566	oldt = uatomic_cmpxchg((uint32_t *) addr,
	567	old, old + val);
	568	} while (oldt != old);
	569
	570	return old + val;
	571	}
	572	#if (CAA_BITS_PER_LONG == 64)
	573	case 8:
	574	{
	575	uint64_t old, oldt;
	576
	577	oldt = uatomic_read((uint64_t *) addr);
	578	do {
	579	old = oldt;
	580	oldt = uatomic_cmpxchg((uint64_t *) addr,
	581	old, old + val);
	582	} while (oldt != old);
	583
	584	return old + val;
	585	}
	586	#endif
	587	}
	588	_uatomic_link_error();
	589	return 0;
	590	}
	591
	592	#define uatomic_add_return(addr, v) \
	593	((__typeof__(*(addr))) _uatomic_add_return((addr), \
	594	caa_cast_long_keep_sign(v), \
	595	sizeof(*(addr))))
	596	#endif /* #ifndef uatomic_add_return */
	597
	598	#ifndef uatomic_xchg
	599	/* xchg */
	600
	601	static inline __attribute__((always_inline))
	602	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
	603	{
	604	switch (len) {
	605	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	606	case 1:
	607	{
	608	uint8_t old, oldt;
	609
	610	oldt = uatomic_read((uint8_t *) addr);
	611	do {
	612	old = oldt;
	613	oldt = uatomic_cmpxchg((uint8_t *) addr,
	614	old, val);
	615	} while (oldt != old);
	616
	617	return old;
	618	}
	619	#endif
	620	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	621	case 2:
	622	{
	623	uint16_t old, oldt;
	624
	625	oldt = uatomic_read((uint16_t *) addr);
	626	do {
	627	old = oldt;
	628	oldt = uatomic_cmpxchg((uint16_t *) addr,
	629	old, val);
	630	} while (oldt != old);
	631
	632	return old;
	633	}
	634	#endif
	635	case 4:
	636	{
	637	uint32_t old, oldt;
	638
	639	oldt = uatomic_read((uint32_t *) addr);
	640	do {
	641	old = oldt;
	642	oldt = uatomic_cmpxchg((uint32_t *) addr,
	643	old, val);
	644	} while (oldt != old);
	645
	646	return old;
	647	}
	648	#if (CAA_BITS_PER_LONG == 64)
	649	case 8:
	650	{
	651	uint64_t old, oldt;
	652
	653	oldt = uatomic_read((uint64_t *) addr);
	654	do {
	655	old = oldt;
	656	oldt = uatomic_cmpxchg((uint64_t *) addr,
	657	old, val);
	658	} while (oldt != old);
	659
	660	return old;
	661	}
	662	#endif
	663	}
	664	_uatomic_link_error();
	665	return 0;
	666	}
	667
	668	#define uatomic_xchg(addr, v) \
	669	((__typeof__(*(addr))) _uatomic_exchange((addr), \
	670	caa_cast_long_keep_sign(v), \
	671	sizeof(*(addr))))
	672	#endif /* #ifndef uatomic_xchg */
	673
	674	#endif /* #else #ifndef uatomic_cmpxchg */
	675
	676	/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
	677
	678	#ifndef uatomic_add
	679	#define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
	680	#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
	681	#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
	682	#endif
	683
	684	#define uatomic_sub_return(addr, v) \
	685	uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
	686	#define uatomic_sub(addr, v) \
	687	uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
	688	#define cmm_smp_mb__before_uatomic_sub() cmm_smp_mb__before_uatomic_add()
	689	#define cmm_smp_mb__after_uatomic_sub() cmm_smp_mb__after_uatomic_add()
	690
	691	#ifndef uatomic_inc
	692	#define uatomic_inc(addr) uatomic_add((addr), 1)
	693	#define cmm_smp_mb__before_uatomic_inc() cmm_smp_mb__before_uatomic_add()
	694	#define cmm_smp_mb__after_uatomic_inc() cmm_smp_mb__after_uatomic_add()
	695	#endif
	696
	697	#ifndef uatomic_dec
	698	#define uatomic_dec(addr) uatomic_add((addr), -1)
	699	#define cmm_smp_mb__before_uatomic_dec() cmm_smp_mb__before_uatomic_add()
	700	#define cmm_smp_mb__after_uatomic_dec() cmm_smp_mb__after_uatomic_add()
	701	#endif
	702
	703	#ifdef __cplusplus
	704	}
	705	#endif
	706
	707	#endif /* _URCU_UATOMIC_GENERIC_H */