[urcu.git] / urcu / uatomic / generic.h

#ifndef _URCU_UATOMIC_GENERIC_H
#define _URCU_UATOMIC_GENERIC_H

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 * Copyright (c) 2010      Paolo Bonzini
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <stdint.h>
#include <urcu/compiler.h>
#include <urcu/system.h>

#ifdef __cplusplus
extern "C" {
#endif

#ifndef uatomic_set
#define uatomic_set(addr, v)	((void) CMM_STORE_SHARED(*(addr), (v)))
#endif

#ifndef uatomic_read
#define uatomic_read(addr)	CMM_LOAD_SHARED(*(addr))
#endif

#if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR
static inline __attribute__((always_inline))
void _uatomic_link_error(void)
{
#ifdef ILLEGAL_INSTR
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__(ILLEGAL_INSTR);
#else
	__builtin_trap();
#endif
}

#else /* #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */
extern void _uatomic_link_error(void);
#endif /* #else #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */

/* cmpxchg */

#ifndef uatomic_cmpxchg
static inline __attribute__((always_inline))
unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
				_new);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
				_new);
#endif
	case 4:
		return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
				_new);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
				_new);
#endif
	}
	_uatomic_link_error();
	return 0;
}


#define uatomic_cmpxchg(addr, old, _new)				      \
	((__typeof__(*(addr))) _uatomic_cmpxchg((addr),			      \
						caa_cast_long_keep_sign(old), \
						caa_cast_long_keep_sign(_new),\
						sizeof(*(addr))))


/* uatomic_and */

#ifndef uatomic_and
static inline __attribute__((always_inline))
void _uatomic_and(void *addr, unsigned long val,
		  int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		__sync_and_and_fetch_1((uint8_t *) addr, val);
		return;
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		__sync_and_and_fetch_2((uint16_t *) addr, val);
		return;
#endif
	case 4:
		__sync_and_and_fetch_4((uint32_t *) addr, val);
		return;
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		__sync_and_and_fetch_8((uint64_t *) addr, val);
		return;
#endif
	}
	_uatomic_link_error();
}

#define uatomic_and(addr, v)			\
	(_uatomic_and((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_and()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_and()		cmm_barrier()

#endif

/* uatomic_or */

#ifndef uatomic_or
static inline __attribute__((always_inline))
void _uatomic_or(void *addr, unsigned long val,
		 int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		__sync_or_and_fetch_1((uint8_t *) addr, val);
		return;
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		__sync_or_and_fetch_2((uint16_t *) addr, val);
		return;
#endif
	case 4:
		__sync_or_and_fetch_4((uint32_t *) addr, val);
		return;
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		__sync_or_and_fetch_8((uint64_t *) addr, val);
		return;
#endif
	}
	_uatomic_link_error();
	return;
}

#define uatomic_or(addr, v)			\
	(_uatomic_or((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_or()		cmm_barrier()
#define cmm_smp_mb__after_uatomic_or()		cmm_barrier()

#endif


/* uatomic_add_return */

#ifndef uatomic_add_return
static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		return __sync_add_and_fetch_1((uint8_t *) addr, val);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		return __sync_add_and_fetch_2((uint16_t *) addr, val);
#endif
	case 4:
		return __sync_add_and_fetch_4((uint32_t *) addr, val);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		return __sync_add_and_fetch_8((uint64_t *) addr, val);
#endif
	}
	_uatomic_link_error();
	return 0;
}


#define uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) _uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_add_return */

#ifndef uatomic_xchg
/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old;

		do {
			old = uatomic_read((uint8_t *) addr);
		} while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
				old, val));

		return old;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old;

		do {
			old = uatomic_read((uint16_t *) addr);
		} while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
				old, val));

		return old;
	}
#endif
	case 4:
	{
		uint32_t old;

		do {
			old = uatomic_read((uint32_t *) addr);
		} while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
				old, val));

		return old;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old;

		do {
			old = uatomic_read((uint64_t *) addr);
		} while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
				old, val));

		return old;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_xchg */

#else /* #ifndef uatomic_cmpxchg */

#ifndef uatomic_and
/* uatomic_and */

static inline __attribute__((always_inline))
void _uatomic_and(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
		} while (oldt != old);

		return;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
		} while (oldt != old);
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
		} while (oldt != old);

		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
		} while (oldt != old);

		return;
	}
#endif
	}
	_uatomic_link_error();
}

#define uatomic_and(addr, v)			\
	(_uatomic_and((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_and()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_and()		cmm_barrier()

#endif /* #ifndef uatomic_and */

#ifndef uatomic_or
/* uatomic_or */

static inline __attribute__((always_inline))
void _uatomic_or(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
		} while (oldt != old);

		return;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
		} while (oldt != old);

		return;
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
		} while (oldt != old);

		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
		} while (oldt != old);

		return;
	}
#endif
	}
	_uatomic_link_error();
}

#define uatomic_or(addr, v)			\
	(_uatomic_or((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_or()		cmm_barrier()
#define cmm_smp_mb__after_uatomic_or()		cmm_barrier()

#endif /* #ifndef uatomic_or */

#ifndef uatomic_add_return
/* uatomic_add_return */

static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint8_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint16_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint32_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint64_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) _uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_add_return */

#ifndef uatomic_xchg
/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint8_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint16_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint32_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint64_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_xchg */

#endif /* #else #ifndef uatomic_cmpxchg */

/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */

#ifndef uatomic_add
#define uatomic_add(addr, v)		(void)uatomic_add_return((addr), (v))
#define cmm_smp_mb__before_uatomic_add()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_add()		cmm_barrier()
#endif

#define uatomic_sub_return(addr, v)	\
	uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
#define uatomic_sub(addr, v)		\
	uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
#define cmm_smp_mb__before_uatomic_sub()	cmm_smp_mb__before_uatomic_add()
#define cmm_smp_mb__after_uatomic_sub()		cmm_smp_mb__after_uatomic_add()

#ifndef uatomic_inc
#define uatomic_inc(addr)		uatomic_add((addr), 1)
#define cmm_smp_mb__before_uatomic_inc()	cmm_smp_mb__before_uatomic_add()
#define cmm_smp_mb__after_uatomic_inc()		cmm_smp_mb__after_uatomic_add()
#endif

#ifndef uatomic_dec
#define uatomic_dec(addr)		uatomic_add((addr), -1)
#define cmm_smp_mb__before_uatomic_dec()	cmm_smp_mb__before_uatomic_add()
#define cmm_smp_mb__after_uatomic_dec()		cmm_smp_mb__after_uatomic_add()
#endif

#ifdef __cplusplus
}
#endif

#endif /* _URCU_UATOMIC_GENERIC_H */
Commit	Line	Data
8760d94e PB	1	#ifndef _URCU_UATOMIC_GENERIC_H
	2	#define _URCU_UATOMIC_GENERIC_H
	3
	4	/*
	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	* Copyright (c) 2010 Paolo Bonzini
	10	*
	11	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	12	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	13	*
	14	* Permission is hereby granted to use or copy this program
	15	* for any purpose, provided the above notices are retained on all copies.
	16	* Permission to modify the code and to distribute modified code is granted,
	17	* provided the above notices are retained, and a notice that the code was
	18	* modified is included with the above copyright notice.
	19	*
	20	* Code inspired from libuatomic_ops-1.2, inherited in part from the
	21	* Boehm-Demers-Weiser conservative garbage collector.
	22	*/
	23
2917c006	24	#include <stdint.h>
8760d94e PB	25	#include <urcu/compiler.h>
	26	#include <urcu/system.h>
	27
	28	#ifdef __cplusplus
	29	extern "C" {
	30	#endif
	31
8760d94e	32	#ifndef uatomic_set
3daae22a	33	#define uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
8760d94e PB	34	#endif
	35
	36	#ifndef uatomic_read
6cf3827c	37	#define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
8760d94e PB	38	#endif
	39
	40	#if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR
	41	static inline __attribute__((always_inline))
5c28497b	42	void _uatomic_link_error(void)
8760d94e PB	43	{
8760d94e PB	44	#ifdef ILLEGAL_INSTR
d0bbd9c2 MD	45	/*
	46	* generate an illegal instruction. Cannot catch this with
	47	* linker tricks when optimizations are disabled.
	48	*/
8760d94e PB	49	__asm__ __volatile__(ILLEGAL_INSTR);
8760d94e PB	50	#else
5c28497b	51	__builtin_trap();
8760d94e PB	52	#endif
	53	}
	54
	55	#else /* #if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR */
5c28497b	56	extern void _uatomic_link_error(void);
8760d94e PB	57	#endif /* #else #if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR */
	58
	59	/* cmpxchg */
	60
	61	#ifndef uatomic_cmpxchg
	62	static inline __attribute__((always_inline))
	63	unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
	64	unsigned long _new, int len)
	65	{
	66	switch (len) {
f469d839 PB	67	#ifdef UATOMIC_HAS_ATOMIC_BYTE
f469d839 PB	68	case 1:
2917c006 KR	69	return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
2917c006 KR	70	_new);
f469d839 PB	71	#endif
	72	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	73	case 2:
2917c006 KR	74	return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
2917c006 KR	75	_new);
f469d839	76	#endif
8760d94e	77	case 4:
2917c006 KR	78	return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
2917c006 KR	79	_new);
b39e1761	80	#if (CAA_BITS_PER_LONG == 64)
8760d94e	81	case 8:
2917c006 KR	82	return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
2917c006 KR	83	_new);
8760d94e PB	84	#endif
	85	}
	86	_uatomic_link_error();
	87	return 0;
	88	}
	89
	90
e56d99bf MD	91	#define uatomic_cmpxchg(addr, old, _new) \
	92	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
	93	caa_cast_long_keep_sign(old), \
	94	caa_cast_long_keep_sign(_new),\
8760d94e PB	95	sizeof(*(addr))))
	96
	97
bf33aaea PB	98	/* uatomic_and */
	99
	100	#ifndef uatomic_and
	101	static inline __attribute__((always_inline))
	102	void _uatomic_and(void *addr, unsigned long val,
	103	int len)
	104	{
	105	switch (len) {
	106	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	107	case 1:
2917c006	108	__sync_and_and_fetch_1((uint8_t *) addr, val);
c51d5c6e	109	return;
bf33aaea PB	110	#endif
	111	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	112	case 2:
2917c006	113	__sync_and_and_fetch_2((uint16_t *) addr, val);
c51d5c6e	114	return;
bf33aaea PB	115	#endif
bf33aaea PB	116	case 4:
2917c006	117	__sync_and_and_fetch_4((uint32_t *) addr, val);
c51d5c6e	118	return;
bf33aaea PB	119	#if (CAA_BITS_PER_LONG == 64)
bf33aaea PB	120	case 8:
2917c006	121	__sync_and_and_fetch_8((uint64_t *) addr, val);
c51d5c6e	122	return;
bf33aaea PB	123	#endif
	124	}
	125	_uatomic_link_error();
bf33aaea PB	126	}
	127
	128	#define uatomic_and(addr, v) \
	129	(_uatomic_and((addr), \
e56d99bf MD	130	caa_cast_long_keep_sign(v), \
e56d99bf MD	131	sizeof(*(addr))))
42e83919 MD	132	#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
42e83919 MD	133	#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
2812a2d2	134
bf33aaea PB	135	#endif
bf33aaea PB	136
985b35b1 PB	137	/* uatomic_or */
	138
	139	#ifndef uatomic_or
	140	static inline __attribute__((always_inline))
	141	void _uatomic_or(void *addr, unsigned long val,
	142	int len)
	143	{
	144	switch (len) {
	145	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	146	case 1:
2917c006	147	__sync_or_and_fetch_1((uint8_t *) addr, val);
c51d5c6e	148	return;
985b35b1 PB	149	#endif
	150	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	151	case 2:
2917c006	152	__sync_or_and_fetch_2((uint16_t *) addr, val);
c51d5c6e	153	return;
985b35b1 PB	154	#endif
985b35b1 PB	155	case 4:
2917c006	156	__sync_or_and_fetch_4((uint32_t *) addr, val);
c51d5c6e	157	return;
985b35b1 PB	158	#if (CAA_BITS_PER_LONG == 64)
985b35b1 PB	159	case 8:
2917c006	160	__sync_or_and_fetch_8((uint64_t *) addr, val);
c51d5c6e	161	return;
985b35b1 PB	162	#endif
	163	}
	164	_uatomic_link_error();
c51d5c6e	165	return;
985b35b1 PB	166	}
	167
	168	#define uatomic_or(addr, v) \
	169	(_uatomic_or((addr), \
e56d99bf MD	170	caa_cast_long_keep_sign(v), \
e56d99bf MD	171	sizeof(*(addr))))
42e83919 MD	172	#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
42e83919 MD	173	#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
2812a2d2	174
985b35b1 PB	175	#endif
985b35b1 PB	176
2812a2d2	177
8760d94e PB	178	/* uatomic_add_return */
	179
	180	#ifndef uatomic_add_return
	181	static inline __attribute__((always_inline))
	182	unsigned long _uatomic_add_return(void *addr, unsigned long val,
	183	int len)
	184	{
	185	switch (len) {
f469d839 PB	186	#ifdef UATOMIC_HAS_ATOMIC_BYTE
f469d839 PB	187	case 1:
2917c006	188	return __sync_add_and_fetch_1((uint8_t *) addr, val);
f469d839 PB	189	#endif
	190	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	191	case 2:
2917c006	192	return __sync_add_and_fetch_2((uint16_t *) addr, val);
f469d839	193	#endif
8760d94e	194	case 4:
2917c006	195	return __sync_add_and_fetch_4((uint32_t *) addr, val);
b39e1761	196	#if (CAA_BITS_PER_LONG == 64)
8760d94e	197	case 8:
2917c006	198	return __sync_add_and_fetch_8((uint64_t *) addr, val);
8760d94e PB	199	#endif
	200	}
	201	_uatomic_link_error();
	202	return 0;
	203	}
	204
	205
e56d99bf MD	206	#define uatomic_add_return(addr, v) \
	207	((__typeof__(*(addr))) _uatomic_add_return((addr), \
	208	caa_cast_long_keep_sign(v), \
	209	sizeof(*(addr))))
8760d94e PB	210	#endif /* #ifndef uatomic_add_return */
	211
	212	#ifndef uatomic_xchg
	213	/* xchg */
	214
	215	static inline __attribute__((always_inline))
	216	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
	217	{
	218	switch (len) {
f469d839 PB	219	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	220	case 1:
	221	{
2917c006	222	uint8_t old;
f469d839 PB	223
f469d839 PB	224	do {
2917c006 KR	225	old = uatomic_read((uint8_t *) addr);
	226	} while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
	227	old, val));
f469d839 PB	228
	229	return old;
	230	}
	231	#endif
	232	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	233	case 2:
	234	{
2917c006	235	uint16_t old;
f469d839 PB	236
f469d839 PB	237	do {
2917c006 KR	238	old = uatomic_read((uint16_t *) addr);
	239	} while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
	240	old, val));
f469d839 PB	241
	242	return old;
	243	}
	244	#endif
8760d94e PB	245	case 4:
8760d94e PB	246	{
2917c006	247	uint32_t old;
8760d94e PB	248
8760d94e PB	249	do {
2917c006 KR	250	old = uatomic_read((uint32_t *) addr);
	251	} while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
	252	old, val));
8760d94e	253
2f2908d0	254	return old;
8760d94e	255	}
b39e1761	256	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	257	case 8:
8760d94e PB	258	{
2917c006	259	uint64_t old;
8760d94e PB	260
8760d94e PB	261	do {
2917c006 KR	262	old = uatomic_read((uint64_t *) addr);
	263	} while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
	264	old, val));
8760d94e PB	265
	266	return old;
	267	}
	268	#endif
	269	}
	270	_uatomic_link_error();
	271	return 0;
	272	}
	273
	274	#define uatomic_xchg(addr, v) \
e56d99bf MD	275	((__typeof__(*(addr))) _uatomic_exchange((addr), \
e56d99bf MD	276	caa_cast_long_keep_sign(v), \
8760d94e PB	277	sizeof(*(addr))))
	278	#endif /* #ifndef uatomic_xchg */
	279
	280	#else /* #ifndef uatomic_cmpxchg */
	281
bf33aaea PB	282	#ifndef uatomic_and
	283	/* uatomic_and */
	284
	285	static inline __attribute__((always_inline))
	286	void _uatomic_and(void *addr, unsigned long val, int len)
	287	{
	288	switch (len) {
	289	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	290	case 1:
	291	{
2917c006	292	uint8_t old, oldt;
bf33aaea	293
2917c006	294	oldt = uatomic_read((uint8_t *) addr);
bf33aaea PB	295	do {
	296	old = oldt;
	297	oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
	298	} while (oldt != old);
e6e5957d MD	299
e6e5957d MD	300	return;
bf33aaea PB	301	}
	302	#endif
	303	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	304	case 2:
	305	{
2917c006	306	uint16_t old, oldt;
bf33aaea	307
2917c006	308	oldt = uatomic_read((uint16_t *) addr);
bf33aaea PB	309	do {
	310	old = oldt;
	311	oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
	312	} while (oldt != old);
	313	}
	314	#endif
	315	case 4:
	316	{
2917c006	317	uint32_t old, oldt;
bf33aaea	318
2917c006	319	oldt = uatomic_read((uint32_t *) addr);
bf33aaea PB	320	do {
	321	old = oldt;
	322	oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
	323	} while (oldt != old);
e6e5957d MD	324
e6e5957d MD	325	return;
bf33aaea PB	326	}
	327	#if (CAA_BITS_PER_LONG == 64)
	328	case 8:
	329	{
2917c006	330	uint64_t old, oldt;
bf33aaea	331
2917c006	332	oldt = uatomic_read((uint64_t *) addr);
bf33aaea PB	333	do {
	334	old = oldt;
	335	oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
	336	} while (oldt != old);
e6e5957d MD	337
e6e5957d MD	338	return;
bf33aaea PB	339	}
	340	#endif
	341	}
	342	_uatomic_link_error();
bf33aaea PB	343	}
bf33aaea PB	344
e56d99bf MD	345	#define uatomic_and(addr, v) \
	346	(_uatomic_and((addr), \
	347	caa_cast_long_keep_sign(v), \
	348	sizeof(*(addr))))
42e83919 MD	349	#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
42e83919 MD	350	#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
2812a2d2	351
bf33aaea PB	352	#endif /* #ifndef uatomic_and */
bf33aaea PB	353
985b35b1 PB	354	#ifndef uatomic_or
	355	/* uatomic_or */
	356
	357	static inline __attribute__((always_inline))
	358	void _uatomic_or(void *addr, unsigned long val, int len)
	359	{
	360	switch (len) {
	361	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	362	case 1:
	363	{
2917c006	364	uint8_t old, oldt;
985b35b1	365
2917c006	366	oldt = uatomic_read((uint8_t *) addr);
985b35b1 PB	367	do {
	368	old = oldt;
	369	oldt = _uatomic_cmpxchg(addr, old, old \| val, 1);
	370	} while (oldt != old);
e6e5957d MD	371
e6e5957d MD	372	return;
985b35b1 PB	373	}
	374	#endif
	375	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	376	case 2:
	377	{
2917c006	378	uint16_t old, oldt;
985b35b1	379
2917c006	380	oldt = uatomic_read((uint16_t *) addr);
985b35b1 PB	381	do {
	382	old = oldt;
	383	oldt = _uatomic_cmpxchg(addr, old, old \| val, 2);
	384	} while (oldt != old);
e6e5957d MD	385
e6e5957d MD	386	return;
985b35b1 PB	387	}
	388	#endif
	389	case 4:
	390	{
2917c006	391	uint32_t old, oldt;
985b35b1	392
2917c006	393	oldt = uatomic_read((uint32_t *) addr);
985b35b1 PB	394	do {
	395	old = oldt;
	396	oldt = _uatomic_cmpxchg(addr, old, old \| val, 4);
	397	} while (oldt != old);
e6e5957d MD	398
e6e5957d MD	399	return;
985b35b1 PB	400	}
	401	#if (CAA_BITS_PER_LONG == 64)
	402	case 8:
	403	{
2917c006	404	uint64_t old, oldt;
985b35b1	405
2917c006	406	oldt = uatomic_read((uint64_t *) addr);
985b35b1 PB	407	do {
	408	old = oldt;
	409	oldt = _uatomic_cmpxchg(addr, old, old \| val, 8);
	410	} while (oldt != old);
e6e5957d MD	411
e6e5957d MD	412	return;
985b35b1 PB	413	}
	414	#endif
	415	}
	416	_uatomic_link_error();
985b35b1 PB	417	}
985b35b1 PB	418
e56d99bf MD	419	#define uatomic_or(addr, v) \
	420	(_uatomic_or((addr), \
	421	caa_cast_long_keep_sign(v), \
	422	sizeof(*(addr))))
42e83919 MD	423	#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
42e83919 MD	424	#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
2812a2d2	425
985b35b1 PB	426	#endif /* #ifndef uatomic_or */
985b35b1 PB	427
8760d94e PB	428	#ifndef uatomic_add_return
	429	/* uatomic_add_return */
	430
	431	static inline __attribute__((always_inline))
	432	unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
	433	{
	434	switch (len) {
f469d839 PB	435	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	436	case 1:
	437	{
2917c006	438	uint8_t old, oldt;
f469d839	439
2917c006	440	oldt = uatomic_read((uint8_t *) addr);
f469d839 PB	441	do {
f469d839 PB	442	old = oldt;
2917c006	443	oldt = uatomic_cmpxchg((uint8_t *) addr,
b4e6d540	444	old, old + val);
f469d839 PB	445	} while (oldt != old);
	446
	447	return old + val;
	448	}
	449	#endif
	450	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	451	case 2:
	452	{
2917c006	453	uint16_t old, oldt;
f469d839	454
2917c006	455	oldt = uatomic_read((uint16_t *) addr);
f469d839 PB	456	do {
f469d839 PB	457	old = oldt;
2917c006	458	oldt = uatomic_cmpxchg((uint16_t *) addr,
b4e6d540	459	old, old + val);
f469d839 PB	460	} while (oldt != old);
	461
	462	return old + val;
	463	}
	464	#endif
8760d94e PB	465	case 4:
8760d94e PB	466	{
2917c006	467	uint32_t old, oldt;
8760d94e	468
2917c006	469	oldt = uatomic_read((uint32_t *) addr);
8760d94e PB	470	do {
8760d94e PB	471	old = oldt;
2917c006	472	oldt = uatomic_cmpxchg((uint32_t *) addr,
b4e6d540	473	old, old + val);
8760d94e PB	474	} while (oldt != old);
	475
	476	return old + val;
	477	}
b39e1761	478	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	479	case 8:
8760d94e PB	480	{
2917c006	481	uint64_t old, oldt;
8760d94e	482
2917c006	483	oldt = uatomic_read((uint64_t *) addr);
8760d94e PB	484	do {
8760d94e PB	485	old = oldt;
2917c006	486	oldt = uatomic_cmpxchg((uint64_t *) addr,
b4e6d540	487	old, old + val);
8760d94e PB	488	} while (oldt != old);
	489
	490	return old + val;
	491	}
	492	#endif
	493	}
	494	_uatomic_link_error();
	495	return 0;
	496	}
	497
e56d99bf MD	498	#define uatomic_add_return(addr, v) \
	499	((__typeof__(*(addr))) _uatomic_add_return((addr), \
	500	caa_cast_long_keep_sign(v), \
	501	sizeof(*(addr))))
8760d94e PB	502	#endif /* #ifndef uatomic_add_return */
	503
	504	#ifndef uatomic_xchg
	505	/* xchg */
	506
	507	static inline __attribute__((always_inline))
	508	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
	509	{
	510	switch (len) {
f469d839 PB	511	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	512	case 1:
	513	{
2917c006	514	uint8_t old, oldt;
f469d839	515
2917c006	516	oldt = uatomic_read((uint8_t *) addr);
f469d839 PB	517	do {
f469d839 PB	518	old = oldt;
2917c006	519	oldt = uatomic_cmpxchg((uint8_t *) addr,
b4e6d540	520	old, val);
f469d839 PB	521	} while (oldt != old);
	522
	523	return old;
	524	}
	525	#endif
	526	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	527	case 2:
	528	{
2917c006	529	uint16_t old, oldt;
f469d839	530
2917c006	531	oldt = uatomic_read((uint16_t *) addr);
f469d839 PB	532	do {
f469d839 PB	533	old = oldt;
2917c006	534	oldt = uatomic_cmpxchg((uint16_t *) addr,
b4e6d540	535	old, val);
f469d839 PB	536	} while (oldt != old);
	537
	538	return old;
	539	}
	540	#endif
8760d94e PB	541	case 4:
8760d94e PB	542	{
2917c006	543	uint32_t old, oldt;
8760d94e	544
2917c006	545	oldt = uatomic_read((uint32_t *) addr);
8760d94e PB	546	do {
8760d94e PB	547	old = oldt;
2917c006	548	oldt = uatomic_cmpxchg((uint32_t *) addr,
b4e6d540	549	old, val);
8760d94e PB	550	} while (oldt != old);
	551
	552	return old;
	553	}
b39e1761	554	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	555	case 8:
8760d94e PB	556	{
2917c006	557	uint64_t old, oldt;
8760d94e	558
2917c006	559	oldt = uatomic_read((uint64_t *) addr);
8760d94e PB	560	do {
8760d94e PB	561	old = oldt;
2917c006	562	oldt = uatomic_cmpxchg((uint64_t *) addr,
b4e6d540	563	old, val);
8760d94e PB	564	} while (oldt != old);
	565
	566	return old;
	567	}
	568	#endif
	569	}
	570	_uatomic_link_error();
	571	return 0;
	572	}
	573
	574	#define uatomic_xchg(addr, v) \
e56d99bf MD	575	((__typeof__(*(addr))) _uatomic_exchange((addr), \
e56d99bf MD	576	caa_cast_long_keep_sign(v), \
8760d94e PB	577	sizeof(*(addr))))
	578	#endif /* #ifndef uatomic_xchg */
	579
	580	#endif /* #else #ifndef uatomic_cmpxchg */
	581
	582	/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
	583
	584	#ifndef uatomic_add
	585	#define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
42e83919 MD	586	#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
42e83919 MD	587	#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
8760d94e PB	588	#endif
8760d94e PB	589
e56d99bf MD	590	#define uatomic_sub_return(addr, v) \
	591	uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
	592	#define uatomic_sub(addr, v) \
	593	uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
42e83919 MD	594	#define cmm_smp_mb__before_uatomic_sub() cmm_smp_mb__before_uatomic_add()
42e83919 MD	595	#define cmm_smp_mb__after_uatomic_sub() cmm_smp_mb__after_uatomic_add()
8760d94e PB	596
	597	#ifndef uatomic_inc
	598	#define uatomic_inc(addr) uatomic_add((addr), 1)
42e83919 MD	599	#define cmm_smp_mb__before_uatomic_inc() cmm_smp_mb__before_uatomic_add()
42e83919 MD	600	#define cmm_smp_mb__after_uatomic_inc() cmm_smp_mb__after_uatomic_add()
8760d94e PB	601	#endif
	602
	603	#ifndef uatomic_dec
	604	#define uatomic_dec(addr) uatomic_add((addr), -1)
42e83919 MD	605	#define cmm_smp_mb__before_uatomic_dec() cmm_smp_mb__before_uatomic_add()
42e83919 MD	606	#define cmm_smp_mb__after_uatomic_dec() cmm_smp_mb__after_uatomic_add()
8760d94e PB	607	#endif
	608
	609	#ifdef __cplusplus
	610	}
	611	#endif
	612
	613	#endif /* _URCU_UATOMIC_GENERIC_H */