[urcu.git] / urcu / uatomic / generic.h

#ifndef _URCU_UATOMIC_GENERIC_H
#define _URCU_UATOMIC_GENERIC_H

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 * Copyright (c) 2010      Paolo Bonzini
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <urcu/compiler.h>
#include <urcu/system.h>

#ifdef __cplusplus
extern "C" {
#endif

#ifndef uatomic_set
#define uatomic_set(addr, v)	CMM_STORE_SHARED(*(addr), (v))
#endif

#ifndef uatomic_read
#define uatomic_read(addr)	CMM_LOAD_SHARED(*(addr))
#endif

#if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR
static inline __attribute__((always_inline))
void _uatomic_link_error()
{
#ifdef ILLEGAL_INSTR
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__(ILLEGAL_INSTR);
#else
	__builtin_trap ();
#endif
}

#else /* #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */
extern void _uatomic_link_error ();
#endif /* #else #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */

/* cmpxchg */

#ifndef uatomic_cmpxchg
static inline __attribute__((always_inline))
unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		return __sync_val_compare_and_swap_1(addr, old, _new);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		return __sync_val_compare_and_swap_2(addr, old, _new);
#endif
	case 4:
		return __sync_val_compare_and_swap_4(addr, old, _new);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		return __sync_val_compare_and_swap_8(addr, old, _new);
#endif
	}
	_uatomic_link_error();
	return 0;
}


#define uatomic_cmpxchg(addr, old, _new)				      \
	((__typeof__(*(addr))) _uatomic_cmpxchg((addr),			      \
						caa_cast_long_keep_sign(old), \
						caa_cast_long_keep_sign(_new),\
						sizeof(*(addr))))


/* uatomic_and */

#ifndef uatomic_and
static inline __attribute__((always_inline))
void _uatomic_and(void *addr, unsigned long val,
		  int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		__sync_and_and_fetch_1(addr, val);
		return;
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		__sync_and_and_fetch_2(addr, val);
		return;
#endif
	case 4:
		__sync_and_and_fetch_4(addr, val);
		return;
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		__sync_and_and_fetch_8(addr, val);
		return;
#endif
	}
	_uatomic_link_error();
}

#define uatomic_and(addr, v)			\
	(_uatomic_and((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#endif

/* uatomic_or */

#ifndef uatomic_or
static inline __attribute__((always_inline))
void _uatomic_or(void *addr, unsigned long val,
		 int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		__sync_or_and_fetch_1(addr, val);
		return;
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		__sync_or_and_fetch_2(addr, val);
		return;
#endif
	case 4:
		__sync_or_and_fetch_4(addr, val);
		return;
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		__sync_or_and_fetch_8(addr, val);
		return;
#endif
	}
	_uatomic_link_error();
	return;
}

#define uatomic_or(addr, v)			\
	(_uatomic_or((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#endif

/* uatomic_add_return */

#ifndef uatomic_add_return
static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		return __sync_add_and_fetch_1(addr, val);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		return __sync_add_and_fetch_2(addr, val);
#endif
	case 4:
		return __sync_add_and_fetch_4(addr, val);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		return __sync_add_and_fetch_8(addr, val);
#endif
	}
	_uatomic_link_error();
	return 0;
}


#define uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) _uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_add_return */

#ifndef uatomic_xchg
/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		unsigned char old;

		do {
			old = uatomic_read((unsigned char *)addr);
		} while (!__sync_bool_compare_and_swap_1(addr, old, val));

		return old;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		unsigned short old;

		do {
			old = uatomic_read((unsigned short *)addr);
		} while (!__sync_bool_compare_and_swap_2(addr, old, val));

		return old;
	}
#endif
	case 4:
	{
		unsigned int old;

		do {
			old = uatomic_read((unsigned int *)addr);
		} while (!__sync_bool_compare_and_swap_4(addr, old, val));

		return old;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old;

		do {
			old = uatomic_read((unsigned long *)addr);
		} while (!__sync_bool_compare_and_swap_8(addr, old, val));

		return old;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_xchg */

#else /* #ifndef uatomic_cmpxchg */

#ifndef uatomic_and
/* uatomic_and */

static inline __attribute__((always_inline))
void _uatomic_and(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		unsigned char old, oldt;

		oldt = uatomic_read((unsigned char *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
		} while (oldt != old);

		return;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		unsigned short old, oldt;

		oldt = uatomic_read((unsigned short *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
		} while (oldt != old);
	}
#endif
	case 4:
	{
		unsigned int old, oldt;

		oldt = uatomic_read((unsigned int *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
		} while (oldt != old);

		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old, oldt;

		oldt = uatomic_read((unsigned long *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
		} while (oldt != old);

		return;
	}
#endif
	}
	_uatomic_link_error();
}

#define uatomic_and(addr, v)			\
	(_uatomic_and((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#endif /* #ifndef uatomic_and */

#ifndef uatomic_or
/* uatomic_or */

static inline __attribute__((always_inline))
void _uatomic_or(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		unsigned char old, oldt;

		oldt = uatomic_read((unsigned char *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
		} while (oldt != old);

		return;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		unsigned short old, oldt;

		oldt = uatomic_read((unsigned short *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
		} while (oldt != old);

		return;
	}
#endif
	case 4:
	{
		unsigned int old, oldt;

		oldt = uatomic_read((unsigned int *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
		} while (oldt != old);

		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old, oldt;

		oldt = uatomic_read((unsigned long *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
		} while (oldt != old);

		return;
	}
#endif
	}
	_uatomic_link_error();
}

#define uatomic_or(addr, v)			\
	(_uatomic_or((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#endif /* #ifndef uatomic_or */

#ifndef uatomic_add_return
/* uatomic_add_return */

static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		unsigned char old, oldt;

		oldt = uatomic_read((unsigned char *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned char *)addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		unsigned short old, oldt;

		oldt = uatomic_read((unsigned short *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned short *)addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
	case 4:
	{
		unsigned int old, oldt;

		oldt = uatomic_read((unsigned int *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned int *)addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old, oldt;

		oldt = uatomic_read((unsigned long *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned long *)addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) _uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_add_return */

#ifndef uatomic_xchg
/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		unsigned char old, oldt;

		oldt = uatomic_read((unsigned char *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned char *)addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		unsigned short old, oldt;

		oldt = uatomic_read((unsigned short *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned short *)addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
	case 4:
	{
		unsigned int old, oldt;

		oldt = uatomic_read((unsigned int *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned int *)addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old, oldt;

		oldt = uatomic_read((unsigned long *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned long *)addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_xchg */

#endif /* #else #ifndef uatomic_cmpxchg */

/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */

#ifndef uatomic_add
#define uatomic_add(addr, v)		(void)uatomic_add_return((addr), (v))
#endif

#define uatomic_sub_return(addr, v)	\
	uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
#define uatomic_sub(addr, v)		\
	uatomic_add((addr), -(caa_cast_long_keep_sign(v)))

#ifndef uatomic_inc
#define uatomic_inc(addr)		uatomic_add((addr), 1)
#endif

#ifndef uatomic_dec
#define uatomic_dec(addr)		uatomic_add((addr), -1)
#endif

#ifdef __cplusplus
}
#endif

#endif /* _URCU_UATOMIC_GENERIC_H */
Commit	Line	Data
8760d94e PB	1	#ifndef _URCU_UATOMIC_GENERIC_H
	2	#define _URCU_UATOMIC_GENERIC_H
	3
	4	/*
	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	* Copyright (c) 2010 Paolo Bonzini
	10	*
	11	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	12	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	13	*
	14	* Permission is hereby granted to use or copy this program
	15	* for any purpose, provided the above notices are retained on all copies.
	16	* Permission to modify the code and to distribute modified code is granted,
	17	* provided the above notices are retained, and a notice that the code was
	18	* modified is included with the above copyright notice.
	19	*
	20	* Code inspired from libuatomic_ops-1.2, inherited in part from the
	21	* Boehm-Demers-Weiser conservative garbage collector.
	22	*/
	23
	24	#include <urcu/compiler.h>
	25	#include <urcu/system.h>
	26
	27	#ifdef __cplusplus
	28	extern "C" {
	29	#endif
	30
8760d94e	31	#ifndef uatomic_set
6cf3827c	32	#define uatomic_set(addr, v) CMM_STORE_SHARED(*(addr), (v))
8760d94e PB	33	#endif
	34
	35	#ifndef uatomic_read
6cf3827c	36	#define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
8760d94e PB	37	#endif
	38
	39	#if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR
	40	static inline __attribute__((always_inline))
	41	void _uatomic_link_error()
	42	{
	43	#ifdef ILLEGAL_INSTR
	44	/* generate an illegal instruction. Cannot catch this with linker tricks
	45	* when optimizations are disabled. */
	46	__asm__ __volatile__(ILLEGAL_INSTR);
	47	#else
	48	__builtin_trap ();
	49	#endif
	50	}
	51
	52	#else /* #if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR */
	53	extern void _uatomic_link_error ();
	54	#endif /* #else #if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR */
	55
	56	/* cmpxchg */
	57
	58	#ifndef uatomic_cmpxchg
	59	static inline __attribute__((always_inline))
	60	unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
	61	unsigned long _new, int len)
	62	{
	63	switch (len) {
f469d839 PB	64	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	65	case 1:
	66	return __sync_val_compare_and_swap_1(addr, old, _new);
	67	#endif
	68	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	69	case 2:
	70	return __sync_val_compare_and_swap_2(addr, old, _new);
	71	#endif
8760d94e PB	72	case 4:
8760d94e PB	73	return __sync_val_compare_and_swap_4(addr, old, _new);
b39e1761	74	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	75	case 8:
	76	return __sync_val_compare_and_swap_8(addr, old, _new);
	77	#endif
	78	}
	79	_uatomic_link_error();
	80	return 0;
	81	}
	82
	83
e56d99bf MD	84	#define uatomic_cmpxchg(addr, old, _new) \
	85	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
	86	caa_cast_long_keep_sign(old), \
	87	caa_cast_long_keep_sign(_new),\
8760d94e PB	88	sizeof(*(addr))))
	89
	90
bf33aaea PB	91	/* uatomic_and */
	92
	93	#ifndef uatomic_and
	94	static inline __attribute__((always_inline))
	95	void _uatomic_and(void *addr, unsigned long val,
	96	int len)
	97	{
	98	switch (len) {
	99	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	100	case 1:
	101	__sync_and_and_fetch_1(addr, val);
c51d5c6e	102	return;
bf33aaea PB	103	#endif
	104	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	105	case 2:
	106	__sync_and_and_fetch_2(addr, val);
c51d5c6e	107	return;
bf33aaea PB	108	#endif
	109	case 4:
	110	__sync_and_and_fetch_4(addr, val);
c51d5c6e	111	return;
bf33aaea PB	112	#if (CAA_BITS_PER_LONG == 64)
	113	case 8:
	114	__sync_and_and_fetch_8(addr, val);
c51d5c6e	115	return;
bf33aaea PB	116	#endif
	117	}
	118	_uatomic_link_error();
bf33aaea PB	119	}
	120
	121	#define uatomic_and(addr, v) \
	122	(_uatomic_and((addr), \
e56d99bf MD	123	caa_cast_long_keep_sign(v), \
e56d99bf MD	124	sizeof(*(addr))))
bf33aaea PB	125	#endif
bf33aaea PB	126
985b35b1 PB	127	/* uatomic_or */
	128
	129	#ifndef uatomic_or
	130	static inline __attribute__((always_inline))
	131	void _uatomic_or(void *addr, unsigned long val,
	132	int len)
	133	{
	134	switch (len) {
	135	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	136	case 1:
	137	__sync_or_and_fetch_1(addr, val);
c51d5c6e	138	return;
985b35b1 PB	139	#endif
	140	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	141	case 2:
	142	__sync_or_and_fetch_2(addr, val);
c51d5c6e	143	return;
985b35b1 PB	144	#endif
	145	case 4:
	146	__sync_or_and_fetch_4(addr, val);
c51d5c6e	147	return;
985b35b1 PB	148	#if (CAA_BITS_PER_LONG == 64)
	149	case 8:
	150	__sync_or_and_fetch_8(addr, val);
c51d5c6e	151	return;
985b35b1 PB	152	#endif
	153	}
	154	_uatomic_link_error();
c51d5c6e	155	return;
985b35b1 PB	156	}
	157
	158	#define uatomic_or(addr, v) \
	159	(_uatomic_or((addr), \
e56d99bf MD	160	caa_cast_long_keep_sign(v), \
e56d99bf MD	161	sizeof(*(addr))))
985b35b1 PB	162	#endif
985b35b1 PB	163
8760d94e PB	164	/* uatomic_add_return */
	165
	166	#ifndef uatomic_add_return
	167	static inline __attribute__((always_inline))
	168	unsigned long _uatomic_add_return(void *addr, unsigned long val,
	169	int len)
	170	{
	171	switch (len) {
f469d839 PB	172	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	173	case 1:
	174	return __sync_add_and_fetch_1(addr, val);
	175	#endif
	176	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	177	case 2:
	178	return __sync_add_and_fetch_2(addr, val);
	179	#endif
8760d94e PB	180	case 4:
8760d94e PB	181	return __sync_add_and_fetch_4(addr, val);
b39e1761	182	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	183	case 8:
	184	return __sync_add_and_fetch_8(addr, val);
	185	#endif
	186	}
	187	_uatomic_link_error();
	188	return 0;
	189	}
	190
	191
e56d99bf MD	192	#define uatomic_add_return(addr, v) \
	193	((__typeof__(*(addr))) _uatomic_add_return((addr), \
	194	caa_cast_long_keep_sign(v), \
	195	sizeof(*(addr))))
8760d94e PB	196	#endif /* #ifndef uatomic_add_return */
	197
	198	#ifndef uatomic_xchg
	199	/* xchg */
	200
	201	static inline __attribute__((always_inline))
	202	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
	203	{
	204	switch (len) {
f469d839 PB	205	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	206	case 1:
	207	{
	208	unsigned char old;
	209
	210	do {
	211	old = uatomic_read((unsigned char *)addr);
	212	} while (!__sync_bool_compare_and_swap_1(addr, old, val));
	213
	214	return old;
	215	}
	216	#endif
	217	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	218	case 2:
	219	{
	220	unsigned short old;
	221
	222	do {
	223	old = uatomic_read((unsigned short *)addr);
	224	} while (!__sync_bool_compare_and_swap_2(addr, old, val));
	225
	226	return old;
	227	}
	228	#endif
8760d94e PB	229	case 4:
	230	{
	231	unsigned int old;
	232
	233	do {
	234	old = uatomic_read((unsigned int *)addr);
2f2908d0	235	} while (!__sync_bool_compare_and_swap_4(addr, old, val));
8760d94e	236
2f2908d0	237	return old;
8760d94e	238	}
b39e1761	239	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	240	case 8:
	241	{
	242	unsigned long old;
	243
	244	do {
	245	old = uatomic_read((unsigned long *)addr);
	246	} while (!__sync_bool_compare_and_swap_8(addr, old, val));
	247
	248	return old;
	249	}
	250	#endif
	251	}
	252	_uatomic_link_error();
	253	return 0;
	254	}
	255
	256	#define uatomic_xchg(addr, v) \
e56d99bf MD	257	((__typeof__(*(addr))) _uatomic_exchange((addr), \
e56d99bf MD	258	caa_cast_long_keep_sign(v), \
8760d94e PB	259	sizeof(*(addr))))
	260	#endif /* #ifndef uatomic_xchg */
	261
	262	#else /* #ifndef uatomic_cmpxchg */
	263
bf33aaea PB	264	#ifndef uatomic_and
	265	/* uatomic_and */
	266
	267	static inline __attribute__((always_inline))
	268	void _uatomic_and(void *addr, unsigned long val, int len)
	269	{
	270	switch (len) {
	271	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	272	case 1:
	273	{
	274	unsigned char old, oldt;
	275
	276	oldt = uatomic_read((unsigned char *)addr);
	277	do {
	278	old = oldt;
	279	oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
	280	} while (oldt != old);
e6e5957d MD	281
e6e5957d MD	282	return;
bf33aaea PB	283	}
	284	#endif
	285	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	286	case 2:
	287	{
	288	unsigned short old, oldt;
	289
	290	oldt = uatomic_read((unsigned short *)addr);
	291	do {
	292	old = oldt;
	293	oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
	294	} while (oldt != old);
	295	}
	296	#endif
	297	case 4:
	298	{
	299	unsigned int old, oldt;
	300
	301	oldt = uatomic_read((unsigned int *)addr);
	302	do {
	303	old = oldt;
	304	oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
	305	} while (oldt != old);
e6e5957d MD	306
e6e5957d MD	307	return;
bf33aaea PB	308	}
	309	#if (CAA_BITS_PER_LONG == 64)
	310	case 8:
	311	{
	312	unsigned long old, oldt;
	313
	314	oldt = uatomic_read((unsigned long *)addr);
	315	do {
	316	old = oldt;
	317	oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
	318	} while (oldt != old);
e6e5957d MD	319
e6e5957d MD	320	return;
bf33aaea PB	321	}
	322	#endif
	323	}
	324	_uatomic_link_error();
bf33aaea PB	325	}
bf33aaea PB	326
e56d99bf MD	327	#define uatomic_and(addr, v) \
	328	(_uatomic_and((addr), \
	329	caa_cast_long_keep_sign(v), \
	330	sizeof(*(addr))))
bf33aaea PB	331	#endif /* #ifndef uatomic_and */
bf33aaea PB	332
985b35b1 PB	333	#ifndef uatomic_or
	334	/* uatomic_or */
	335
	336	static inline __attribute__((always_inline))
	337	void _uatomic_or(void *addr, unsigned long val, int len)
	338	{
	339	switch (len) {
	340	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	341	case 1:
	342	{
	343	unsigned char old, oldt;
	344
	345	oldt = uatomic_read((unsigned char *)addr);
	346	do {
	347	old = oldt;
	348	oldt = _uatomic_cmpxchg(addr, old, old \| val, 1);
	349	} while (oldt != old);
e6e5957d MD	350
e6e5957d MD	351	return;
985b35b1 PB	352	}
	353	#endif
	354	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	355	case 2:
	356	{
	357	unsigned short old, oldt;
	358
	359	oldt = uatomic_read((unsigned short *)addr);
	360	do {
	361	old = oldt;
	362	oldt = _uatomic_cmpxchg(addr, old, old \| val, 2);
	363	} while (oldt != old);
e6e5957d MD	364
e6e5957d MD	365	return;
985b35b1 PB	366	}
	367	#endif
	368	case 4:
	369	{
	370	unsigned int old, oldt;
	371
	372	oldt = uatomic_read((unsigned int *)addr);
	373	do {
	374	old = oldt;
	375	oldt = _uatomic_cmpxchg(addr, old, old \| val, 4);
	376	} while (oldt != old);
e6e5957d MD	377
e6e5957d MD	378	return;
985b35b1 PB	379	}
	380	#if (CAA_BITS_PER_LONG == 64)
	381	case 8:
	382	{
	383	unsigned long old, oldt;
	384
	385	oldt = uatomic_read((unsigned long *)addr);
	386	do {
	387	old = oldt;
	388	oldt = _uatomic_cmpxchg(addr, old, old \| val, 8);
	389	} while (oldt != old);
e6e5957d MD	390
e6e5957d MD	391	return;
985b35b1 PB	392	}
	393	#endif
	394	}
	395	_uatomic_link_error();
985b35b1 PB	396	}
985b35b1 PB	397
e56d99bf MD	398	#define uatomic_or(addr, v) \
	399	(_uatomic_or((addr), \
	400	caa_cast_long_keep_sign(v), \
	401	sizeof(*(addr))))
985b35b1 PB	402	#endif /* #ifndef uatomic_or */
985b35b1 PB	403
8760d94e PB	404	#ifndef uatomic_add_return
	405	/* uatomic_add_return */
	406
	407	static inline __attribute__((always_inline))
	408	unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
	409	{
	410	switch (len) {
f469d839 PB	411	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	412	case 1:
	413	{
	414	unsigned char old, oldt;
	415
	416	oldt = uatomic_read((unsigned char *)addr);
	417	do {
	418	old = oldt;
b4e6d540 PB	419	oldt = uatomic_cmpxchg((unsigned char *)addr,
b4e6d540 PB	420	old, old + val);
f469d839 PB	421	} while (oldt != old);
	422
	423	return old + val;
	424	}
	425	#endif
	426	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	427	case 2:
	428	{
	429	unsigned short old, oldt;
	430
	431	oldt = uatomic_read((unsigned short *)addr);
	432	do {
	433	old = oldt;
b4e6d540 PB	434	oldt = uatomic_cmpxchg((unsigned short *)addr,
b4e6d540 PB	435	old, old + val);
f469d839 PB	436	} while (oldt != old);
	437
	438	return old + val;
	439	}
	440	#endif
8760d94e PB	441	case 4:
	442	{
	443	unsigned int old, oldt;
	444
	445	oldt = uatomic_read((unsigned int *)addr);
	446	do {
	447	old = oldt;
b4e6d540 PB	448	oldt = uatomic_cmpxchg((unsigned int *)addr,
b4e6d540 PB	449	old, old + val);
8760d94e PB	450	} while (oldt != old);
	451
	452	return old + val;
	453	}
b39e1761	454	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	455	case 8:
	456	{
	457	unsigned long old, oldt;
	458
	459	oldt = uatomic_read((unsigned long *)addr);
	460	do {
	461	old = oldt;
b4e6d540 PB	462	oldt = uatomic_cmpxchg((unsigned long *)addr,
b4e6d540 PB	463	old, old + val);
8760d94e PB	464	} while (oldt != old);
	465
	466	return old + val;
	467	}
	468	#endif
	469	}
	470	_uatomic_link_error();
	471	return 0;
	472	}
	473
e56d99bf MD	474	#define uatomic_add_return(addr, v) \
	475	((__typeof__(*(addr))) _uatomic_add_return((addr), \
	476	caa_cast_long_keep_sign(v), \
	477	sizeof(*(addr))))
8760d94e PB	478	#endif /* #ifndef uatomic_add_return */
	479
	480	#ifndef uatomic_xchg
	481	/* xchg */
	482
	483	static inline __attribute__((always_inline))
	484	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
	485	{
	486	switch (len) {
f469d839 PB	487	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	488	case 1:
	489	{
	490	unsigned char old, oldt;
	491
	492	oldt = uatomic_read((unsigned char *)addr);
	493	do {
	494	old = oldt;
b4e6d540 PB	495	oldt = uatomic_cmpxchg((unsigned char *)addr,
b4e6d540 PB	496	old, val);
f469d839 PB	497	} while (oldt != old);
	498
	499	return old;
	500	}
	501	#endif
	502	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	503	case 2:
	504	{
	505	unsigned short old, oldt;
	506
	507	oldt = uatomic_read((unsigned short *)addr);
	508	do {
	509	old = oldt;
b4e6d540 PB	510	oldt = uatomic_cmpxchg((unsigned short *)addr,
b4e6d540 PB	511	old, val);
f469d839 PB	512	} while (oldt != old);
	513
	514	return old;
	515	}
	516	#endif
8760d94e PB	517	case 4:
	518	{
	519	unsigned int old, oldt;
	520
	521	oldt = uatomic_read((unsigned int *)addr);
	522	do {
	523	old = oldt;
b4e6d540 PB	524	oldt = uatomic_cmpxchg((unsigned int *)addr,
b4e6d540 PB	525	old, val);
8760d94e PB	526	} while (oldt != old);
	527
	528	return old;
	529	}
b39e1761	530	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	531	case 8:
	532	{
	533	unsigned long old, oldt;
	534
	535	oldt = uatomic_read((unsigned long *)addr);
	536	do {
	537	old = oldt;
b4e6d540 PB	538	oldt = uatomic_cmpxchg((unsigned long *)addr,
b4e6d540 PB	539	old, val);
8760d94e PB	540	} while (oldt != old);
	541
	542	return old;
	543	}
	544	#endif
	545	}
	546	_uatomic_link_error();
	547	return 0;
	548	}
	549
	550	#define uatomic_xchg(addr, v) \
e56d99bf MD	551	((__typeof__(*(addr))) _uatomic_exchange((addr), \
e56d99bf MD	552	caa_cast_long_keep_sign(v), \
8760d94e PB	553	sizeof(*(addr))))
	554	#endif /* #ifndef uatomic_xchg */
	555
	556	#endif /* #else #ifndef uatomic_cmpxchg */
	557
	558	/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
	559
	560	#ifndef uatomic_add
	561	#define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
	562	#endif
	563
e56d99bf MD	564	#define uatomic_sub_return(addr, v) \
	565	uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
	566	#define uatomic_sub(addr, v) \
	567	uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
8760d94e PB	568
	569	#ifndef uatomic_inc
	570	#define uatomic_inc(addr) uatomic_add((addr), 1)
	571	#endif
	572
	573	#ifndef uatomic_dec
	574	#define uatomic_dec(addr) uatomic_add((addr), -1)
	575	#endif
	576
	577	#ifdef __cplusplus
	578	}
	579	#endif
	580
	581	#endif /* _URCU_UATOMIC_GENERIC_H */