[urcu.git] / urcu / uatomic / generic.h

#ifndef _URCU_UATOMIC_GENERIC_H
#define _URCU_UATOMIC_GENERIC_H

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 * Copyright (c) 2010      Paolo Bonzini
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <urcu/compiler.h>
#include <urcu/system.h>

#ifdef __cplusplus
extern "C" {
#endif

#ifndef uatomic_set
#define uatomic_set(addr, v)	CMM_STORE_SHARED(*(addr), (v))
#endif

#ifndef uatomic_read
#define uatomic_read(addr)	CMM_LOAD_SHARED(*(addr))
#endif

#if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR
static inline __attribute__((always_inline))
void _uatomic_link_error()
{
#ifdef ILLEGAL_INSTR
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__(ILLEGAL_INSTR);
#else
	__builtin_trap ();
#endif
}

#else /* #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */
extern void _uatomic_link_error ();
#endif /* #else #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */

/* cmpxchg */

#ifndef uatomic_cmpxchg
static inline __attribute__((always_inline))
unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		return __sync_val_compare_and_swap_1(addr, old, _new);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		return __sync_val_compare_and_swap_2(addr, old, _new);
#endif
	case 4:
		return __sync_val_compare_and_swap_4(addr, old, _new);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		return __sync_val_compare_and_swap_8(addr, old, _new);
#endif
	}
	_uatomic_link_error();
	return 0;
}


#define uatomic_cmpxchg(addr, old, _new)				    \
	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
						(unsigned long)(_new), 	    \
						sizeof(*(addr))))


/* uatomic_and */

#ifndef uatomic_and
static inline __attribute__((always_inline))
void _uatomic_and(void *addr, unsigned long val,
		  int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		__sync_and_and_fetch_1(addr, val);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		__sync_and_and_fetch_2(addr, val);
#endif
	case 4:
		__sync_and_and_fetch_4(addr, val);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		__sync_and_and_fetch_8(addr, val);
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_and(addr, v)			\
	(_uatomic_and((addr),			\
		      (unsigned long)(v),	\
		      sizeof(*(addr))))
#endif

/* uatomic_or */

#ifndef uatomic_or
static inline __attribute__((always_inline))
void _uatomic_or(void *addr, unsigned long val,
		 int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		__sync_or_and_fetch_1(addr, val);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		__sync_or_and_fetch_2(addr, val);
#endif
	case 4:
		__sync_or_and_fetch_4(addr, val);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		__sync_or_and_fetch_8(addr, val);
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_or(addr, v)			\
	(_uatomic_or((addr),			\
		     (unsigned long)(v),	\
		     sizeof(*(addr))))
#endif

/* uatomic_add_return */

#ifndef uatomic_add_return
static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		return __sync_add_and_fetch_1(addr, val);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		return __sync_add_and_fetch_2(addr, val);
#endif
	case 4:
		return __sync_add_and_fetch_4(addr, val);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		return __sync_add_and_fetch_8(addr, val);
#endif
	}
	_uatomic_link_error();
	return 0;
}


#define uatomic_add_return(addr, v)					\
	((__typeof__(*(addr))) _uatomic_add_return((addr),		\
						  (unsigned long)(v),	\
						  sizeof(*(addr))))
#endif /* #ifndef uatomic_add_return */

#ifndef uatomic_xchg
/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		unsigned char old;

		do {
			old = uatomic_read((unsigned char *)addr);
		} while (!__sync_bool_compare_and_swap_1(addr, old, val));

		return old;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		unsigned short old;

		do {
			old = uatomic_read((unsigned short *)addr);
		} while (!__sync_bool_compare_and_swap_2(addr, old, val));

		return old;
	}
#endif
	case 4:
	{
		unsigned int old;

		do {
			old = uatomic_read((unsigned int *)addr);
		} while (!__sync_bool_compare_and_swap_4(addr, old, val));

		return old;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old;

		do {
			old = uatomic_read((unsigned long *)addr);
		} while (!__sync_bool_compare_and_swap_8(addr, old, val));

		return old;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_xchg */

#else /* #ifndef uatomic_cmpxchg */

#ifndef uatomic_and
/* uatomic_and */

static inline __attribute__((always_inline))
void _uatomic_and(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		unsigned char old, oldt;

		oldt = uatomic_read((unsigned char *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
		} while (oldt != old);

		return;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		unsigned short old, oldt;

		oldt = uatomic_read((unsigned short *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
		} while (oldt != old);
	}
#endif
	case 4:
	{
		unsigned int old, oldt;

		oldt = uatomic_read((unsigned int *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
		} while (oldt != old);

		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old, oldt;

		oldt = uatomic_read((unsigned long *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
		} while (oldt != old);

		return;
	}
#endif
	}
	_uatomic_link_error();
}

#define uatomic_and(addr, v)		\
	(uatomic_and((addr),		\
		    (unsigned long)(v),	\
		    sizeof(*(addr))))
#endif /* #ifndef uatomic_and */

#ifndef uatomic_or
/* uatomic_or */

static inline __attribute__((always_inline))
void _uatomic_or(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		unsigned char old, oldt;

		oldt = uatomic_read((unsigned char *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
		} while (oldt != old);

		return;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		unsigned short old, oldt;

		oldt = uatomic_read((unsigned short *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
		} while (oldt != old);

		return;
	}
#endif
	case 4:
	{
		unsigned int old, oldt;

		oldt = uatomic_read((unsigned int *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
		} while (oldt != old);

		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old, oldt;

		oldt = uatomic_read((unsigned long *)addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
		} while (oldt != old);

		return;
	}
#endif
	}
	_uatomic_link_error();
}

#define uatomic_or(addr, v)		\
	(uatomic_or((addr),		\
		    (unsigned long)(v),	\
		    sizeof(*(addr))))
#endif /* #ifndef uatomic_or */

#ifndef uatomic_add_return
/* uatomic_add_return */

static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		unsigned char old, oldt;

		oldt = uatomic_read((unsigned char *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned char *)addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		unsigned short old, oldt;

		oldt = uatomic_read((unsigned short *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned short *)addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
	case 4:
	{
		unsigned int old, oldt;

		oldt = uatomic_read((unsigned int *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned int *)addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old, oldt;

		oldt = uatomic_read((unsigned long *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned long *)addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_add_return(addr, v)					\
	((__typeof__(*(addr))) _uatomic_add_return((addr),		\
						  (unsigned long)(v),	\
						  sizeof(*(addr))))
#endif /* #ifndef uatomic_add_return */

#ifndef uatomic_xchg
/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		unsigned char old, oldt;

		oldt = uatomic_read((unsigned char *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned char *)addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		unsigned short old, oldt;

		oldt = uatomic_read((unsigned short *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned short *)addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
	case 4:
	{
		unsigned int old, oldt;

		oldt = uatomic_read((unsigned int *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned int *)addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old, oldt;

		oldt = uatomic_read((unsigned long *)addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((unsigned long *)addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_xchg */

#endif /* #else #ifndef uatomic_cmpxchg */

/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */

#ifndef uatomic_add
#define uatomic_add(addr, v)		(void)uatomic_add_return((addr), (v))
#endif

#define uatomic_sub_return(addr, v)	uatomic_add_return((addr), -(v))
#define uatomic_sub(addr, v)		uatomic_add((addr), -(v))

#ifndef uatomic_inc
#define uatomic_inc(addr)		uatomic_add((addr), 1)
#endif

#ifndef uatomic_dec
#define uatomic_dec(addr)		uatomic_add((addr), -1)
#endif

#ifdef __cplusplus
}
#endif

#endif /* _URCU_UATOMIC_GENERIC_H */
Commit	Line	Data
8760d94e PB	1	#ifndef _URCU_UATOMIC_GENERIC_H
	2	#define _URCU_UATOMIC_GENERIC_H
	3
	4	/*
	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	* Copyright (c) 2010 Paolo Bonzini
	10	*
	11	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	12	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	13	*
	14	* Permission is hereby granted to use or copy this program
	15	* for any purpose, provided the above notices are retained on all copies.
	16	* Permission to modify the code and to distribute modified code is granted,
	17	* provided the above notices are retained, and a notice that the code was
	18	* modified is included with the above copyright notice.
	19	*
	20	* Code inspired from libuatomic_ops-1.2, inherited in part from the
	21	* Boehm-Demers-Weiser conservative garbage collector.
	22	*/
	23
	24	#include <urcu/compiler.h>
	25	#include <urcu/system.h>
	26
	27	#ifdef __cplusplus
	28	extern "C" {
	29	#endif
	30
8760d94e	31	#ifndef uatomic_set
6cf3827c	32	#define uatomic_set(addr, v) CMM_STORE_SHARED(*(addr), (v))
8760d94e PB	33	#endif
	34
	35	#ifndef uatomic_read
6cf3827c	36	#define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
8760d94e PB	37	#endif
	38
	39	#if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR
	40	static inline __attribute__((always_inline))
	41	void _uatomic_link_error()
	42	{
	43	#ifdef ILLEGAL_INSTR
	44	/* generate an illegal instruction. Cannot catch this with linker tricks
	45	* when optimizations are disabled. */
	46	__asm__ __volatile__(ILLEGAL_INSTR);
	47	#else
	48	__builtin_trap ();
	49	#endif
	50	}
	51
	52	#else /* #if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR */
	53	extern void _uatomic_link_error ();
	54	#endif /* #else #if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR */
	55
	56	/* cmpxchg */
	57
	58	#ifndef uatomic_cmpxchg
	59	static inline __attribute__((always_inline))
	60	unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
	61	unsigned long _new, int len)
	62	{
	63	switch (len) {
f469d839 PB	64	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	65	case 1:
	66	return __sync_val_compare_and_swap_1(addr, old, _new);
	67	#endif
	68	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	69	case 2:
	70	return __sync_val_compare_and_swap_2(addr, old, _new);
	71	#endif
8760d94e PB	72	case 4:
8760d94e PB	73	return __sync_val_compare_and_swap_4(addr, old, _new);
b39e1761	74	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	75	case 8:
	76	return __sync_val_compare_and_swap_8(addr, old, _new);
	77	#endif
	78	}
	79	_uatomic_link_error();
	80	return 0;
	81	}
	82
	83
	84	#define uatomic_cmpxchg(addr, old, _new) \
	85	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
	86	(unsigned long)(_new), \
	87	sizeof(*(addr))))
	88
	89
bf33aaea PB	90	/* uatomic_and */
	91
	92	#ifndef uatomic_and
	93	static inline __attribute__((always_inline))
	94	void _uatomic_and(void *addr, unsigned long val,
	95	int len)
	96	{
	97	switch (len) {
	98	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	99	case 1:
	100	__sync_and_and_fetch_1(addr, val);
	101	#endif
	102	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	103	case 2:
	104	__sync_and_and_fetch_2(addr, val);
	105	#endif
	106	case 4:
	107	__sync_and_and_fetch_4(addr, val);
	108	#if (CAA_BITS_PER_LONG == 64)
	109	case 8:
	110	__sync_and_and_fetch_8(addr, val);
	111	#endif
	112	}
	113	_uatomic_link_error();
	114	return 0;
	115	}
	116
	117	#define uatomic_and(addr, v) \
	118	(_uatomic_and((addr), \
	119	(unsigned long)(v), \
	120	sizeof(*(addr))))
	121	#endif
	122
985b35b1 PB	123	/* uatomic_or */
	124
	125	#ifndef uatomic_or
	126	static inline __attribute__((always_inline))
	127	void _uatomic_or(void *addr, unsigned long val,
	128	int len)
	129	{
	130	switch (len) {
	131	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	132	case 1:
	133	__sync_or_and_fetch_1(addr, val);
	134	#endif
	135	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	136	case 2:
	137	__sync_or_and_fetch_2(addr, val);
	138	#endif
	139	case 4:
	140	__sync_or_and_fetch_4(addr, val);
	141	#if (CAA_BITS_PER_LONG == 64)
	142	case 8:
	143	__sync_or_and_fetch_8(addr, val);
	144	#endif
	145	}
	146	_uatomic_link_error();
	147	return 0;
	148	}
	149
	150	#define uatomic_or(addr, v) \
	151	(_uatomic_or((addr), \
	152	(unsigned long)(v), \
	153	sizeof(*(addr))))
	154	#endif
	155
8760d94e PB	156	/* uatomic_add_return */
	157
	158	#ifndef uatomic_add_return
	159	static inline __attribute__((always_inline))
	160	unsigned long _uatomic_add_return(void *addr, unsigned long val,
	161	int len)
	162	{
	163	switch (len) {
f469d839 PB	164	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	165	case 1:
	166	return __sync_add_and_fetch_1(addr, val);
	167	#endif
	168	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	169	case 2:
	170	return __sync_add_and_fetch_2(addr, val);
	171	#endif
8760d94e PB	172	case 4:
8760d94e PB	173	return __sync_add_and_fetch_4(addr, val);
b39e1761	174	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	175	case 8:
	176	return __sync_add_and_fetch_8(addr, val);
	177	#endif
	178	}
	179	_uatomic_link_error();
	180	return 0;
	181	}
	182
	183
	184	#define uatomic_add_return(addr, v) \
	185	((__typeof__(*(addr))) _uatomic_add_return((addr), \
	186	(unsigned long)(v), \
	187	sizeof(*(addr))))
	188	#endif /* #ifndef uatomic_add_return */
	189
	190	#ifndef uatomic_xchg
	191	/* xchg */
	192
	193	static inline __attribute__((always_inline))
	194	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
	195	{
	196	switch (len) {
f469d839 PB	197	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	198	case 1:
	199	{
	200	unsigned char old;
	201
	202	do {
	203	old = uatomic_read((unsigned char *)addr);
	204	} while (!__sync_bool_compare_and_swap_1(addr, old, val));
	205
	206	return old;
	207	}
	208	#endif
	209	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	210	case 2:
	211	{
	212	unsigned short old;
	213
	214	do {
	215	old = uatomic_read((unsigned short *)addr);
	216	} while (!__sync_bool_compare_and_swap_2(addr, old, val));
	217
	218	return old;
	219	}
	220	#endif
8760d94e PB	221	case 4:
	222	{
	223	unsigned int old;
	224
	225	do {
	226	old = uatomic_read((unsigned int *)addr);
2f2908d0	227	} while (!__sync_bool_compare_and_swap_4(addr, old, val));
8760d94e	228
2f2908d0	229	return old;
8760d94e	230	}
b39e1761	231	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	232	case 8:
	233	{
	234	unsigned long old;
	235
	236	do {
	237	old = uatomic_read((unsigned long *)addr);
	238	} while (!__sync_bool_compare_and_swap_8(addr, old, val));
	239
	240	return old;
	241	}
	242	#endif
	243	}
	244	_uatomic_link_error();
	245	return 0;
	246	}
	247
	248	#define uatomic_xchg(addr, v) \
	249	((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
	250	sizeof(*(addr))))
	251	#endif /* #ifndef uatomic_xchg */
	252
	253	#else /* #ifndef uatomic_cmpxchg */
	254
bf33aaea PB	255	#ifndef uatomic_and
	256	/* uatomic_and */
	257
	258	static inline __attribute__((always_inline))
	259	void _uatomic_and(void *addr, unsigned long val, int len)
	260	{
	261	switch (len) {
	262	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	263	case 1:
	264	{
	265	unsigned char old, oldt;
	266
	267	oldt = uatomic_read((unsigned char *)addr);
	268	do {
	269	old = oldt;
	270	oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
	271	} while (oldt != old);
e6e5957d MD	272
e6e5957d MD	273	return;
bf33aaea PB	274	}
	275	#endif
	276	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	277	case 2:
	278	{
	279	unsigned short old, oldt;
	280
	281	oldt = uatomic_read((unsigned short *)addr);
	282	do {
	283	old = oldt;
	284	oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
	285	} while (oldt != old);
	286	}
	287	#endif
	288	case 4:
	289	{
	290	unsigned int old, oldt;
	291
	292	oldt = uatomic_read((unsigned int *)addr);
	293	do {
	294	old = oldt;
	295	oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
	296	} while (oldt != old);
e6e5957d MD	297
e6e5957d MD	298	return;
bf33aaea PB	299	}
	300	#if (CAA_BITS_PER_LONG == 64)
	301	case 8:
	302	{
	303	unsigned long old, oldt;
	304
	305	oldt = uatomic_read((unsigned long *)addr);
	306	do {
	307	old = oldt;
	308	oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
	309	} while (oldt != old);
e6e5957d MD	310
e6e5957d MD	311	return;
bf33aaea PB	312	}
	313	#endif
	314	}
	315	_uatomic_link_error();
bf33aaea PB	316	}
	317
	318	#define uatomic_and(addr, v) \
	319	(uatomic_and((addr), \
	320	(unsigned long)(v), \
	321	sizeof(*(addr))))
	322	#endif /* #ifndef uatomic_and */
	323
985b35b1 PB	324	#ifndef uatomic_or
	325	/* uatomic_or */
	326
	327	static inline __attribute__((always_inline))
	328	void _uatomic_or(void *addr, unsigned long val, int len)
	329	{
	330	switch (len) {
	331	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	332	case 1:
	333	{
	334	unsigned char old, oldt;
	335
	336	oldt = uatomic_read((unsigned char *)addr);
	337	do {
	338	old = oldt;
	339	oldt = _uatomic_cmpxchg(addr, old, old \| val, 1);
	340	} while (oldt != old);
e6e5957d MD	341
e6e5957d MD	342	return;
985b35b1 PB	343	}
	344	#endif
	345	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	346	case 2:
	347	{
	348	unsigned short old, oldt;
	349
	350	oldt = uatomic_read((unsigned short *)addr);
	351	do {
	352	old = oldt;
	353	oldt = _uatomic_cmpxchg(addr, old, old \| val, 2);
	354	} while (oldt != old);
e6e5957d MD	355
e6e5957d MD	356	return;
985b35b1 PB	357	}
	358	#endif
	359	case 4:
	360	{
	361	unsigned int old, oldt;
	362
	363	oldt = uatomic_read((unsigned int *)addr);
	364	do {
	365	old = oldt;
	366	oldt = _uatomic_cmpxchg(addr, old, old \| val, 4);
	367	} while (oldt != old);
e6e5957d MD	368
e6e5957d MD	369	return;
985b35b1 PB	370	}
	371	#if (CAA_BITS_PER_LONG == 64)
	372	case 8:
	373	{
	374	unsigned long old, oldt;
	375
	376	oldt = uatomic_read((unsigned long *)addr);
	377	do {
	378	old = oldt;
	379	oldt = _uatomic_cmpxchg(addr, old, old \| val, 8);
	380	} while (oldt != old);
e6e5957d MD	381
e6e5957d MD	382	return;
985b35b1 PB	383	}
	384	#endif
	385	}
	386	_uatomic_link_error();
985b35b1 PB	387	}
	388
	389	#define uatomic_or(addr, v) \
	390	(uatomic_or((addr), \
	391	(unsigned long)(v), \
	392	sizeof(*(addr))))
	393	#endif /* #ifndef uatomic_or */
	394
8760d94e PB	395	#ifndef uatomic_add_return
	396	/* uatomic_add_return */
	397
	398	static inline __attribute__((always_inline))
	399	unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
	400	{
	401	switch (len) {
f469d839 PB	402	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	403	case 1:
	404	{
	405	unsigned char old, oldt;
	406
	407	oldt = uatomic_read((unsigned char *)addr);
	408	do {
	409	old = oldt;
b4e6d540 PB	410	oldt = uatomic_cmpxchg((unsigned char *)addr,
b4e6d540 PB	411	old, old + val);
f469d839 PB	412	} while (oldt != old);
	413
	414	return old + val;
	415	}
	416	#endif
	417	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	418	case 2:
	419	{
	420	unsigned short old, oldt;
	421
	422	oldt = uatomic_read((unsigned short *)addr);
	423	do {
	424	old = oldt;
b4e6d540 PB	425	oldt = uatomic_cmpxchg((unsigned short *)addr,
b4e6d540 PB	426	old, old + val);
f469d839 PB	427	} while (oldt != old);
	428
	429	return old + val;
	430	}
	431	#endif
8760d94e PB	432	case 4:
	433	{
	434	unsigned int old, oldt;
	435
	436	oldt = uatomic_read((unsigned int *)addr);
	437	do {
	438	old = oldt;
b4e6d540 PB	439	oldt = uatomic_cmpxchg((unsigned int *)addr,
b4e6d540 PB	440	old, old + val);
8760d94e PB	441	} while (oldt != old);
	442
	443	return old + val;
	444	}
b39e1761	445	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	446	case 8:
	447	{
	448	unsigned long old, oldt;
	449
	450	oldt = uatomic_read((unsigned long *)addr);
	451	do {
	452	old = oldt;
b4e6d540 PB	453	oldt = uatomic_cmpxchg((unsigned long *)addr,
b4e6d540 PB	454	old, old + val);
8760d94e PB	455	} while (oldt != old);
	456
	457	return old + val;
	458	}
	459	#endif
	460	}
	461	_uatomic_link_error();
	462	return 0;
	463	}
	464
	465	#define uatomic_add_return(addr, v) \
	466	((__typeof__(*(addr))) _uatomic_add_return((addr), \
	467	(unsigned long)(v), \
	468	sizeof(*(addr))))
	469	#endif /* #ifndef uatomic_add_return */
	470
	471	#ifndef uatomic_xchg
	472	/* xchg */
	473
	474	static inline __attribute__((always_inline))
	475	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
	476	{
	477	switch (len) {
f469d839 PB	478	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	479	case 1:
	480	{
	481	unsigned char old, oldt;
	482
	483	oldt = uatomic_read((unsigned char *)addr);
	484	do {
	485	old = oldt;
b4e6d540 PB	486	oldt = uatomic_cmpxchg((unsigned char *)addr,
b4e6d540 PB	487	old, val);
f469d839 PB	488	} while (oldt != old);
	489
	490	return old;
	491	}
	492	#endif
	493	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	494	case 2:
	495	{
	496	unsigned short old, oldt;
	497
	498	oldt = uatomic_read((unsigned short *)addr);
	499	do {
	500	old = oldt;
b4e6d540 PB	501	oldt = uatomic_cmpxchg((unsigned short *)addr,
b4e6d540 PB	502	old, val);
f469d839 PB	503	} while (oldt != old);
	504
	505	return old;
	506	}
	507	#endif
8760d94e PB	508	case 4:
	509	{
	510	unsigned int old, oldt;
	511
	512	oldt = uatomic_read((unsigned int *)addr);
	513	do {
	514	old = oldt;
b4e6d540 PB	515	oldt = uatomic_cmpxchg((unsigned int *)addr,
b4e6d540 PB	516	old, val);
8760d94e PB	517	} while (oldt != old);
	518
	519	return old;
	520	}
b39e1761	521	#if (CAA_BITS_PER_LONG == 64)
8760d94e PB	522	case 8:
	523	{
	524	unsigned long old, oldt;
	525
	526	oldt = uatomic_read((unsigned long *)addr);
	527	do {
	528	old = oldt;
b4e6d540 PB	529	oldt = uatomic_cmpxchg((unsigned long *)addr,
b4e6d540 PB	530	old, val);
8760d94e PB	531	} while (oldt != old);
	532
	533	return old;
	534	}
	535	#endif
	536	}
	537	_uatomic_link_error();
	538	return 0;
	539	}
	540
	541	#define uatomic_xchg(addr, v) \
	542	((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
	543	sizeof(*(addr))))
	544	#endif /* #ifndef uatomic_xchg */
	545
	546	#endif /* #else #ifndef uatomic_cmpxchg */
	547
	548	/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
	549
	550	#ifndef uatomic_add
	551	#define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
	552	#endif
	553
	554	#define uatomic_sub_return(addr, v) uatomic_add_return((addr), -(v))
	555	#define uatomic_sub(addr, v) uatomic_add((addr), -(v))
	556
	557	#ifndef uatomic_inc
	558	#define uatomic_inc(addr) uatomic_add((addr), 1)
	559	#endif
	560
	561	#ifndef uatomic_dec
	562	#define uatomic_dec(addr) uatomic_add((addr), -1)
	563	#endif
	564
	565	#ifdef __cplusplus
	566	}
	567	#endif
	568
	569	#endif /* _URCU_UATOMIC_GENERIC_H */