[urcu.git] / urcu / uatomic / x86.h

#ifndef _URCU_ARCH_UATOMIC_X86_H
#define _URCU_ARCH_UATOMIC_X86_H

/* 
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <urcu/compiler.h>
#include <urcu/system.h>

#define UATOMIC_HAS_ATOMIC_BYTE
#define UATOMIC_HAS_ATOMIC_SHORT

#ifdef __cplusplus
extern "C" {
#endif 

/*
 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
 */

struct __uatomic_dummy {
	unsigned long v[10];
};
#define __hp(x)	((struct __uatomic_dummy *)(x))

#define _uatomic_set(addr, v)	CMM_STORE_SHARED(*(addr), (v))

/* cmpxchg */

static inline __attribute__((always_inline))
unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = old;

		__asm__ __volatile__(
		"lock; cmpxchgb %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "q"((unsigned char)_new)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result = old;

		__asm__ __volatile__(
		"lock; cmpxchgw %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned short)_new)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result = old;

		__asm__ __volatile__(
		"lock; cmpxchgl %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned int)_new)
			: "memory");
		return result;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = old;

		__asm__ __volatile__(
		"lock; cmpxchgq %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned long)_new)
			: "memory");
		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_cmpxchg(addr, old, _new)				      \
	((__typeof__(*(addr))) __uatomic_cmpxchg((addr),		      \
						caa_cast_long_keep_sign(old), \
						caa_cast_long_keep_sign(_new),\
						sizeof(*(addr))))

/* xchg */

static inline __attribute__((always_inline))
unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
{
	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	switch (len) {
	case 1:
	{
		unsigned char result;
		__asm__ __volatile__(
		"xchgb %0, %1"
			: "=q"(result), "+m"(*__hp(addr))
			: "0" ((unsigned char)val)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result;
		__asm__ __volatile__(
		"xchgw %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned short)val)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result;
		__asm__ __volatile__(
		"xchgl %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned int)val)
			: "memory");
		return result;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result;
		__asm__ __volatile__(
		"xchgq %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned long)val)
			: "memory");
		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_xchg(addr, v)						      \
	((__typeof__(*(addr))) __uatomic_exchange((addr),		      \
						caa_cast_long_keep_sign(v),   \
						sizeof(*(addr))))

/* uatomic_add_return */

static inline __attribute__((always_inline))
unsigned long __uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = val;

		__asm__ __volatile__(
		"lock; xaddb %1, %0"
			: "+m"(*__hp(addr)), "+q" (result)
			:
			: "memory");
		return result + (unsigned char)val;
	}
	case 2:
	{
		unsigned short result = val;

		__asm__ __volatile__(
		"lock; xaddw %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned short)val;
	}
	case 4:
	{
		unsigned int result = val;

		__asm__ __volatile__(
		"lock; xaddl %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned int)val;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = val;

		__asm__ __volatile__(
		"lock; xaddq %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned long)val;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) __uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))

/* uatomic_and */

static inline __attribute__((always_inline))
void __uatomic_and(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; andb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; andw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; andl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; andq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_and(addr, v)						   \
	(__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))

/* uatomic_or */

static inline __attribute__((always_inline))
void __uatomic_or(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; orb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; orw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; orl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; orq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_or(addr, v)						   \
	(__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))

/* uatomic_add */

static inline __attribute__((always_inline))
void __uatomic_add(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; addb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; addw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; addl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; addq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_add(addr, v)						   \
	(__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))


/* uatomic_inc */

static inline __attribute__((always_inline))
void __uatomic_inc(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; incb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; incw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; incl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; incq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_inc(addr)	(__uatomic_inc((addr), sizeof(*(addr))))

/* uatomic_dec */

static inline __attribute__((always_inline))
void __uatomic_dec(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; decb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; decw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; decl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; decq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_dec(addr)	(__uatomic_dec((addr), sizeof(*(addr))))

#if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
extern int __rcu_cas_avail;
extern int __rcu_cas_init(void);

#define UATOMIC_COMPAT(insn)							\
	((caa_likely(__rcu_cas_avail > 0))						\
	? (_uatomic_##insn)							\
		: ((caa_unlikely(__rcu_cas_avail < 0)				\
			? ((__rcu_cas_init() > 0)				\
				? (_uatomic_##insn)				\
				: (compat_uatomic_##insn))			\
			: (compat_uatomic_##insn))))

extern unsigned long _compat_uatomic_set(void *addr,
					 unsigned long _new, int len);
#define compat_uatomic_set(addr, _new)				     	       \
	((__typeof__(*(addr))) _compat_uatomic_set((addr),		       \
						caa_cast_long_keep_sign(_new), \
						sizeof(*(addr))))


extern unsigned long _compat_uatomic_xchg(void *addr,
					  unsigned long _new, int len);
#define compat_uatomic_xchg(addr, _new)					       \
	((__typeof__(*(addr))) _compat_uatomic_xchg((addr),		       \
						caa_cast_long_keep_sign(_new), \
						sizeof(*(addr))))

extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
					     unsigned long _new, int len);
#define compat_uatomic_cmpxchg(addr, old, _new)				       \
	((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),		       \
						caa_cast_long_keep_sign(old),  \
						caa_cast_long_keep_sign(_new), \
						sizeof(*(addr))))

extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
#define compat_uatomic_and(addr, v)				       \
	(_compat_uatomic_and((addr),				       \
			caa_cast_long_keep_sign(v),		       \
			sizeof(*(addr))))

extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
#define compat_uatomic_or(addr, v)				       \
	(_compat_uatomic_or((addr),				       \
			  caa_cast_long_keep_sign(v),		       \
			  sizeof(*(addr))))

extern unsigned long _compat_uatomic_add_return(void *addr,
						unsigned long _new, int len);
#define compat_uatomic_add_return(addr, v)			            \
	((__typeof__(*(addr))) _compat_uatomic_add_return((addr),     	    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))

#define compat_uatomic_add(addr, v)					       \
		((void)compat_uatomic_add_return((addr), (v)))
#define compat_uatomic_inc(addr)					       \
		(compat_uatomic_add((addr), 1))
#define compat_uatomic_dec(addr)					       \
		(compat_uatomic_add((addr), -1))

#else
#define UATOMIC_COMPAT(insn)	(_uatomic_##insn)
#endif

/* Read is atomic even in compat mode */
#define uatomic_set(addr, v)			\
		UATOMIC_COMPAT(set(addr, v))

#define uatomic_cmpxchg(addr, old, _new)	\
		UATOMIC_COMPAT(cmpxchg(addr, old, _new))
#define uatomic_xchg(addr, v)			\
		UATOMIC_COMPAT(xchg(addr, v))

#define uatomic_and(addr, v)		\
		UATOMIC_COMPAT(and(addr, v))
#define cmm_smp_mb__before_uatomic_and()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_and()		cmm_barrier()

#define uatomic_or(addr, v)		\
		UATOMIC_COMPAT(or(addr, v))
#define cmm_smp_mb__before_uatomic_or()		cmm_barrier()
#define cmm_smp_mb__after_uatomic_or()		cmm_barrier()

#define uatomic_add_return(addr, v)		\
		UATOMIC_COMPAT(add_return(addr, v))

#define uatomic_add(addr, v)	UATOMIC_COMPAT(add(addr, v))
#define cmm_smp_mb__before_uatomic_add()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_add()		cmm_barrier()

#define uatomic_inc(addr)	UATOMIC_COMPAT(inc(addr))
#define cmm_smp_mb__before_uatomic_inc()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_inc()		cmm_barrier()

#define uatomic_dec(addr)	UATOMIC_COMPAT(dec(addr))
#define cmm_smp_mb__before_uatomic_dec()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_dec()		cmm_barrier()

#ifdef __cplusplus 
}
#endif

#include <urcu/uatomic/generic.h>

#endif /* _URCU_ARCH_UATOMIC_X86_H */
Commit	Line	Data
ec4e58a3 MD	1	#ifndef _URCU_ARCH_UATOMIC_X86_H
ec4e58a3 MD	2	#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f MD	3
	4	/*
	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	*
	10	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	11	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	12	*
	13	* Permission is hereby granted to use or copy this program
	14	* for any purpose, provided the above notices are retained on all copies.
	15	* Permission to modify the code and to distribute modified code is granted,
	16	* provided the above notices are retained, and a notice that the code was
	17	* modified is included with the above copyright notice.
	18	*
ec4e58a3	19	* Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f MD	20	* Boehm-Demers-Weiser conservative garbage collector.
	21	*/
	22
ec4e58a3	23	#include <urcu/compiler.h>
bf9de1b7	24	#include <urcu/system.h>
0fad128b	25
f469d839 PB	26	#define UATOMIC_HAS_ATOMIC_BYTE
	27	#define UATOMIC_HAS_ATOMIC_SHORT
	28
36bc70a8 MD	29	#ifdef __cplusplus
	30	extern "C" {
	31	#endif
	32
0114ba7f	33	/*
0114ba7f MD	34	* Derived from AO_compare_and_swap() and AO_test_and_set_full().
	35	*/
	36
ec4e58a3	37	struct __uatomic_dummy {
cc1be41b MD	38	unsigned long v[10];
cc1be41b MD	39	};
ec4e58a3	40	#define __hp(x) ((struct __uatomic_dummy *)(x))
cc1be41b	41
6cf3827c	42	#define _uatomic_set(addr, v) CMM_STORE_SHARED(*(addr), (v))
0fad128b	43
cc1be41b MD	44	/* cmpxchg */
cc1be41b MD	45
5dba80f9	46	static inline __attribute__((always_inline))
bf9de1b7	47	unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b	48	unsigned long _new, int len)
0114ba7f	49	{
cc1be41b MD	50	switch (len) {
	51	case 1:
	52	{
	53	unsigned char result = old;
0fad128b	54
cc1be41b MD	55	__asm__ __volatile__(
	56	"lock; cmpxchgb %2, %1"
	57	: "+a"(result), "+m"(*__hp(addr))
	58	: "q"((unsigned char)_new)
0114ba7f	59	: "memory");
cc1be41b MD	60	return result;
	61	}
	62	case 2:
	63	{
	64	unsigned short result = old;
0fad128b	65
cc1be41b MD	66	__asm__ __volatile__(
	67	"lock; cmpxchgw %2, %1"
	68	: "+a"(result), "+m"(*__hp(addr))
	69	: "r"((unsigned short)_new)
	70	: "memory");
	71	return result;
	72	}
	73	case 4:
	74	{
	75	unsigned int result = old;
0fad128b	76
cc1be41b MD	77	__asm__ __volatile__(
	78	"lock; cmpxchgl %2, %1"
	79	: "+a"(result), "+m"(*__hp(addr))
	80	: "r"((unsigned int)_new)
	81	: "memory");
	82	return result;
	83	}
e040d717	84	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	85	case 8:
cc1be41b MD	86	{
6edb297e	87	unsigned long result = old;
0fad128b	88
cc1be41b	89	__asm__ __volatile__(
2c5e5fb3	90	"lock; cmpxchgq %2, %1"
cc1be41b MD	91	: "+a"(result), "+m"(*__hp(addr))
	92	: "r"((unsigned long)_new)
	93	: "memory");
	94	return result;
	95	}
	96	#endif
	97	}
	98	/* generate an illegal instruction. Cannot catch this with linker tricks
	99	* when optimizations are disabled. */
	100	__asm__ __volatile__("ud2");
	101	return 0;
0114ba7f MD	102	}
0114ba7f MD	103
bf9de1b7	104	#define _uatomic_cmpxchg(addr, old, _new) \
e56d99bf MD	105	((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
	106	caa_cast_long_keep_sign(old), \
	107	caa_cast_long_keep_sign(_new),\
cc1be41b MD	108	sizeof(*(addr))))
	109
	110	/* xchg */
0114ba7f	111
5dba80f9	112	static inline __attribute__((always_inline))
bf9de1b7	113	unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f	114	{
cc1be41b MD	115	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	116	switch (len) {
	117	case 1:
	118	{
	119	unsigned char result;
	120	__asm__ __volatile__(
	121	"xchgb %0, %1"
	122	: "=q"(result), "+m"(*__hp(addr))
	123	: "0" ((unsigned char)val)
	124	: "memory");
	125	return result;
	126	}
	127	case 2:
	128	{
	129	unsigned short result;
	130	__asm__ __volatile__(
	131	"xchgw %0, %1"
	132	: "=r"(result), "+m"(*__hp(addr))
	133	: "0" ((unsigned short)val)
	134	: "memory");
	135	return result;
	136	}
	137	case 4:
	138	{
	139	unsigned int result;
	140	__asm__ __volatile__(
	141	"xchgl %0, %1"
	142	: "=r"(result), "+m"(*__hp(addr))
	143	: "0" ((unsigned int)val)
	144	: "memory");
	145	return result;
	146	}
e040d717	147	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	148	case 8:
	149	{
	150	unsigned long result;
	151	__asm__ __volatile__(
0114ba7f	152	"xchgq %0, %1"
cc1be41b MD	153	: "=r"(result), "+m"(*__hp(addr))
cc1be41b MD	154	: "0" ((unsigned long)val)
0114ba7f	155	: "memory");
cc1be41b MD	156	return result;
	157	}
	158	#endif
	159	}
	160	/* generate an illegal instruction. Cannot catch this with linker tricks
	161	* when optimizations are disabled. */
	162	__asm__ __volatile__("ud2");
	163	return 0;
0114ba7f MD	164	}
0114ba7f MD	165
bf9de1b7	166	#define _uatomic_xchg(addr, v) \
e56d99bf MD	167	((__typeof__(*(addr))) __uatomic_exchange((addr), \
e56d99bf MD	168	caa_cast_long_keep_sign(v), \
cc1be41b MD	169	sizeof(*(addr))))
cc1be41b MD	170
8760d94e	171	/* uatomic_add_return */
0fad128b MD	172
0fad128b MD	173	static inline __attribute__((always_inline))
bf9de1b7	174	unsigned long __uatomic_add_return(void *addr, unsigned long val,
0fad128b MD	175	int len)
	176	{
	177	switch (len) {
	178	case 1:
	179	{
	180	unsigned char result = val;
	181
	182	__asm__ __volatile__(
	183	"lock; xaddb %1, %0"
	184	: "+m"(*__hp(addr)), "+q" (result)
	185	:
	186	: "memory");
	187	return result + (unsigned char)val;
	188	}
	189	case 2:
	190	{
	191	unsigned short result = val;
	192
	193	__asm__ __volatile__(
	194	"lock; xaddw %1, %0"
	195	: "+m"(*__hp(addr)), "+r" (result)
	196	:
	197	: "memory");
	198	return result + (unsigned short)val;
	199	}
	200	case 4:
	201	{
	202	unsigned int result = val;
	203
	204	__asm__ __volatile__(
	205	"lock; xaddl %1, %0"
	206	: "+m"(*__hp(addr)), "+r" (result)
	207	:
	208	: "memory");
	209	return result + (unsigned int)val;
	210	}
e040d717	211	#if (CAA_BITS_PER_LONG == 64)
0fad128b MD	212	case 8:
	213	{
	214	unsigned long result = val;
	215
	216	__asm__ __volatile__(
	217	"lock; xaddq %1, %0"
	218	: "+m"(*__hp(addr)), "+r" (result)
	219	:
	220	: "memory");
	221	return result + (unsigned long)val;
	222	}
	223	#endif
	224	}
	225	/* generate an illegal instruction. Cannot catch this with linker tricks
	226	* when optimizations are disabled. */
	227	__asm__ __volatile__("ud2");
	228	return 0;
	229	}
	230
e56d99bf MD	231	#define _uatomic_add_return(addr, v) \
	232	((__typeof__(*(addr))) __uatomic_add_return((addr), \
	233	caa_cast_long_keep_sign(v), \
	234	sizeof(*(addr))))
0fad128b	235
bf33aaea PB	236	/* uatomic_and */
	237
	238	static inline __attribute__((always_inline))
	239	void __uatomic_and(void *addr, unsigned long val, int len)
	240	{
	241	switch (len) {
	242	case 1:
	243	{
	244	__asm__ __volatile__(
	245	"lock; andb %1, %0"
	246	: "=m"(*__hp(addr))
	247	: "iq" ((unsigned char)val)
	248	: "memory");
	249	return;
	250	}
	251	case 2:
	252	{
	253	__asm__ __volatile__(
	254	"lock; andw %1, %0"
	255	: "=m"(*__hp(addr))
	256	: "ir" ((unsigned short)val)
	257	: "memory");
	258	return;
	259	}
	260	case 4:
	261	{
	262	__asm__ __volatile__(
	263	"lock; andl %1, %0"
	264	: "=m"(*__hp(addr))
	265	: "ir" ((unsigned int)val)
	266	: "memory");
	267	return;
	268	}
	269	#if (CAA_BITS_PER_LONG == 64)
	270	case 8:
	271	{
	272	__asm__ __volatile__(
	273	"lock; andq %1, %0"
	274	: "=m"(*__hp(addr))
	275	: "er" ((unsigned long)val)
	276	: "memory");
	277	return;
	278	}
	279	#endif
	280	}
	281	/* generate an illegal instruction. Cannot catch this with linker tricks
	282	* when optimizations are disabled. */
	283	__asm__ __volatile__("ud2");
	284	return;
	285	}
	286
	287	#define _uatomic_and(addr, v) \
e56d99bf	288	(__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
bf33aaea	289
985b35b1 PB	290	/* uatomic_or */
	291
	292	static inline __attribute__((always_inline))
	293	void __uatomic_or(void *addr, unsigned long val, int len)
	294	{
	295	switch (len) {
	296	case 1:
	297	{
	298	__asm__ __volatile__(
	299	"lock; orb %1, %0"
	300	: "=m"(*__hp(addr))
	301	: "iq" ((unsigned char)val)
	302	: "memory");
	303	return;
	304	}
	305	case 2:
	306	{
	307	__asm__ __volatile__(
	308	"lock; orw %1, %0"
	309	: "=m"(*__hp(addr))
	310	: "ir" ((unsigned short)val)
	311	: "memory");
	312	return;
	313	}
	314	case 4:
	315	{
	316	__asm__ __volatile__(
	317	"lock; orl %1, %0"
	318	: "=m"(*__hp(addr))
	319	: "ir" ((unsigned int)val)
	320	: "memory");
	321	return;
	322	}
	323	#if (CAA_BITS_PER_LONG == 64)
	324	case 8:
	325	{
	326	__asm__ __volatile__(
	327	"lock; orq %1, %0"
	328	: "=m"(*__hp(addr))
	329	: "er" ((unsigned long)val)
	330	: "memory");
	331	return;
	332	}
	333	#endif
	334	}
	335	/* generate an illegal instruction. Cannot catch this with linker tricks
	336	* when optimizations are disabled. */
	337	__asm__ __volatile__("ud2");
	338	return;
	339	}
	340
	341	#define _uatomic_or(addr, v) \
e56d99bf	342	(__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
985b35b1	343
8760d94e	344	/* uatomic_add */
0114ba7f	345
5dba80f9	346	static inline __attribute__((always_inline))
bf9de1b7	347	void __uatomic_add(void *addr, unsigned long val, int len)
0114ba7f MD	348	{
0114ba7f MD	349	switch (len) {
cc1be41b MD	350	case 1:
	351	{
	352	__asm__ __volatile__(
	353	"lock; addb %1, %0"
	354	: "=m"(*__hp(addr))
87322fe8 MD	355	: "iq" ((unsigned char)val)
87322fe8 MD	356	: "memory");
cc1be41b MD	357	return;
	358	}
	359	case 2:
	360	{
	361	__asm__ __volatile__(
	362	"lock; addw %1, %0"
	363	: "=m"(*__hp(addr))
87322fe8 MD	364	: "ir" ((unsigned short)val)
87322fe8 MD	365	: "memory");
cc1be41b MD	366	return;
	367	}
	368	case 4:
	369	{
	370	__asm__ __volatile__(
	371	"lock; addl %1, %0"
	372	: "=m"(*__hp(addr))
87322fe8 MD	373	: "ir" ((unsigned int)val)
87322fe8 MD	374	: "memory");
cc1be41b MD	375	return;
cc1be41b MD	376	}
e040d717	377	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	378	case 8:
	379	{
	380	__asm__ __volatile__(
	381	"lock; addq %1, %0"
	382	: "=m"(*__hp(addr))
87322fe8 MD	383	: "er" ((unsigned long)val)
87322fe8 MD	384	: "memory");
cc1be41b MD	385	return;
cc1be41b MD	386	}
0114ba7f MD	387	#endif
	388	}
	389	/* generate an illegal instruction. Cannot catch this with linker tricks
	390	* when optimizations are disabled. */
	391	__asm__ __volatile__("ud2");
a81b8e5e	392	return;
0114ba7f MD	393	}
0114ba7f MD	394
bf9de1b7	395	#define _uatomic_add(addr, v) \
e56d99bf	396	(__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
0114ba7f	397
2c5e5fb3	398
ec4e58a3	399	/* uatomic_inc */
2c5e5fb3 MD	400
2c5e5fb3 MD	401	static inline __attribute__((always_inline))
bf9de1b7	402	void __uatomic_inc(void *addr, int len)
2c5e5fb3 MD	403	{
	404	switch (len) {
	405	case 1:
	406	{
	407	__asm__ __volatile__(
	408	"lock; incb %0"
	409	: "=m"(*__hp(addr))
	410	:
	411	: "memory");
	412	return;
	413	}
	414	case 2:
	415	{
	416	__asm__ __volatile__(
	417	"lock; incw %0"
	418	: "=m"(*__hp(addr))
	419	:
	420	: "memory");
	421	return;
	422	}
	423	case 4:
	424	{
	425	__asm__ __volatile__(
	426	"lock; incl %0"
	427	: "=m"(*__hp(addr))
	428	:
	429	: "memory");
	430	return;
	431	}
e040d717	432	#if (CAA_BITS_PER_LONG == 64)
2c5e5fb3 MD	433	case 8:
	434	{
	435	__asm__ __volatile__(
	436	"lock; incq %0"
	437	: "=m"(*__hp(addr))
	438	:
	439	: "memory");
	440	return;
	441	}
	442	#endif
	443	}
	444	/* generate an illegal instruction. Cannot catch this with linker tricks
	445	* when optimizations are disabled. */
	446	__asm__ __volatile__("ud2");
	447	return;
	448	}
	449
bf9de1b7	450	#define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3	451
ec4e58a3	452	/* uatomic_dec */
2c5e5fb3 MD	453
2c5e5fb3 MD	454	static inline __attribute__((always_inline))
bf9de1b7	455	void __uatomic_dec(void *addr, int len)
2c5e5fb3 MD	456	{
	457	switch (len) {
	458	case 1:
	459	{
	460	__asm__ __volatile__(
	461	"lock; decb %0"
	462	: "=m"(*__hp(addr))
	463	:
	464	: "memory");
	465	return;
	466	}
	467	case 2:
	468	{
	469	__asm__ __volatile__(
	470	"lock; decw %0"
	471	: "=m"(*__hp(addr))
	472	:
	473	: "memory");
	474	return;
	475	}
	476	case 4:
	477	{
	478	__asm__ __volatile__(
	479	"lock; decl %0"
	480	: "=m"(*__hp(addr))
	481	:
	482	: "memory");
	483	return;
	484	}
e040d717	485	#if (CAA_BITS_PER_LONG == 64)
2c5e5fb3 MD	486	case 8:
	487	{
	488	__asm__ __volatile__(
	489	"lock; decq %0"
	490	: "=m"(*__hp(addr))
	491	:
	492	: "memory");
	493	return;
	494	}
	495	#endif
	496	}
	497	/* generate an illegal instruction. Cannot catch this with linker tricks
	498	* when optimizations are disabled. */
	499	__asm__ __volatile__("ud2");
	500	return;
	501	}
	502
bf9de1b7	503	#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
0114ba7f	504
e040d717	505	#if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
02be5561 MD	506	extern int __rcu_cas_avail;
02be5561 MD	507	extern int __rcu_cas_init(void);
bf9de1b7 MD	508
bf9de1b7 MD	509	#define UATOMIC_COMPAT(insn) \
a0b7f7ea	510	((caa_likely(__rcu_cas_avail > 0)) \
bf9de1b7	511	? (_uatomic_##insn) \
a0b7f7ea	512	: ((caa_unlikely(__rcu_cas_avail < 0) \
02be5561	513	? ((__rcu_cas_init() > 0) \
bf9de1b7 MD	514	? (_uatomic_##insn) \
	515	: (compat_uatomic_##insn)) \
	516	: (compat_uatomic_##insn))))
	517
	518	extern unsigned long _compat_uatomic_set(void *addr,
	519	unsigned long _new, int len);
	520	#define compat_uatomic_set(addr, _new) \
	521	((__typeof__(*(addr))) _compat_uatomic_set((addr), \
e56d99bf	522	caa_cast_long_keep_sign(_new), \
bf9de1b7 MD	523	sizeof(*(addr))))
	524
	525
	526	extern unsigned long _compat_uatomic_xchg(void *addr,
	527	unsigned long _new, int len);
	528	#define compat_uatomic_xchg(addr, _new) \
	529	((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
e56d99bf	530	caa_cast_long_keep_sign(_new), \
bf9de1b7	531	sizeof(*(addr))))
7d413817 MD	532
7d413817 MD	533	extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
bf9de1b7 MD	534	unsigned long _new, int len);
	535	#define compat_uatomic_cmpxchg(addr, old, _new) \
	536	((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
e56d99bf MD	537	caa_cast_long_keep_sign(old), \
e56d99bf MD	538	caa_cast_long_keep_sign(_new), \
bf9de1b7	539	sizeof(*(addr))))
7d413817	540
8c43fe72	541	extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
bf33aaea	542	#define compat_uatomic_and(addr, v) \
8c43fe72	543	(_compat_uatomic_and((addr), \
e56d99bf	544	caa_cast_long_keep_sign(v), \
8c43fe72	545	sizeof(*(addr))))
bf33aaea	546
8c43fe72	547	extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
985b35b1	548	#define compat_uatomic_or(addr, v) \
8c43fe72	549	(_compat_uatomic_or((addr), \
e56d99bf	550	caa_cast_long_keep_sign(v), \
8c43fe72	551	sizeof(*(addr))))
985b35b1	552
28ca843d PB	553	extern unsigned long _compat_uatomic_add_return(void *addr,
28ca843d PB	554	unsigned long _new, int len);
e56d99bf MD	555	#define compat_uatomic_add_return(addr, v) \
	556	((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
	557	caa_cast_long_keep_sign(v), \
	558	sizeof(*(addr))))
bf9de1b7	559
bf9de1b7 MD	560	#define compat_uatomic_add(addr, v) \
bf9de1b7 MD	561	((void)compat_uatomic_add_return((addr), (v)))
bf9de1b7 MD	562	#define compat_uatomic_inc(addr) \
	563	(compat_uatomic_add((addr), 1))
	564	#define compat_uatomic_dec(addr) \
8760d94e	565	(compat_uatomic_add((addr), -1))
bf9de1b7 MD	566
	567	#else
	568	#define UATOMIC_COMPAT(insn) (_uatomic_##insn)
7d413817 MD	569	#endif
7d413817 MD	570
bf9de1b7	571	/* Read is atomic even in compat mode */
bf9de1b7 MD	572	#define uatomic_set(addr, v) \
bf9de1b7 MD	573	UATOMIC_COMPAT(set(addr, v))
8760d94e	574
bf9de1b7 MD	575	#define uatomic_cmpxchg(addr, old, _new) \
	576	UATOMIC_COMPAT(cmpxchg(addr, old, _new))
	577	#define uatomic_xchg(addr, v) \
	578	UATOMIC_COMPAT(xchg(addr, v))
2812a2d2	579
bf33aaea PB	580	#define uatomic_and(addr, v) \
bf33aaea PB	581	UATOMIC_COMPAT(and(addr, v))
42e83919 MD	582	#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
42e83919 MD	583	#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
2812a2d2	584
985b35b1 PB	585	#define uatomic_or(addr, v) \
985b35b1 PB	586	UATOMIC_COMPAT(or(addr, v))
42e83919 MD	587	#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
42e83919 MD	588	#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
2812a2d2	589
bf9de1b7 MD	590	#define uatomic_add_return(addr, v) \
bf9de1b7 MD	591	UATOMIC_COMPAT(add_return(addr, v))
8760d94e	592
bf9de1b7	593	#define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
42e83919 MD	594	#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
42e83919 MD	595	#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
2812a2d2	596
bf9de1b7	597	#define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
42e83919 MD	598	#define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
42e83919 MD	599	#define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
2812a2d2	600
bf9de1b7	601	#define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
42e83919 MD	602	#define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
42e83919 MD	603	#define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
bf9de1b7	604
36bc70a8 MD	605	#ifdef __cplusplus
	606	}
	607	#endif
	608
a2e7bf9c	609	#include <urcu/uatomic/generic.h>
8760d94e	610
ec4e58a3	611	#endif /* _URCU_ARCH_UATOMIC_X86_H */