[urcu.git] / urcu / uatomic_arch_x86.h

#ifndef _URCU_ARCH_UATOMIC_X86_H
#define _URCU_ARCH_UATOMIC_X86_H

/* 
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <urcu/compiler.h>
#include <urcu/system.h>

#ifdef __cplusplus
extern "C" {
#endif 

#ifndef __SIZEOF_LONG__
#if defined(__x86_64__) || defined(__amd64__)
#define __SIZEOF_LONG__ 8
#else
#define __SIZEOF_LONG__ 4
#endif
#endif

#ifndef BITS_PER_LONG
#define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
#endif

/*
 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
 */

struct __uatomic_dummy {
	unsigned long v[10];
};
#define __hp(x)	((struct __uatomic_dummy *)(x))

#define _uatomic_set(addr, v)	STORE_SHARED(*(addr), (v))

/* cmpxchg */

static inline __attribute__((always_inline))
unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = old;

		__asm__ __volatile__(
		"lock; cmpxchgb %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "q"((unsigned char)_new)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result = old;

		__asm__ __volatile__(
		"lock; cmpxchgw %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned short)_new)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result = old;

		__asm__ __volatile__(
		"lock; cmpxchgl %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned int)_new)
			: "memory");
		return result;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = old;

		__asm__ __volatile__(
		"lock; cmpxchgq %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned long)_new)
			: "memory");
		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_cmpxchg(addr, old, _new)				      \
	((__typeof__(*(addr))) __uatomic_cmpxchg((addr), (unsigned long)(old),\
						(unsigned long)(_new), 	      \
						sizeof(*(addr))))

/* xchg */

static inline __attribute__((always_inline))
unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
{
	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	switch (len) {
	case 1:
	{
		unsigned char result;
		__asm__ __volatile__(
		"xchgb %0, %1"
			: "=q"(result), "+m"(*__hp(addr))
			: "0" ((unsigned char)val)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result;
		__asm__ __volatile__(
		"xchgw %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned short)val)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result;
		__asm__ __volatile__(
		"xchgl %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned int)val)
			: "memory");
		return result;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result;
		__asm__ __volatile__(
		"xchgq %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned long)val)
			: "memory");
		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_xchg(addr, v)						      \
	((__typeof__(*(addr))) __uatomic_exchange((addr), (unsigned long)(v), \
						sizeof(*(addr))))

/* uatomic_add_return */

static inline __attribute__((always_inline))
unsigned long __uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = val;

		__asm__ __volatile__(
		"lock; xaddb %1, %0"
			: "+m"(*__hp(addr)), "+q" (result)
			:
			: "memory");
		return result + (unsigned char)val;
	}
	case 2:
	{
		unsigned short result = val;

		__asm__ __volatile__(
		"lock; xaddw %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned short)val;
	}
	case 4:
	{
		unsigned int result = val;

		__asm__ __volatile__(
		"lock; xaddl %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned int)val;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = val;

		__asm__ __volatile__(
		"lock; xaddq %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned long)val;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_add_return(addr, v)					\
	((__typeof__(*(addr))) __uatomic_add_return((addr),		\
						  (unsigned long)(v),	\
						  sizeof(*(addr))))

/* uatomic_add */

static inline __attribute__((always_inline))
void __uatomic_add(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; addb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; addw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; addl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; addq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_add(addr, v)						   \
	(__uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))


/* uatomic_inc */

static inline __attribute__((always_inline))
void __uatomic_inc(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; incb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; incw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; incl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; incq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_inc(addr)	(__uatomic_inc((addr), sizeof(*(addr))))

/* uatomic_dec */

static inline __attribute__((always_inline))
void __uatomic_dec(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; decb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; decw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; decl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; decq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_dec(addr)	(__uatomic_dec((addr), sizeof(*(addr))))

#if ((BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
extern int __rcu_cas_avail;
extern int __rcu_cas_init(void);

#define UATOMIC_COMPAT(insn)							\
	((likely(__rcu_cas_avail > 0))						\
	? (_uatomic_##insn)							\
		: ((unlikely(__rcu_cas_avail < 0)				\
			? ((__rcu_cas_init() > 0)				\
				? (_uatomic_##insn)				\
				: (compat_uatomic_##insn))			\
			: (compat_uatomic_##insn))))

extern unsigned long _compat_uatomic_set(void *addr,
					 unsigned long _new, int len);
#define compat_uatomic_set(addr, _new)				     	       \
	((__typeof__(*(addr))) _compat_uatomic_set((addr),		       \
						(unsigned long)(_new), 	       \
						sizeof(*(addr))))


extern unsigned long _compat_uatomic_xchg(void *addr,
					  unsigned long _new, int len);
#define compat_uatomic_xchg(addr, _new)					       \
	((__typeof__(*(addr))) _compat_uatomic_xchg((addr),		       \
						(unsigned long)(_new), 	       \
						sizeof(*(addr))))

extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
					     unsigned long _new, int len);
#define compat_uatomic_cmpxchg(addr, old, _new)				       \
	((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),		       \
						(unsigned long)(old),	       \
						(unsigned long)(_new), 	       \
						sizeof(*(addr))))

extern unsigned long _compat_uatomic_xchg(void *addr,
					  unsigned long _new, int len);
#define compat_uatomic_add_return(addr, v)				       \
	((__typeof__(*(addr))) _compat_uatomic_add_return((addr),	       \
						(unsigned long)(v), 	       \
						sizeof(*(addr))))

#define compat_uatomic_add(addr, v)					       \
		((void)compat_uatomic_add_return((addr), (v)))
#define compat_uatomic_inc(addr)					       \
		(compat_uatomic_add((addr), 1))
#define compat_uatomic_dec(addr)					       \
		(compat_uatomic_add((addr), -1))

#else
#define UATOMIC_COMPAT(insn)	(_uatomic_##insn)
#endif

/* Read is atomic even in compat mode */
#define uatomic_set(addr, v)			\
		UATOMIC_COMPAT(set(addr, v))

#define uatomic_cmpxchg(addr, old, _new)	\
		UATOMIC_COMPAT(cmpxchg(addr, old, _new))
#define uatomic_xchg(addr, v)			\
		UATOMIC_COMPAT(xchg(addr, v))
#define uatomic_add_return(addr, v)		\
		UATOMIC_COMPAT(add_return(addr, v))

#define uatomic_add(addr, v)	UATOMIC_COMPAT(add(addr, v))
#define uatomic_inc(addr)	UATOMIC_COMPAT(inc(addr))
#define uatomic_dec(addr)	UATOMIC_COMPAT(dec(addr))

#ifdef __cplusplus 
}
#endif

#include <urcu/uatomic_generic.h>

#endif /* _URCU_ARCH_UATOMIC_X86_H */
Commit	Line	Data
ec4e58a3 MD	1	#ifndef _URCU_ARCH_UATOMIC_X86_H
ec4e58a3 MD	2	#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f MD	3
	4	/*
	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	*
	10	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	11	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	12	*
	13	* Permission is hereby granted to use or copy this program
	14	* for any purpose, provided the above notices are retained on all copies.
	15	* Permission to modify the code and to distribute modified code is granted,
	16	* provided the above notices are retained, and a notice that the code was
	17	* modified is included with the above copyright notice.
	18	*
ec4e58a3	19	* Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f MD	20	* Boehm-Demers-Weiser conservative garbage collector.
	21	*/
	22
ec4e58a3	23	#include <urcu/compiler.h>
bf9de1b7	24	#include <urcu/system.h>
0fad128b	25
36bc70a8 MD	26	#ifdef __cplusplus
	27	extern "C" {
	28	#endif
	29
618c4ae1 MD	30	#ifndef __SIZEOF_LONG__
	31	#if defined(__x86_64__) \|\| defined(__amd64__)
	32	#define __SIZEOF_LONG__ 8
	33	#else
	34	#define __SIZEOF_LONG__ 4
	35	#endif
	36	#endif
	37
0114ba7f MD	38	#ifndef BITS_PER_LONG
	39	#define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
	40	#endif
	41
0114ba7f	42	/*
0114ba7f MD	43	* Derived from AO_compare_and_swap() and AO_test_and_set_full().
	44	*/
	45
ec4e58a3	46	struct __uatomic_dummy {
cc1be41b MD	47	unsigned long v[10];
cc1be41b MD	48	};
ec4e58a3	49	#define __hp(x) ((struct __uatomic_dummy *)(x))
cc1be41b	50
bf9de1b7	51	#define _uatomic_set(addr, v) STORE_SHARED(*(addr), (v))
0fad128b	52
cc1be41b MD	53	/* cmpxchg */
cc1be41b MD	54
5dba80f9	55	static inline __attribute__((always_inline))
bf9de1b7	56	unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b	57	unsigned long _new, int len)
0114ba7f	58	{
cc1be41b MD	59	switch (len) {
	60	case 1:
	61	{
	62	unsigned char result = old;
0fad128b	63
cc1be41b MD	64	__asm__ __volatile__(
	65	"lock; cmpxchgb %2, %1"
	66	: "+a"(result), "+m"(*__hp(addr))
	67	: "q"((unsigned char)_new)
0114ba7f	68	: "memory");
cc1be41b MD	69	return result;
	70	}
	71	case 2:
	72	{
	73	unsigned short result = old;
0fad128b	74
cc1be41b MD	75	__asm__ __volatile__(
	76	"lock; cmpxchgw %2, %1"
	77	: "+a"(result), "+m"(*__hp(addr))
	78	: "r"((unsigned short)_new)
	79	: "memory");
	80	return result;
	81	}
	82	case 4:
	83	{
	84	unsigned int result = old;
0fad128b	85
cc1be41b MD	86	__asm__ __volatile__(
	87	"lock; cmpxchgl %2, %1"
	88	: "+a"(result), "+m"(*__hp(addr))
	89	: "r"((unsigned int)_new)
	90	: "memory");
	91	return result;
	92	}
	93	#if (BITS_PER_LONG == 64)
	94	case 8:
	95	{
6edb297e	96	unsigned long result = old;
0fad128b	97
cc1be41b	98	__asm__ __volatile__(
2c5e5fb3	99	"lock; cmpxchgq %2, %1"
cc1be41b MD	100	: "+a"(result), "+m"(*__hp(addr))
	101	: "r"((unsigned long)_new)
	102	: "memory");
	103	return result;
	104	}
	105	#endif
	106	}
	107	/* generate an illegal instruction. Cannot catch this with linker tricks
	108	* when optimizations are disabled. */
	109	__asm__ __volatile__("ud2");
	110	return 0;
0114ba7f MD	111	}
0114ba7f MD	112
bf9de1b7 MD	113	#define _uatomic_cmpxchg(addr, old, _new) \
	114	((__typeof__(*(addr))) __uatomic_cmpxchg((addr), (unsigned long)(old),\
	115	(unsigned long)(_new), \
cc1be41b MD	116	sizeof(*(addr))))
	117
	118	/* xchg */
0114ba7f	119
5dba80f9	120	static inline __attribute__((always_inline))
bf9de1b7	121	unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f	122	{
cc1be41b MD	123	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	124	switch (len) {
	125	case 1:
	126	{
	127	unsigned char result;
	128	__asm__ __volatile__(
	129	"xchgb %0, %1"
	130	: "=q"(result), "+m"(*__hp(addr))
	131	: "0" ((unsigned char)val)
	132	: "memory");
	133	return result;
	134	}
	135	case 2:
	136	{
	137	unsigned short result;
	138	__asm__ __volatile__(
	139	"xchgw %0, %1"
	140	: "=r"(result), "+m"(*__hp(addr))
	141	: "0" ((unsigned short)val)
	142	: "memory");
	143	return result;
	144	}
	145	case 4:
	146	{
	147	unsigned int result;
	148	__asm__ __volatile__(
	149	"xchgl %0, %1"
	150	: "=r"(result), "+m"(*__hp(addr))
	151	: "0" ((unsigned int)val)
	152	: "memory");
	153	return result;
	154	}
	155	#if (BITS_PER_LONG == 64)
	156	case 8:
	157	{
	158	unsigned long result;
	159	__asm__ __volatile__(
0114ba7f	160	"xchgq %0, %1"
cc1be41b MD	161	: "=r"(result), "+m"(*__hp(addr))
cc1be41b MD	162	: "0" ((unsigned long)val)
0114ba7f	163	: "memory");
cc1be41b MD	164	return result;
	165	}
	166	#endif
	167	}
	168	/* generate an illegal instruction. Cannot catch this with linker tricks
	169	* when optimizations are disabled. */
	170	__asm__ __volatile__("ud2");
	171	return 0;
0114ba7f MD	172	}
0114ba7f MD	173
bf9de1b7 MD	174	#define _uatomic_xchg(addr, v) \
bf9de1b7 MD	175	((__typeof__(*(addr))) __uatomic_exchange((addr), (unsigned long)(v), \
cc1be41b MD	176	sizeof(*(addr))))
cc1be41b MD	177
8760d94e	178	/* uatomic_add_return */
0fad128b MD	179
0fad128b MD	180	static inline __attribute__((always_inline))
bf9de1b7	181	unsigned long __uatomic_add_return(void *addr, unsigned long val,
0fad128b MD	182	int len)
	183	{
	184	switch (len) {
	185	case 1:
	186	{
	187	unsigned char result = val;
	188
	189	__asm__ __volatile__(
	190	"lock; xaddb %1, %0"
	191	: "+m"(*__hp(addr)), "+q" (result)
	192	:
	193	: "memory");
	194	return result + (unsigned char)val;
	195	}
	196	case 2:
	197	{
	198	unsigned short result = val;
	199
	200	__asm__ __volatile__(
	201	"lock; xaddw %1, %0"
	202	: "+m"(*__hp(addr)), "+r" (result)
	203	:
	204	: "memory");
	205	return result + (unsigned short)val;
	206	}
	207	case 4:
	208	{
	209	unsigned int result = val;
	210
	211	__asm__ __volatile__(
	212	"lock; xaddl %1, %0"
	213	: "+m"(*__hp(addr)), "+r" (result)
	214	:
	215	: "memory");
	216	return result + (unsigned int)val;
	217	}
	218	#if (BITS_PER_LONG == 64)
	219	case 8:
	220	{
	221	unsigned long result = val;
	222
	223	__asm__ __volatile__(
	224	"lock; xaddq %1, %0"
	225	: "+m"(*__hp(addr)), "+r" (result)
	226	:
	227	: "memory");
	228	return result + (unsigned long)val;
	229	}
	230	#endif
	231	}
	232	/* generate an illegal instruction. Cannot catch this with linker tricks
	233	* when optimizations are disabled. */
	234	__asm__ __volatile__("ud2");
	235	return 0;
	236	}
	237
bf9de1b7 MD	238	#define _uatomic_add_return(addr, v) \
bf9de1b7 MD	239	((__typeof__(*(addr))) __uatomic_add_return((addr), \
0fad128b MD	240	(unsigned long)(v), \
	241	sizeof(*(addr))))
	242
8760d94e	243	/* uatomic_add */
0114ba7f	244
5dba80f9	245	static inline __attribute__((always_inline))
bf9de1b7	246	void __uatomic_add(void *addr, unsigned long val, int len)
0114ba7f MD	247	{
0114ba7f MD	248	switch (len) {
cc1be41b MD	249	case 1:
	250	{
	251	__asm__ __volatile__(
	252	"lock; addb %1, %0"
	253	: "=m"(*__hp(addr))
87322fe8 MD	254	: "iq" ((unsigned char)val)
87322fe8 MD	255	: "memory");
cc1be41b MD	256	return;
	257	}
	258	case 2:
	259	{
	260	__asm__ __volatile__(
	261	"lock; addw %1, %0"
	262	: "=m"(*__hp(addr))
87322fe8 MD	263	: "ir" ((unsigned short)val)
87322fe8 MD	264	: "memory");
cc1be41b MD	265	return;
	266	}
	267	case 4:
	268	{
	269	__asm__ __volatile__(
	270	"lock; addl %1, %0"
	271	: "=m"(*__hp(addr))
87322fe8 MD	272	: "ir" ((unsigned int)val)
87322fe8 MD	273	: "memory");
cc1be41b MD	274	return;
cc1be41b MD	275	}
0114ba7f	276	#if (BITS_PER_LONG == 64)
cc1be41b MD	277	case 8:
	278	{
	279	__asm__ __volatile__(
	280	"lock; addq %1, %0"
	281	: "=m"(*__hp(addr))
87322fe8 MD	282	: "er" ((unsigned long)val)
87322fe8 MD	283	: "memory");
cc1be41b MD	284	return;
cc1be41b MD	285	}
0114ba7f MD	286	#endif
	287	}
	288	/* generate an illegal instruction. Cannot catch this with linker tricks
	289	* when optimizations are disabled. */
	290	__asm__ __volatile__("ud2");
a81b8e5e	291	return;
0114ba7f MD	292	}
0114ba7f MD	293
bf9de1b7 MD	294	#define _uatomic_add(addr, v) \
bf9de1b7 MD	295	(__uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
0114ba7f	296
2c5e5fb3	297
ec4e58a3	298	/* uatomic_inc */
2c5e5fb3 MD	299
2c5e5fb3 MD	300	static inline __attribute__((always_inline))
bf9de1b7	301	void __uatomic_inc(void *addr, int len)
2c5e5fb3 MD	302	{
	303	switch (len) {
	304	case 1:
	305	{
	306	__asm__ __volatile__(
	307	"lock; incb %0"
	308	: "=m"(*__hp(addr))
	309	:
	310	: "memory");
	311	return;
	312	}
	313	case 2:
	314	{
	315	__asm__ __volatile__(
	316	"lock; incw %0"
	317	: "=m"(*__hp(addr))
	318	:
	319	: "memory");
	320	return;
	321	}
	322	case 4:
	323	{
	324	__asm__ __volatile__(
	325	"lock; incl %0"
	326	: "=m"(*__hp(addr))
	327	:
	328	: "memory");
	329	return;
	330	}
	331	#if (BITS_PER_LONG == 64)
	332	case 8:
	333	{
	334	__asm__ __volatile__(
	335	"lock; incq %0"
	336	: "=m"(*__hp(addr))
	337	:
	338	: "memory");
	339	return;
	340	}
	341	#endif
	342	}
	343	/* generate an illegal instruction. Cannot catch this with linker tricks
	344	* when optimizations are disabled. */
	345	__asm__ __volatile__("ud2");
	346	return;
	347	}
	348
bf9de1b7	349	#define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3	350
ec4e58a3	351	/* uatomic_dec */
2c5e5fb3 MD	352
2c5e5fb3 MD	353	static inline __attribute__((always_inline))
bf9de1b7	354	void __uatomic_dec(void *addr, int len)
2c5e5fb3 MD	355	{
	356	switch (len) {
	357	case 1:
	358	{
	359	__asm__ __volatile__(
	360	"lock; decb %0"
	361	: "=m"(*__hp(addr))
	362	:
	363	: "memory");
	364	return;
	365	}
	366	case 2:
	367	{
	368	__asm__ __volatile__(
	369	"lock; decw %0"
	370	: "=m"(*__hp(addr))
	371	:
	372	: "memory");
	373	return;
	374	}
	375	case 4:
	376	{
	377	__asm__ __volatile__(
	378	"lock; decl %0"
	379	: "=m"(*__hp(addr))
	380	:
	381	: "memory");
	382	return;
	383	}
	384	#if (BITS_PER_LONG == 64)
	385	case 8:
	386	{
	387	__asm__ __volatile__(
	388	"lock; decq %0"
	389	: "=m"(*__hp(addr))
	390	:
	391	: "memory");
	392	return;
	393	}
	394	#endif
	395	}
	396	/* generate an illegal instruction. Cannot catch this with linker tricks
	397	* when optimizations are disabled. */
	398	__asm__ __volatile__("ud2");
	399	return;
	400	}
	401
bf9de1b7	402	#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
0114ba7f	403
02be5561 MD	404	#if ((BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
	405	extern int __rcu_cas_avail;
	406	extern int __rcu_cas_init(void);
bf9de1b7 MD	407
bf9de1b7 MD	408	#define UATOMIC_COMPAT(insn) \
02be5561	409	((likely(__rcu_cas_avail > 0)) \
bf9de1b7	410	? (_uatomic_##insn) \
02be5561 MD	411	: ((unlikely(__rcu_cas_avail < 0) \
02be5561 MD	412	? ((__rcu_cas_init() > 0) \
bf9de1b7 MD	413	? (_uatomic_##insn) \
	414	: (compat_uatomic_##insn)) \
	415	: (compat_uatomic_##insn))))
	416
	417	extern unsigned long _compat_uatomic_set(void *addr,
	418	unsigned long _new, int len);
	419	#define compat_uatomic_set(addr, _new) \
	420	((__typeof__(*(addr))) _compat_uatomic_set((addr), \
	421	(unsigned long)(_new), \
	422	sizeof(*(addr))))
	423
	424
	425	extern unsigned long _compat_uatomic_xchg(void *addr,
	426	unsigned long _new, int len);
	427	#define compat_uatomic_xchg(addr, _new) \
	428	((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
	429	(unsigned long)(_new), \
	430	sizeof(*(addr))))
7d413817 MD	431
7d413817 MD	432	extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
bf9de1b7 MD	433	unsigned long _new, int len);
	434	#define compat_uatomic_cmpxchg(addr, old, _new) \
	435	((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
	436	(unsigned long)(old), \
	437	(unsigned long)(_new), \
	438	sizeof(*(addr))))
7d413817	439
bf9de1b7 MD	440	extern unsigned long _compat_uatomic_xchg(void *addr,
	441	unsigned long _new, int len);
	442	#define compat_uatomic_add_return(addr, v) \
	443	((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
	444	(unsigned long)(v), \
7d413817	445	sizeof(*(addr))))
bf9de1b7	446
bf9de1b7 MD	447	#define compat_uatomic_add(addr, v) \
bf9de1b7 MD	448	((void)compat_uatomic_add_return((addr), (v)))
bf9de1b7 MD	449	#define compat_uatomic_inc(addr) \
	450	(compat_uatomic_add((addr), 1))
	451	#define compat_uatomic_dec(addr) \
8760d94e	452	(compat_uatomic_add((addr), -1))
bf9de1b7 MD	453
	454	#else
	455	#define UATOMIC_COMPAT(insn) (_uatomic_##insn)
7d413817 MD	456	#endif
7d413817 MD	457
bf9de1b7	458	/* Read is atomic even in compat mode */
bf9de1b7 MD	459	#define uatomic_set(addr, v) \
bf9de1b7 MD	460	UATOMIC_COMPAT(set(addr, v))
8760d94e	461
bf9de1b7 MD	462	#define uatomic_cmpxchg(addr, old, _new) \
	463	UATOMIC_COMPAT(cmpxchg(addr, old, _new))
	464	#define uatomic_xchg(addr, v) \
	465	UATOMIC_COMPAT(xchg(addr, v))
	466	#define uatomic_add_return(addr, v) \
	467	UATOMIC_COMPAT(add_return(addr, v))
8760d94e	468
bf9de1b7	469	#define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
bf9de1b7 MD	470	#define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
	471	#define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
	472
36bc70a8 MD	473	#ifdef __cplusplus
	474	}
	475	#endif
	476
8760d94e PB	477	#include <urcu/uatomic_generic.h>
8760d94e PB	478
ec4e58a3	479	#endif /* _URCU_ARCH_UATOMIC_X86_H */