[urcu.git] / include / urcu / uatomic / x86.h

#ifndef _URCU_ARCH_UATOMIC_X86_H
#define _URCU_ARCH_UATOMIC_X86_H

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <urcu/config.h>
#include <urcu/compiler.h>
#include <urcu/system.h>

#define UATOMIC_HAS_ATOMIC_BYTE
#define UATOMIC_HAS_ATOMIC_SHORT

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
 */

struct __uatomic_dummy {
	unsigned long v[10];
};
#define __hp(x)	((struct __uatomic_dummy *)(x))

#define _uatomic_set(addr, v)	((void) CMM_STORE_SHARED(*(addr), (v)))

/* cmpxchg */

static inline __attribute__((always_inline))
unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = old;

		__asm__ __volatile__(
		"lock; cmpxchgb %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "q"((unsigned char)_new)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result = old;

		__asm__ __volatile__(
		"lock; cmpxchgw %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned short)_new)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result = old;

		__asm__ __volatile__(
		"lock; cmpxchgl %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned int)_new)
			: "memory");
		return result;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = old;

		__asm__ __volatile__(
		"lock; cmpxchgq %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned long)_new)
			: "memory");
		return result;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_cmpxchg(addr, old, _new)				      \
	((__typeof__(*(addr))) __uatomic_cmpxchg((addr),		      \
						caa_cast_long_keep_sign(old), \
						caa_cast_long_keep_sign(_new),\
						sizeof(*(addr))))

/* xchg */

static inline __attribute__((always_inline))
unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
{
	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	switch (len) {
	case 1:
	{
		unsigned char result;
		__asm__ __volatile__(
		"xchgb %0, %1"
			: "=q"(result), "+m"(*__hp(addr))
			: "0" ((unsigned char)val)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result;
		__asm__ __volatile__(
		"xchgw %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned short)val)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result;
		__asm__ __volatile__(
		"xchgl %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned int)val)
			: "memory");
		return result;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result;
		__asm__ __volatile__(
		"xchgq %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned long)val)
			: "memory");
		return result;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_xchg(addr, v)						      \
	((__typeof__(*(addr))) __uatomic_exchange((addr),		      \
						caa_cast_long_keep_sign(v),   \
						sizeof(*(addr))))

/* uatomic_add_return */

static inline __attribute__((always_inline))
unsigned long __uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = val;

		__asm__ __volatile__(
		"lock; xaddb %1, %0"
			: "+m"(*__hp(addr)), "+q" (result)
			:
			: "memory");
		return result + (unsigned char)val;
	}
	case 2:
	{
		unsigned short result = val;

		__asm__ __volatile__(
		"lock; xaddw %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned short)val;
	}
	case 4:
	{
		unsigned int result = val;

		__asm__ __volatile__(
		"lock; xaddl %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned int)val;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = val;

		__asm__ __volatile__(
		"lock; xaddq %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned long)val;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) __uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))

/* uatomic_and */

static inline __attribute__((always_inline))
void __uatomic_and(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; andb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; andw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; andl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; andq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_and(addr, v)						   \
	(__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))

/* uatomic_or */

static inline __attribute__((always_inline))
void __uatomic_or(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; orb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; orw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; orl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; orq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_or(addr, v)						   \
	(__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))

/* uatomic_add */

static inline __attribute__((always_inline))
void __uatomic_add(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; addb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; addw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; addl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; addq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_add(addr, v)						   \
	(__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))


/* uatomic_inc */

static inline __attribute__((always_inline))
void __uatomic_inc(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; incb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; incw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; incl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; incq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_inc(addr)	(__uatomic_inc((addr), sizeof(*(addr))))

/* uatomic_dec */

static inline __attribute__((always_inline))
void __uatomic_dec(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; decb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; decw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; decl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; decq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_dec(addr)	(__uatomic_dec((addr), sizeof(*(addr))))

#if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
extern int __rcu_cas_avail;
extern int __rcu_cas_init(void);

#define UATOMIC_COMPAT(insn)							\
	((caa_likely(__rcu_cas_avail > 0))						\
	? (_uatomic_##insn)							\
		: ((caa_unlikely(__rcu_cas_avail < 0)				\
			? ((__rcu_cas_init() > 0)				\
				? (_uatomic_##insn)				\
				: (compat_uatomic_##insn))			\
			: (compat_uatomic_##insn))))

/*
 * We leave the return value so we don't break the ABI, but remove the
 * return value from the API.
 */
extern unsigned long _compat_uatomic_set(void *addr,
					 unsigned long _new, int len);
#define compat_uatomic_set(addr, _new)				     	       \
	((void) _compat_uatomic_set((addr),				       \
				caa_cast_long_keep_sign(_new),		       \
				sizeof(*(addr))))


extern unsigned long _compat_uatomic_xchg(void *addr,
					  unsigned long _new, int len);
#define compat_uatomic_xchg(addr, _new)					       \
	((__typeof__(*(addr))) _compat_uatomic_xchg((addr),		       \
						caa_cast_long_keep_sign(_new), \
						sizeof(*(addr))))

extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
					     unsigned long _new, int len);
#define compat_uatomic_cmpxchg(addr, old, _new)				       \
	((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),		       \
						caa_cast_long_keep_sign(old),  \
						caa_cast_long_keep_sign(_new), \
						sizeof(*(addr))))

extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
#define compat_uatomic_and(addr, v)				       \
	(_compat_uatomic_and((addr),				       \
			caa_cast_long_keep_sign(v),		       \
			sizeof(*(addr))))

extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
#define compat_uatomic_or(addr, v)				       \
	(_compat_uatomic_or((addr),				       \
			  caa_cast_long_keep_sign(v),		       \
			  sizeof(*(addr))))

extern unsigned long _compat_uatomic_add_return(void *addr,
						unsigned long _new, int len);
#define compat_uatomic_add_return(addr, v)			            \
	((__typeof__(*(addr))) _compat_uatomic_add_return((addr),     	    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))

#define compat_uatomic_add(addr, v)					       \
		((void)compat_uatomic_add_return((addr), (v)))
#define compat_uatomic_inc(addr)					       \
		(compat_uatomic_add((addr), 1))
#define compat_uatomic_dec(addr)					       \
		(compat_uatomic_add((addr), -1))

#else
#define UATOMIC_COMPAT(insn)	(_uatomic_##insn)
#endif

/* Read is atomic even in compat mode */
#define uatomic_set(addr, v)			\
		UATOMIC_COMPAT(set(addr, v))

#define uatomic_cmpxchg(addr, old, _new)	\
		UATOMIC_COMPAT(cmpxchg(addr, old, _new))
#define uatomic_xchg(addr, v)			\
		UATOMIC_COMPAT(xchg(addr, v))

#define uatomic_and(addr, v)		\
		UATOMIC_COMPAT(and(addr, v))
#define cmm_smp_mb__before_uatomic_and()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_and()		cmm_barrier()

#define uatomic_or(addr, v)		\
		UATOMIC_COMPAT(or(addr, v))
#define cmm_smp_mb__before_uatomic_or()		cmm_barrier()
#define cmm_smp_mb__after_uatomic_or()		cmm_barrier()

#define uatomic_add_return(addr, v)		\
		UATOMIC_COMPAT(add_return(addr, v))

#define uatomic_add(addr, v)	UATOMIC_COMPAT(add(addr, v))
#define cmm_smp_mb__before_uatomic_add()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_add()		cmm_barrier()

#define uatomic_inc(addr)	UATOMIC_COMPAT(inc(addr))
#define cmm_smp_mb__before_uatomic_inc()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_inc()		cmm_barrier()

#define uatomic_dec(addr)	UATOMIC_COMPAT(dec(addr))
#define cmm_smp_mb__before_uatomic_dec()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_dec()		cmm_barrier()

#ifdef __cplusplus
}
#endif

#include <urcu/uatomic/generic.h>

#endif /* _URCU_ARCH_UATOMIC_X86_H */
Commit	Line	Data
ec4e58a3 MD	1	#ifndef _URCU_ARCH_UATOMIC_X86_H
ec4e58a3 MD	2	#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f	3
67ecffc0	4	/*
0114ba7f MD	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	*
	10	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	11	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	12	*
	13	* Permission is hereby granted to use or copy this program
	14	* for any purpose, provided the above notices are retained on all copies.
	15	* Permission to modify the code and to distribute modified code is granted,
	16	* provided the above notices are retained, and a notice that the code was
	17	* modified is included with the above copyright notice.
	18	*
ec4e58a3	19	* Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f MD	20	* Boehm-Demers-Weiser conservative garbage collector.
	21	*/
	22
375db287	23	#include <urcu/config.h>
ec4e58a3	24	#include <urcu/compiler.h>
bf9de1b7	25	#include <urcu/system.h>
0fad128b	26
f469d839 PB	27	#define UATOMIC_HAS_ATOMIC_BYTE
	28	#define UATOMIC_HAS_ATOMIC_SHORT
	29
36bc70a8 MD	30	#ifdef __cplusplus
36bc70a8 MD	31	extern "C" {
67ecffc0	32	#endif
36bc70a8	33
0114ba7f	34	/*
0114ba7f MD	35	* Derived from AO_compare_and_swap() and AO_test_and_set_full().
	36	*/
	37
ec4e58a3	38	struct __uatomic_dummy {
cc1be41b MD	39	unsigned long v[10];
cc1be41b MD	40	};
ec4e58a3	41	#define __hp(x) ((struct __uatomic_dummy *)(x))
cc1be41b	42
424d4ed5	43	#define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
0fad128b	44
cc1be41b MD	45	/* cmpxchg */
cc1be41b MD	46
5dba80f9	47	static inline __attribute__((always_inline))
bf9de1b7	48	unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b	49	unsigned long _new, int len)
0114ba7f	50	{
cc1be41b MD	51	switch (len) {
	52	case 1:
	53	{
	54	unsigned char result = old;
0fad128b	55
cc1be41b MD	56	__asm__ __volatile__(
	57	"lock; cmpxchgb %2, %1"
	58	: "+a"(result), "+m"(*__hp(addr))
	59	: "q"((unsigned char)_new)
0114ba7f	60	: "memory");
cc1be41b MD	61	return result;
	62	}
	63	case 2:
	64	{
	65	unsigned short result = old;
0fad128b	66
cc1be41b MD	67	__asm__ __volatile__(
	68	"lock; cmpxchgw %2, %1"
	69	: "+a"(result), "+m"(*__hp(addr))
	70	: "r"((unsigned short)_new)
	71	: "memory");
	72	return result;
	73	}
	74	case 4:
	75	{
	76	unsigned int result = old;
0fad128b	77
cc1be41b MD	78	__asm__ __volatile__(
	79	"lock; cmpxchgl %2, %1"
	80	: "+a"(result), "+m"(*__hp(addr))
	81	: "r"((unsigned int)_new)
	82	: "memory");
	83	return result;
	84	}
e040d717	85	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	86	case 8:
cc1be41b MD	87	{
6edb297e	88	unsigned long result = old;
0fad128b	89
cc1be41b	90	__asm__ __volatile__(
2c5e5fb3	91	"lock; cmpxchgq %2, %1"
cc1be41b MD	92	: "+a"(result), "+m"(*__hp(addr))
	93	: "r"((unsigned long)_new)
	94	: "memory");
	95	return result;
	96	}
	97	#endif
	98	}
d0bbd9c2 MD	99	/*
	100	* generate an illegal instruction. Cannot catch this with
	101	* linker tricks when optimizations are disabled.
	102	*/
cc1be41b MD	103	__asm__ __volatile__("ud2");
cc1be41b MD	104	return 0;
0114ba7f MD	105	}
0114ba7f MD	106
bf9de1b7	107	#define _uatomic_cmpxchg(addr, old, _new) \
e56d99bf MD	108	((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
	109	caa_cast_long_keep_sign(old), \
	110	caa_cast_long_keep_sign(_new),\
cc1be41b MD	111	sizeof(*(addr))))
	112
	113	/* xchg */
0114ba7f	114
5dba80f9	115	static inline __attribute__((always_inline))
bf9de1b7	116	unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f	117	{
cc1be41b MD	118	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	119	switch (len) {
	120	case 1:
	121	{
	122	unsigned char result;
	123	__asm__ __volatile__(
	124	"xchgb %0, %1"
	125	: "=q"(result), "+m"(*__hp(addr))
	126	: "0" ((unsigned char)val)
	127	: "memory");
	128	return result;
	129	}
	130	case 2:
	131	{
	132	unsigned short result;
	133	__asm__ __volatile__(
	134	"xchgw %0, %1"
	135	: "=r"(result), "+m"(*__hp(addr))
	136	: "0" ((unsigned short)val)
	137	: "memory");
	138	return result;
	139	}
	140	case 4:
	141	{
	142	unsigned int result;
	143	__asm__ __volatile__(
	144	"xchgl %0, %1"
	145	: "=r"(result), "+m"(*__hp(addr))
	146	: "0" ((unsigned int)val)
	147	: "memory");
	148	return result;
	149	}
e040d717	150	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	151	case 8:
	152	{
	153	unsigned long result;
	154	__asm__ __volatile__(
0114ba7f	155	"xchgq %0, %1"
cc1be41b MD	156	: "=r"(result), "+m"(*__hp(addr))
cc1be41b MD	157	: "0" ((unsigned long)val)
0114ba7f	158	: "memory");
cc1be41b MD	159	return result;
	160	}
	161	#endif
	162	}
d0bbd9c2 MD	163	/*
	164	* generate an illegal instruction. Cannot catch this with
	165	* linker tricks when optimizations are disabled.
	166	*/
cc1be41b MD	167	__asm__ __volatile__("ud2");
cc1be41b MD	168	return 0;
0114ba7f MD	169	}
0114ba7f MD	170
bf9de1b7	171	#define _uatomic_xchg(addr, v) \
e56d99bf MD	172	((__typeof__(*(addr))) __uatomic_exchange((addr), \
e56d99bf MD	173	caa_cast_long_keep_sign(v), \
cc1be41b MD	174	sizeof(*(addr))))
cc1be41b MD	175
8760d94e	176	/* uatomic_add_return */
0fad128b MD	177
0fad128b MD	178	static inline __attribute__((always_inline))
bf9de1b7	179	unsigned long __uatomic_add_return(void *addr, unsigned long val,
0fad128b MD	180	int len)
	181	{
	182	switch (len) {
	183	case 1:
	184	{
	185	unsigned char result = val;
	186
	187	__asm__ __volatile__(
	188	"lock; xaddb %1, %0"
	189	: "+m"(*__hp(addr)), "+q" (result)
	190	:
	191	: "memory");
	192	return result + (unsigned char)val;
	193	}
	194	case 2:
	195	{
	196	unsigned short result = val;
	197
	198	__asm__ __volatile__(
	199	"lock; xaddw %1, %0"
	200	: "+m"(*__hp(addr)), "+r" (result)
	201	:
	202	: "memory");
	203	return result + (unsigned short)val;
	204	}
	205	case 4:
	206	{
	207	unsigned int result = val;
	208
	209	__asm__ __volatile__(
	210	"lock; xaddl %1, %0"
	211	: "+m"(*__hp(addr)), "+r" (result)
	212	:
	213	: "memory");
	214	return result + (unsigned int)val;
	215	}
e040d717	216	#if (CAA_BITS_PER_LONG == 64)
0fad128b MD	217	case 8:
	218	{
	219	unsigned long result = val;
	220
	221	__asm__ __volatile__(
	222	"lock; xaddq %1, %0"
	223	: "+m"(*__hp(addr)), "+r" (result)
	224	:
	225	: "memory");
	226	return result + (unsigned long)val;
	227	}
	228	#endif
	229	}
d0bbd9c2 MD	230	/*
	231	* generate an illegal instruction. Cannot catch this with
	232	* linker tricks when optimizations are disabled.
	233	*/
0fad128b MD	234	__asm__ __volatile__("ud2");
	235	return 0;
	236	}
	237
e56d99bf MD	238	#define _uatomic_add_return(addr, v) \
	239	((__typeof__(*(addr))) __uatomic_add_return((addr), \
	240	caa_cast_long_keep_sign(v), \
	241	sizeof(*(addr))))
0fad128b	242
bf33aaea PB	243	/* uatomic_and */
	244
	245	static inline __attribute__((always_inline))
	246	void __uatomic_and(void *addr, unsigned long val, int len)
	247	{
	248	switch (len) {
	249	case 1:
	250	{
	251	__asm__ __volatile__(
	252	"lock; andb %1, %0"
	253	: "=m"(*__hp(addr))
	254	: "iq" ((unsigned char)val)
	255	: "memory");
	256	return;
	257	}
	258	case 2:
	259	{
	260	__asm__ __volatile__(
	261	"lock; andw %1, %0"
	262	: "=m"(*__hp(addr))
	263	: "ir" ((unsigned short)val)
	264	: "memory");
	265	return;
	266	}
	267	case 4:
	268	{
	269	__asm__ __volatile__(
	270	"lock; andl %1, %0"
	271	: "=m"(*__hp(addr))
	272	: "ir" ((unsigned int)val)
	273	: "memory");
	274	return;
	275	}
	276	#if (CAA_BITS_PER_LONG == 64)
	277	case 8:
	278	{
	279	__asm__ __volatile__(
	280	"lock; andq %1, %0"
	281	: "=m"(*__hp(addr))
	282	: "er" ((unsigned long)val)
	283	: "memory");
	284	return;
	285	}
	286	#endif
	287	}
d0bbd9c2 MD	288	/*
	289	* generate an illegal instruction. Cannot catch this with
	290	* linker tricks when optimizations are disabled.
	291	*/
bf33aaea PB	292	__asm__ __volatile__("ud2");
	293	return;
	294	}
	295
	296	#define _uatomic_and(addr, v) \
e56d99bf	297	(__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
bf33aaea	298
985b35b1 PB	299	/* uatomic_or */
	300
	301	static inline __attribute__((always_inline))
	302	void __uatomic_or(void *addr, unsigned long val, int len)
	303	{
	304	switch (len) {
	305	case 1:
	306	{
	307	__asm__ __volatile__(
	308	"lock; orb %1, %0"
	309	: "=m"(*__hp(addr))
	310	: "iq" ((unsigned char)val)
	311	: "memory");
	312	return;
	313	}
	314	case 2:
	315	{
	316	__asm__ __volatile__(
	317	"lock; orw %1, %0"
	318	: "=m"(*__hp(addr))
	319	: "ir" ((unsigned short)val)
	320	: "memory");
	321	return;
	322	}
	323	case 4:
	324	{
	325	__asm__ __volatile__(
	326	"lock; orl %1, %0"
	327	: "=m"(*__hp(addr))
	328	: "ir" ((unsigned int)val)
	329	: "memory");
	330	return;
	331	}
	332	#if (CAA_BITS_PER_LONG == 64)
	333	case 8:
	334	{
	335	__asm__ __volatile__(
	336	"lock; orq %1, %0"
	337	: "=m"(*__hp(addr))
	338	: "er" ((unsigned long)val)
	339	: "memory");
	340	return;
	341	}
	342	#endif
	343	}
d0bbd9c2 MD	344	/*
	345	* generate an illegal instruction. Cannot catch this with
	346	* linker tricks when optimizations are disabled.
	347	*/
985b35b1 PB	348	__asm__ __volatile__("ud2");
	349	return;
	350	}
	351
	352	#define _uatomic_or(addr, v) \
e56d99bf	353	(__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
985b35b1	354
8760d94e	355	/* uatomic_add */
0114ba7f	356
5dba80f9	357	static inline __attribute__((always_inline))
bf9de1b7	358	void __uatomic_add(void *addr, unsigned long val, int len)
0114ba7f MD	359	{
0114ba7f MD	360	switch (len) {
cc1be41b MD	361	case 1:
	362	{
	363	__asm__ __volatile__(
	364	"lock; addb %1, %0"
	365	: "=m"(*__hp(addr))
87322fe8 MD	366	: "iq" ((unsigned char)val)
87322fe8 MD	367	: "memory");
cc1be41b MD	368	return;
	369	}
	370	case 2:
	371	{
	372	__asm__ __volatile__(
	373	"lock; addw %1, %0"
	374	: "=m"(*__hp(addr))
87322fe8 MD	375	: "ir" ((unsigned short)val)
87322fe8 MD	376	: "memory");
cc1be41b MD	377	return;
	378	}
	379	case 4:
	380	{
	381	__asm__ __volatile__(
	382	"lock; addl %1, %0"
	383	: "=m"(*__hp(addr))
87322fe8 MD	384	: "ir" ((unsigned int)val)
87322fe8 MD	385	: "memory");
cc1be41b MD	386	return;
cc1be41b MD	387	}
e040d717	388	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	389	case 8:
	390	{
	391	__asm__ __volatile__(
	392	"lock; addq %1, %0"
	393	: "=m"(*__hp(addr))
87322fe8 MD	394	: "er" ((unsigned long)val)
87322fe8 MD	395	: "memory");
cc1be41b MD	396	return;
cc1be41b MD	397	}
0114ba7f MD	398	#endif
0114ba7f MD	399	}
d0bbd9c2 MD	400	/*
	401	* generate an illegal instruction. Cannot catch this with
	402	* linker tricks when optimizations are disabled.
	403	*/
0114ba7f	404	__asm__ __volatile__("ud2");
a81b8e5e	405	return;
0114ba7f MD	406	}
0114ba7f MD	407
bf9de1b7	408	#define _uatomic_add(addr, v) \
e56d99bf	409	(__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
0114ba7f	410
2c5e5fb3	411
ec4e58a3	412	/* uatomic_inc */
2c5e5fb3 MD	413
2c5e5fb3 MD	414	static inline __attribute__((always_inline))
bf9de1b7	415	void __uatomic_inc(void *addr, int len)
2c5e5fb3 MD	416	{
	417	switch (len) {
	418	case 1:
	419	{
	420	__asm__ __volatile__(
	421	"lock; incb %0"
	422	: "=m"(*__hp(addr))
	423	:
	424	: "memory");
	425	return;
	426	}
	427	case 2:
	428	{
	429	__asm__ __volatile__(
	430	"lock; incw %0"
	431	: "=m"(*__hp(addr))
	432	:
	433	: "memory");
	434	return;
	435	}
	436	case 4:
	437	{
	438	__asm__ __volatile__(
	439	"lock; incl %0"
	440	: "=m"(*__hp(addr))
	441	:
	442	: "memory");
	443	return;
	444	}
e040d717	445	#if (CAA_BITS_PER_LONG == 64)
2c5e5fb3 MD	446	case 8:
	447	{
	448	__asm__ __volatile__(
	449	"lock; incq %0"
	450	: "=m"(*__hp(addr))
	451	:
	452	: "memory");
	453	return;
	454	}
	455	#endif
	456	}
	457	/* generate an illegal instruction. Cannot catch this with linker tricks
	458	* when optimizations are disabled. */
	459	__asm__ __volatile__("ud2");
	460	return;
	461	}
	462
bf9de1b7	463	#define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3	464
ec4e58a3	465	/* uatomic_dec */
2c5e5fb3 MD	466
2c5e5fb3 MD	467	static inline __attribute__((always_inline))
bf9de1b7	468	void __uatomic_dec(void *addr, int len)
2c5e5fb3 MD	469	{
	470	switch (len) {
	471	case 1:
	472	{
	473	__asm__ __volatile__(
	474	"lock; decb %0"
	475	: "=m"(*__hp(addr))
	476	:
	477	: "memory");
	478	return;
	479	}
	480	case 2:
	481	{
	482	__asm__ __volatile__(
	483	"lock; decw %0"
	484	: "=m"(*__hp(addr))
	485	:
	486	: "memory");
	487	return;
	488	}
	489	case 4:
	490	{
	491	__asm__ __volatile__(
	492	"lock; decl %0"
	493	: "=m"(*__hp(addr))
	494	:
	495	: "memory");
	496	return;
	497	}
e040d717	498	#if (CAA_BITS_PER_LONG == 64)
2c5e5fb3 MD	499	case 8:
	500	{
	501	__asm__ __volatile__(
	502	"lock; decq %0"
	503	: "=m"(*__hp(addr))
	504	:
	505	: "memory");
	506	return;
	507	}
	508	#endif
	509	}
d0bbd9c2 MD	510	/*
	511	* generate an illegal instruction. Cannot catch this with
	512	* linker tricks when optimizations are disabled.
	513	*/
2c5e5fb3 MD	514	__asm__ __volatile__("ud2");
	515	return;
	516	}
	517
bf9de1b7	518	#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
0114ba7f	519
e040d717	520	#if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
02be5561 MD	521	extern int __rcu_cas_avail;
02be5561 MD	522	extern int __rcu_cas_init(void);
bf9de1b7 MD	523
bf9de1b7 MD	524	#define UATOMIC_COMPAT(insn) \
a0b7f7ea	525	((caa_likely(__rcu_cas_avail > 0)) \
bf9de1b7	526	? (_uatomic_##insn) \
a0b7f7ea	527	: ((caa_unlikely(__rcu_cas_avail < 0) \
02be5561	528	? ((__rcu_cas_init() > 0) \
bf9de1b7 MD	529	? (_uatomic_##insn) \
	530	: (compat_uatomic_##insn)) \
	531	: (compat_uatomic_##insn))))
	532
424d4ed5 MD	533	/*
	534	* We leave the return value so we don't break the ABI, but remove the
	535	* return value from the API.
	536	*/
bf9de1b7 MD	537	extern unsigned long _compat_uatomic_set(void *addr,
	538	unsigned long _new, int len);
	539	#define compat_uatomic_set(addr, _new) \
424d4ed5 MD	540	((void) _compat_uatomic_set((addr), \
	541	caa_cast_long_keep_sign(_new), \
	542	sizeof(*(addr))))
bf9de1b7 MD	543
	544
	545	extern unsigned long _compat_uatomic_xchg(void *addr,
	546	unsigned long _new, int len);
	547	#define compat_uatomic_xchg(addr, _new) \
	548	((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
e56d99bf	549	caa_cast_long_keep_sign(_new), \
bf9de1b7	550	sizeof(*(addr))))
7d413817 MD	551
7d413817 MD	552	extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
bf9de1b7 MD	553	unsigned long _new, int len);
	554	#define compat_uatomic_cmpxchg(addr, old, _new) \
	555	((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
e56d99bf MD	556	caa_cast_long_keep_sign(old), \
e56d99bf MD	557	caa_cast_long_keep_sign(_new), \
bf9de1b7	558	sizeof(*(addr))))
7d413817	559
8c43fe72	560	extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
bf33aaea	561	#define compat_uatomic_and(addr, v) \
8c43fe72	562	(_compat_uatomic_and((addr), \
e56d99bf	563	caa_cast_long_keep_sign(v), \
8c43fe72	564	sizeof(*(addr))))
bf33aaea	565
8c43fe72	566	extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
985b35b1	567	#define compat_uatomic_or(addr, v) \
8c43fe72	568	(_compat_uatomic_or((addr), \
e56d99bf	569	caa_cast_long_keep_sign(v), \
8c43fe72	570	sizeof(*(addr))))
985b35b1	571
28ca843d PB	572	extern unsigned long _compat_uatomic_add_return(void *addr,
28ca843d PB	573	unsigned long _new, int len);
e56d99bf MD	574	#define compat_uatomic_add_return(addr, v) \
	575	((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
	576	caa_cast_long_keep_sign(v), \
	577	sizeof(*(addr))))
bf9de1b7	578
bf9de1b7 MD	579	#define compat_uatomic_add(addr, v) \
bf9de1b7 MD	580	((void)compat_uatomic_add_return((addr), (v)))
bf9de1b7 MD	581	#define compat_uatomic_inc(addr) \
	582	(compat_uatomic_add((addr), 1))
	583	#define compat_uatomic_dec(addr) \
8760d94e	584	(compat_uatomic_add((addr), -1))
bf9de1b7 MD	585
	586	#else
	587	#define UATOMIC_COMPAT(insn) (_uatomic_##insn)
7d413817 MD	588	#endif
7d413817 MD	589
bf9de1b7	590	/* Read is atomic even in compat mode */
bf9de1b7 MD	591	#define uatomic_set(addr, v) \
bf9de1b7 MD	592	UATOMIC_COMPAT(set(addr, v))
8760d94e	593
bf9de1b7 MD	594	#define uatomic_cmpxchg(addr, old, _new) \
	595	UATOMIC_COMPAT(cmpxchg(addr, old, _new))
	596	#define uatomic_xchg(addr, v) \
	597	UATOMIC_COMPAT(xchg(addr, v))
2812a2d2	598
bf33aaea PB	599	#define uatomic_and(addr, v) \
bf33aaea PB	600	UATOMIC_COMPAT(and(addr, v))
42e83919 MD	601	#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
42e83919 MD	602	#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
2812a2d2	603
985b35b1 PB	604	#define uatomic_or(addr, v) \
985b35b1 PB	605	UATOMIC_COMPAT(or(addr, v))
42e83919 MD	606	#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
42e83919 MD	607	#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
2812a2d2	608
bf9de1b7 MD	609	#define uatomic_add_return(addr, v) \
bf9de1b7 MD	610	UATOMIC_COMPAT(add_return(addr, v))
8760d94e	611
bf9de1b7	612	#define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
42e83919 MD	613	#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
42e83919 MD	614	#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
2812a2d2	615
bf9de1b7	616	#define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
42e83919 MD	617	#define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
42e83919 MD	618	#define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
2812a2d2	619
bf9de1b7	620	#define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
42e83919 MD	621	#define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
42e83919 MD	622	#define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
bf9de1b7	623
67ecffc0	624	#ifdef __cplusplus
36bc70a8 MD	625	}
	626	#endif
	627
a2e7bf9c	628	#include <urcu/uatomic/generic.h>
8760d94e	629
ec4e58a3	630	#endif /* _URCU_ARCH_UATOMIC_X86_H */