[urcu.git] / include / urcu / uatomic / x86.h

#ifndef _URCU_ARCH_UATOMIC_X86_H
#define _URCU_ARCH_UATOMIC_X86_H

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <urcu/arch.h>
#include <urcu/config.h>
#include <urcu/compiler.h>
#include <urcu/system.h>

#define UATOMIC_HAS_ATOMIC_BYTE
#define UATOMIC_HAS_ATOMIC_SHORT

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
 */

struct __uatomic_dummy {
	unsigned long v[10];
};
#define __hp(x)	((struct __uatomic_dummy *)(x))

#define _uatomic_set(addr, v)	((void) CMM_STORE_SHARED(*(addr), (v)))

/* cmpxchg */

static inline __attribute__((always_inline))
unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = old;

		__asm__ __volatile__(
		"lock; cmpxchgb %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "q"((unsigned char)_new)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result = old;

		__asm__ __volatile__(
		"lock; cmpxchgw %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned short)_new)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result = old;

		__asm__ __volatile__(
		"lock; cmpxchgl %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned int)_new)
			: "memory");
		return result;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = old;

		__asm__ __volatile__(
		"lock; cmpxchgq %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned long)_new)
			: "memory");
		return result;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_cmpxchg(addr, old, _new)				      \
	((__typeof__(*(addr))) __uatomic_cmpxchg((addr),		      \
						caa_cast_long_keep_sign(old), \
						caa_cast_long_keep_sign(_new),\
						sizeof(*(addr))))

/* xchg */

static inline __attribute__((always_inline))
unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
{
	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	switch (len) {
	case 1:
	{
		unsigned char result;
		__asm__ __volatile__(
		"xchgb %0, %1"
			: "=q"(result), "+m"(*__hp(addr))
			: "0" ((unsigned char)val)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result;
		__asm__ __volatile__(
		"xchgw %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned short)val)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result;
		__asm__ __volatile__(
		"xchgl %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned int)val)
			: "memory");
		return result;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result;
		__asm__ __volatile__(
		"xchgq %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned long)val)
			: "memory");
		return result;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_xchg(addr, v)						      \
	((__typeof__(*(addr))) __uatomic_exchange((addr),		      \
						caa_cast_long_keep_sign(v),   \
						sizeof(*(addr))))

/* uatomic_add_return */

static inline __attribute__((always_inline))
unsigned long __uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = val;

		__asm__ __volatile__(
		"lock; xaddb %1, %0"
			: "+m"(*__hp(addr)), "+q" (result)
			:
			: "memory");
		return result + (unsigned char)val;
	}
	case 2:
	{
		unsigned short result = val;

		__asm__ __volatile__(
		"lock; xaddw %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned short)val;
	}
	case 4:
	{
		unsigned int result = val;

		__asm__ __volatile__(
		"lock; xaddl %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned int)val;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = val;

		__asm__ __volatile__(
		"lock; xaddq %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned long)val;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) __uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))

/* uatomic_and */

static inline __attribute__((always_inline))
void __uatomic_and(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; andb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; andw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; andl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; andq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_and(addr, v)						   \
	(__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))

/* uatomic_or */

static inline __attribute__((always_inline))
void __uatomic_or(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; orb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; orw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; orl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; orq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_or(addr, v)						   \
	(__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))

/* uatomic_add */

static inline __attribute__((always_inline))
void __uatomic_add(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; addb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; addw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; addl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; addq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_add(addr, v)						   \
	(__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))


/* uatomic_inc */

static inline __attribute__((always_inline))
void __uatomic_inc(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; incb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; incw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; incl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; incq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_inc(addr)	(__uatomic_inc((addr), sizeof(*(addr))))

/* uatomic_dec */

static inline __attribute__((always_inline))
void __uatomic_dec(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; decb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; decw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; decl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; decq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_dec(addr)	(__uatomic_dec((addr), sizeof(*(addr))))

#if ((CAA_BITS_PER_LONG != 64) && defined(URCU_ARCH_I386))

/* For backwards compat */
#define CONFIG_RCU_COMPAT_ARCH 1

extern int __rcu_cas_avail;
extern int __rcu_cas_init(void);

#define UATOMIC_COMPAT(insn)							\
	((caa_likely(__rcu_cas_avail > 0))						\
	? (_uatomic_##insn)							\
		: ((caa_unlikely(__rcu_cas_avail < 0)				\
			? ((__rcu_cas_init() > 0)				\
				? (_uatomic_##insn)				\
				: (compat_uatomic_##insn))			\
			: (compat_uatomic_##insn))))

/*
 * We leave the return value so we don't break the ABI, but remove the
 * return value from the API.
 */
extern unsigned long _compat_uatomic_set(void *addr,
					 unsigned long _new, int len);
#define compat_uatomic_set(addr, _new)				     	       \
	((void) _compat_uatomic_set((addr),				       \
				caa_cast_long_keep_sign(_new),		       \
				sizeof(*(addr))))


extern unsigned long _compat_uatomic_xchg(void *addr,
					  unsigned long _new, int len);
#define compat_uatomic_xchg(addr, _new)					       \
	((__typeof__(*(addr))) _compat_uatomic_xchg((addr),		       \
						caa_cast_long_keep_sign(_new), \
						sizeof(*(addr))))

extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
					     unsigned long _new, int len);
#define compat_uatomic_cmpxchg(addr, old, _new)				       \
	((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),		       \
						caa_cast_long_keep_sign(old),  \
						caa_cast_long_keep_sign(_new), \
						sizeof(*(addr))))

extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
#define compat_uatomic_and(addr, v)				       \
	(_compat_uatomic_and((addr),				       \
			caa_cast_long_keep_sign(v),		       \
			sizeof(*(addr))))

extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
#define compat_uatomic_or(addr, v)				       \
	(_compat_uatomic_or((addr),				       \
			  caa_cast_long_keep_sign(v),		       \
			  sizeof(*(addr))))

extern unsigned long _compat_uatomic_add_return(void *addr,
						unsigned long _new, int len);
#define compat_uatomic_add_return(addr, v)			            \
	((__typeof__(*(addr))) _compat_uatomic_add_return((addr),     	    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))

#define compat_uatomic_add(addr, v)					       \
		((void)compat_uatomic_add_return((addr), (v)))
#define compat_uatomic_inc(addr)					       \
		(compat_uatomic_add((addr), 1))
#define compat_uatomic_dec(addr)					       \
		(compat_uatomic_add((addr), -1))

#else
#define UATOMIC_COMPAT(insn)	(_uatomic_##insn)
#endif

/* Read is atomic even in compat mode */
#define uatomic_set(addr, v)			\
		UATOMIC_COMPAT(set(addr, v))

#define uatomic_cmpxchg(addr, old, _new)	\
		UATOMIC_COMPAT(cmpxchg(addr, old, _new))
#define uatomic_xchg(addr, v)			\
		UATOMIC_COMPAT(xchg(addr, v))

#define uatomic_and(addr, v)		\
		UATOMIC_COMPAT(and(addr, v))
#define cmm_smp_mb__before_uatomic_and()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_and()		cmm_barrier()

#define uatomic_or(addr, v)		\
		UATOMIC_COMPAT(or(addr, v))
#define cmm_smp_mb__before_uatomic_or()		cmm_barrier()
#define cmm_smp_mb__after_uatomic_or()		cmm_barrier()

#define uatomic_add_return(addr, v)		\
		UATOMIC_COMPAT(add_return(addr, v))

#define uatomic_add(addr, v)	UATOMIC_COMPAT(add(addr, v))
#define cmm_smp_mb__before_uatomic_add()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_add()		cmm_barrier()

#define uatomic_inc(addr)	UATOMIC_COMPAT(inc(addr))
#define cmm_smp_mb__before_uatomic_inc()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_inc()		cmm_barrier()

#define uatomic_dec(addr)	UATOMIC_COMPAT(dec(addr))
#define cmm_smp_mb__before_uatomic_dec()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_dec()		cmm_barrier()

#ifdef __cplusplus
}
#endif

#include <urcu/uatomic/generic.h>

#endif /* _URCU_ARCH_UATOMIC_X86_H */
Commit	Line	Data
ec4e58a3 MD	1	#ifndef _URCU_ARCH_UATOMIC_X86_H
ec4e58a3 MD	2	#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f	3
67ecffc0	4	/*
0114ba7f MD	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	*
	10	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	11	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	12	*
	13	* Permission is hereby granted to use or copy this program
	14	* for any purpose, provided the above notices are retained on all copies.
	15	* Permission to modify the code and to distribute modified code is granted,
	16	* provided the above notices are retained, and a notice that the code was
	17	* modified is included with the above copyright notice.
	18	*
ec4e58a3	19	* Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f MD	20	* Boehm-Demers-Weiser conservative garbage collector.
	21	*/
	22
0b1e236d	23	#include <urcu/arch.h>
375db287	24	#include <urcu/config.h>
ec4e58a3	25	#include <urcu/compiler.h>
bf9de1b7	26	#include <urcu/system.h>
0fad128b	27
f469d839 PB	28	#define UATOMIC_HAS_ATOMIC_BYTE
	29	#define UATOMIC_HAS_ATOMIC_SHORT
	30
36bc70a8 MD	31	#ifdef __cplusplus
36bc70a8 MD	32	extern "C" {
67ecffc0	33	#endif
36bc70a8	34
0114ba7f	35	/*
0114ba7f MD	36	* Derived from AO_compare_and_swap() and AO_test_and_set_full().
	37	*/
	38
ec4e58a3	39	struct __uatomic_dummy {
cc1be41b MD	40	unsigned long v[10];
cc1be41b MD	41	};
ec4e58a3	42	#define __hp(x) ((struct __uatomic_dummy *)(x))
cc1be41b	43
424d4ed5	44	#define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
0fad128b	45
cc1be41b MD	46	/* cmpxchg */
cc1be41b MD	47
5dba80f9	48	static inline __attribute__((always_inline))
bf9de1b7	49	unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b	50	unsigned long _new, int len)
0114ba7f	51	{
cc1be41b MD	52	switch (len) {
	53	case 1:
	54	{
	55	unsigned char result = old;
0fad128b	56
cc1be41b MD	57	__asm__ __volatile__(
	58	"lock; cmpxchgb %2, %1"
	59	: "+a"(result), "+m"(*__hp(addr))
	60	: "q"((unsigned char)_new)
0114ba7f	61	: "memory");
cc1be41b MD	62	return result;
	63	}
	64	case 2:
	65	{
	66	unsigned short result = old;
0fad128b	67
cc1be41b MD	68	__asm__ __volatile__(
	69	"lock; cmpxchgw %2, %1"
	70	: "+a"(result), "+m"(*__hp(addr))
	71	: "r"((unsigned short)_new)
	72	: "memory");
	73	return result;
	74	}
	75	case 4:
	76	{
	77	unsigned int result = old;
0fad128b	78
cc1be41b MD	79	__asm__ __volatile__(
	80	"lock; cmpxchgl %2, %1"
	81	: "+a"(result), "+m"(*__hp(addr))
	82	: "r"((unsigned int)_new)
	83	: "memory");
	84	return result;
	85	}
e040d717	86	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	87	case 8:
cc1be41b MD	88	{
6edb297e	89	unsigned long result = old;
0fad128b	90
cc1be41b	91	__asm__ __volatile__(
2c5e5fb3	92	"lock; cmpxchgq %2, %1"
cc1be41b MD	93	: "+a"(result), "+m"(*__hp(addr))
	94	: "r"((unsigned long)_new)
	95	: "memory");
	96	return result;
	97	}
	98	#endif
	99	}
d0bbd9c2 MD	100	/*
	101	* generate an illegal instruction. Cannot catch this with
	102	* linker tricks when optimizations are disabled.
	103	*/
cc1be41b MD	104	__asm__ __volatile__("ud2");
cc1be41b MD	105	return 0;
0114ba7f MD	106	}
0114ba7f MD	107
bf9de1b7	108	#define _uatomic_cmpxchg(addr, old, _new) \
e56d99bf MD	109	((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
	110	caa_cast_long_keep_sign(old), \
	111	caa_cast_long_keep_sign(_new),\
cc1be41b MD	112	sizeof(*(addr))))
	113
	114	/* xchg */
0114ba7f	115
5dba80f9	116	static inline __attribute__((always_inline))
bf9de1b7	117	unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f	118	{
cc1be41b MD	119	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	120	switch (len) {
	121	case 1:
	122	{
	123	unsigned char result;
	124	__asm__ __volatile__(
	125	"xchgb %0, %1"
	126	: "=q"(result), "+m"(*__hp(addr))
	127	: "0" ((unsigned char)val)
	128	: "memory");
	129	return result;
	130	}
	131	case 2:
	132	{
	133	unsigned short result;
	134	__asm__ __volatile__(
	135	"xchgw %0, %1"
	136	: "=r"(result), "+m"(*__hp(addr))
	137	: "0" ((unsigned short)val)
	138	: "memory");
	139	return result;
	140	}
	141	case 4:
	142	{
	143	unsigned int result;
	144	__asm__ __volatile__(
	145	"xchgl %0, %1"
	146	: "=r"(result), "+m"(*__hp(addr))
	147	: "0" ((unsigned int)val)
	148	: "memory");
	149	return result;
	150	}
e040d717	151	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	152	case 8:
	153	{
	154	unsigned long result;
	155	__asm__ __volatile__(
0114ba7f	156	"xchgq %0, %1"
cc1be41b MD	157	: "=r"(result), "+m"(*__hp(addr))
cc1be41b MD	158	: "0" ((unsigned long)val)
0114ba7f	159	: "memory");
cc1be41b MD	160	return result;
	161	}
	162	#endif
	163	}
d0bbd9c2 MD	164	/*
	165	* generate an illegal instruction. Cannot catch this with
	166	* linker tricks when optimizations are disabled.
	167	*/
cc1be41b MD	168	__asm__ __volatile__("ud2");
cc1be41b MD	169	return 0;
0114ba7f MD	170	}
0114ba7f MD	171
bf9de1b7	172	#define _uatomic_xchg(addr, v) \
e56d99bf MD	173	((__typeof__(*(addr))) __uatomic_exchange((addr), \
e56d99bf MD	174	caa_cast_long_keep_sign(v), \
cc1be41b MD	175	sizeof(*(addr))))
cc1be41b MD	176
8760d94e	177	/* uatomic_add_return */
0fad128b MD	178
0fad128b MD	179	static inline __attribute__((always_inline))
bf9de1b7	180	unsigned long __uatomic_add_return(void *addr, unsigned long val,
0fad128b MD	181	int len)
	182	{
	183	switch (len) {
	184	case 1:
	185	{
	186	unsigned char result = val;
	187
	188	__asm__ __volatile__(
	189	"lock; xaddb %1, %0"
	190	: "+m"(*__hp(addr)), "+q" (result)
	191	:
	192	: "memory");
	193	return result + (unsigned char)val;
	194	}
	195	case 2:
	196	{
	197	unsigned short result = val;
	198
	199	__asm__ __volatile__(
	200	"lock; xaddw %1, %0"
	201	: "+m"(*__hp(addr)), "+r" (result)
	202	:
	203	: "memory");
	204	return result + (unsigned short)val;
	205	}
	206	case 4:
	207	{
	208	unsigned int result = val;
	209
	210	__asm__ __volatile__(
	211	"lock; xaddl %1, %0"
	212	: "+m"(*__hp(addr)), "+r" (result)
	213	:
	214	: "memory");
	215	return result + (unsigned int)val;
	216	}
e040d717	217	#if (CAA_BITS_PER_LONG == 64)
0fad128b MD	218	case 8:
	219	{
	220	unsigned long result = val;
	221
	222	__asm__ __volatile__(
	223	"lock; xaddq %1, %0"
	224	: "+m"(*__hp(addr)), "+r" (result)
	225	:
	226	: "memory");
	227	return result + (unsigned long)val;
	228	}
	229	#endif
	230	}
d0bbd9c2 MD	231	/*
	232	* generate an illegal instruction. Cannot catch this with
	233	* linker tricks when optimizations are disabled.
	234	*/
0fad128b MD	235	__asm__ __volatile__("ud2");
	236	return 0;
	237	}
	238
e56d99bf MD	239	#define _uatomic_add_return(addr, v) \
	240	((__typeof__(*(addr))) __uatomic_add_return((addr), \
	241	caa_cast_long_keep_sign(v), \
	242	sizeof(*(addr))))
0fad128b	243
bf33aaea PB	244	/* uatomic_and */
	245
	246	static inline __attribute__((always_inline))
	247	void __uatomic_and(void *addr, unsigned long val, int len)
	248	{
	249	switch (len) {
	250	case 1:
	251	{
	252	__asm__ __volatile__(
	253	"lock; andb %1, %0"
	254	: "=m"(*__hp(addr))
	255	: "iq" ((unsigned char)val)
	256	: "memory");
	257	return;
	258	}
	259	case 2:
	260	{
	261	__asm__ __volatile__(
	262	"lock; andw %1, %0"
	263	: "=m"(*__hp(addr))
	264	: "ir" ((unsigned short)val)
	265	: "memory");
	266	return;
	267	}
	268	case 4:
	269	{
	270	__asm__ __volatile__(
	271	"lock; andl %1, %0"
	272	: "=m"(*__hp(addr))
	273	: "ir" ((unsigned int)val)
	274	: "memory");
	275	return;
	276	}
	277	#if (CAA_BITS_PER_LONG == 64)
	278	case 8:
	279	{
	280	__asm__ __volatile__(
	281	"lock; andq %1, %0"
	282	: "=m"(*__hp(addr))
	283	: "er" ((unsigned long)val)
	284	: "memory");
	285	return;
	286	}
	287	#endif
	288	}
d0bbd9c2 MD	289	/*
	290	* generate an illegal instruction. Cannot catch this with
	291	* linker tricks when optimizations are disabled.
	292	*/
bf33aaea PB	293	__asm__ __volatile__("ud2");
	294	return;
	295	}
	296
	297	#define _uatomic_and(addr, v) \
e56d99bf	298	(__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
bf33aaea	299
985b35b1 PB	300	/* uatomic_or */
	301
	302	static inline __attribute__((always_inline))
	303	void __uatomic_or(void *addr, unsigned long val, int len)
	304	{
	305	switch (len) {
	306	case 1:
	307	{
	308	__asm__ __volatile__(
	309	"lock; orb %1, %0"
	310	: "=m"(*__hp(addr))
	311	: "iq" ((unsigned char)val)
	312	: "memory");
	313	return;
	314	}
	315	case 2:
	316	{
	317	__asm__ __volatile__(
	318	"lock; orw %1, %0"
	319	: "=m"(*__hp(addr))
	320	: "ir" ((unsigned short)val)
	321	: "memory");
	322	return;
	323	}
	324	case 4:
	325	{
	326	__asm__ __volatile__(
	327	"lock; orl %1, %0"
	328	: "=m"(*__hp(addr))
	329	: "ir" ((unsigned int)val)
	330	: "memory");
	331	return;
	332	}
	333	#if (CAA_BITS_PER_LONG == 64)
	334	case 8:
	335	{
	336	__asm__ __volatile__(
	337	"lock; orq %1, %0"
	338	: "=m"(*__hp(addr))
	339	: "er" ((unsigned long)val)
	340	: "memory");
	341	return;
	342	}
	343	#endif
	344	}
d0bbd9c2 MD	345	/*
	346	* generate an illegal instruction. Cannot catch this with
	347	* linker tricks when optimizations are disabled.
	348	*/
985b35b1 PB	349	__asm__ __volatile__("ud2");
	350	return;
	351	}
	352
	353	#define _uatomic_or(addr, v) \
e56d99bf	354	(__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
985b35b1	355
8760d94e	356	/* uatomic_add */
0114ba7f	357
5dba80f9	358	static inline __attribute__((always_inline))
bf9de1b7	359	void __uatomic_add(void *addr, unsigned long val, int len)
0114ba7f MD	360	{
0114ba7f MD	361	switch (len) {
cc1be41b MD	362	case 1:
	363	{
	364	__asm__ __volatile__(
	365	"lock; addb %1, %0"
	366	: "=m"(*__hp(addr))
87322fe8 MD	367	: "iq" ((unsigned char)val)
87322fe8 MD	368	: "memory");
cc1be41b MD	369	return;
	370	}
	371	case 2:
	372	{
	373	__asm__ __volatile__(
	374	"lock; addw %1, %0"
	375	: "=m"(*__hp(addr))
87322fe8 MD	376	: "ir" ((unsigned short)val)
87322fe8 MD	377	: "memory");
cc1be41b MD	378	return;
	379	}
	380	case 4:
	381	{
	382	__asm__ __volatile__(
	383	"lock; addl %1, %0"
	384	: "=m"(*__hp(addr))
87322fe8 MD	385	: "ir" ((unsigned int)val)
87322fe8 MD	386	: "memory");
cc1be41b MD	387	return;
cc1be41b MD	388	}
e040d717	389	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	390	case 8:
	391	{
	392	__asm__ __volatile__(
	393	"lock; addq %1, %0"
	394	: "=m"(*__hp(addr))
87322fe8 MD	395	: "er" ((unsigned long)val)
87322fe8 MD	396	: "memory");
cc1be41b MD	397	return;
cc1be41b MD	398	}
0114ba7f MD	399	#endif
0114ba7f MD	400	}
d0bbd9c2 MD	401	/*
	402	* generate an illegal instruction. Cannot catch this with
	403	* linker tricks when optimizations are disabled.
	404	*/
0114ba7f	405	__asm__ __volatile__("ud2");
a81b8e5e	406	return;
0114ba7f MD	407	}
0114ba7f MD	408
bf9de1b7	409	#define _uatomic_add(addr, v) \
e56d99bf	410	(__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
0114ba7f	411
2c5e5fb3	412
ec4e58a3	413	/* uatomic_inc */
2c5e5fb3 MD	414
2c5e5fb3 MD	415	static inline __attribute__((always_inline))
bf9de1b7	416	void __uatomic_inc(void *addr, int len)
2c5e5fb3 MD	417	{
	418	switch (len) {
	419	case 1:
	420	{
	421	__asm__ __volatile__(
	422	"lock; incb %0"
	423	: "=m"(*__hp(addr))
	424	:
	425	: "memory");
	426	return;
	427	}
	428	case 2:
	429	{
	430	__asm__ __volatile__(
	431	"lock; incw %0"
	432	: "=m"(*__hp(addr))
	433	:
	434	: "memory");
	435	return;
	436	}
	437	case 4:
	438	{
	439	__asm__ __volatile__(
	440	"lock; incl %0"
	441	: "=m"(*__hp(addr))
	442	:
	443	: "memory");
	444	return;
	445	}
e040d717	446	#if (CAA_BITS_PER_LONG == 64)
2c5e5fb3 MD	447	case 8:
	448	{
	449	__asm__ __volatile__(
	450	"lock; incq %0"
	451	: "=m"(*__hp(addr))
	452	:
	453	: "memory");
	454	return;
	455	}
	456	#endif
	457	}
	458	/* generate an illegal instruction. Cannot catch this with linker tricks
	459	* when optimizations are disabled. */
	460	__asm__ __volatile__("ud2");
	461	return;
	462	}
	463
bf9de1b7	464	#define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3	465
ec4e58a3	466	/* uatomic_dec */
2c5e5fb3 MD	467
2c5e5fb3 MD	468	static inline __attribute__((always_inline))
bf9de1b7	469	void __uatomic_dec(void *addr, int len)
2c5e5fb3 MD	470	{
	471	switch (len) {
	472	case 1:
	473	{
	474	__asm__ __volatile__(
	475	"lock; decb %0"
	476	: "=m"(*__hp(addr))
	477	:
	478	: "memory");
	479	return;
	480	}
	481	case 2:
	482	{
	483	__asm__ __volatile__(
	484	"lock; decw %0"
	485	: "=m"(*__hp(addr))
	486	:
	487	: "memory");
	488	return;
	489	}
	490	case 4:
	491	{
	492	__asm__ __volatile__(
	493	"lock; decl %0"
	494	: "=m"(*__hp(addr))
	495	:
	496	: "memory");
	497	return;
	498	}
e040d717	499	#if (CAA_BITS_PER_LONG == 64)
2c5e5fb3 MD	500	case 8:
	501	{
	502	__asm__ __volatile__(
	503	"lock; decq %0"
	504	: "=m"(*__hp(addr))
	505	:
	506	: "memory");
	507	return;
	508	}
	509	#endif
	510	}
d0bbd9c2 MD	511	/*
	512	* generate an illegal instruction. Cannot catch this with
	513	* linker tricks when optimizations are disabled.
	514	*/
2c5e5fb3 MD	515	__asm__ __volatile__("ud2");
	516	return;
	517	}
	518
bf9de1b7	519	#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
0114ba7f	520
0b1e236d MJ	521	#if ((CAA_BITS_PER_LONG != 64) && defined(URCU_ARCH_I386))
	522
	523	/* For backwards compat */
	524	#define CONFIG_RCU_COMPAT_ARCH 1
	525
02be5561 MD	526	extern int __rcu_cas_avail;
02be5561 MD	527	extern int __rcu_cas_init(void);
bf9de1b7 MD	528
bf9de1b7 MD	529	#define UATOMIC_COMPAT(insn) \
a0b7f7ea	530	((caa_likely(__rcu_cas_avail > 0)) \
bf9de1b7	531	? (_uatomic_##insn) \
a0b7f7ea	532	: ((caa_unlikely(__rcu_cas_avail < 0) \
02be5561	533	? ((__rcu_cas_init() > 0) \
bf9de1b7 MD	534	? (_uatomic_##insn) \
	535	: (compat_uatomic_##insn)) \
	536	: (compat_uatomic_##insn))))
	537
424d4ed5 MD	538	/*
	539	* We leave the return value so we don't break the ABI, but remove the
	540	* return value from the API.
	541	*/
bf9de1b7 MD	542	extern unsigned long _compat_uatomic_set(void *addr,
	543	unsigned long _new, int len);
	544	#define compat_uatomic_set(addr, _new) \
424d4ed5 MD	545	((void) _compat_uatomic_set((addr), \
	546	caa_cast_long_keep_sign(_new), \
	547	sizeof(*(addr))))
bf9de1b7 MD	548
	549
	550	extern unsigned long _compat_uatomic_xchg(void *addr,
	551	unsigned long _new, int len);
	552	#define compat_uatomic_xchg(addr, _new) \
	553	((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
e56d99bf	554	caa_cast_long_keep_sign(_new), \
bf9de1b7	555	sizeof(*(addr))))
7d413817 MD	556
7d413817 MD	557	extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
bf9de1b7 MD	558	unsigned long _new, int len);
	559	#define compat_uatomic_cmpxchg(addr, old, _new) \
	560	((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
e56d99bf MD	561	caa_cast_long_keep_sign(old), \
e56d99bf MD	562	caa_cast_long_keep_sign(_new), \
bf9de1b7	563	sizeof(*(addr))))
7d413817	564
8c43fe72	565	extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
bf33aaea	566	#define compat_uatomic_and(addr, v) \
8c43fe72	567	(_compat_uatomic_and((addr), \
e56d99bf	568	caa_cast_long_keep_sign(v), \
8c43fe72	569	sizeof(*(addr))))
bf33aaea	570
8c43fe72	571	extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
985b35b1	572	#define compat_uatomic_or(addr, v) \
8c43fe72	573	(_compat_uatomic_or((addr), \
e56d99bf	574	caa_cast_long_keep_sign(v), \
8c43fe72	575	sizeof(*(addr))))
985b35b1	576
28ca843d PB	577	extern unsigned long _compat_uatomic_add_return(void *addr,
28ca843d PB	578	unsigned long _new, int len);
e56d99bf MD	579	#define compat_uatomic_add_return(addr, v) \
	580	((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
	581	caa_cast_long_keep_sign(v), \
	582	sizeof(*(addr))))
bf9de1b7	583
bf9de1b7 MD	584	#define compat_uatomic_add(addr, v) \
bf9de1b7 MD	585	((void)compat_uatomic_add_return((addr), (v)))
bf9de1b7 MD	586	#define compat_uatomic_inc(addr) \
	587	(compat_uatomic_add((addr), 1))
	588	#define compat_uatomic_dec(addr) \
8760d94e	589	(compat_uatomic_add((addr), -1))
bf9de1b7 MD	590
	591	#else
	592	#define UATOMIC_COMPAT(insn) (_uatomic_##insn)
7d413817 MD	593	#endif
7d413817 MD	594
bf9de1b7	595	/* Read is atomic even in compat mode */
bf9de1b7 MD	596	#define uatomic_set(addr, v) \
bf9de1b7 MD	597	UATOMIC_COMPAT(set(addr, v))
8760d94e	598
bf9de1b7 MD	599	#define uatomic_cmpxchg(addr, old, _new) \
	600	UATOMIC_COMPAT(cmpxchg(addr, old, _new))
	601	#define uatomic_xchg(addr, v) \
	602	UATOMIC_COMPAT(xchg(addr, v))
2812a2d2	603
bf33aaea PB	604	#define uatomic_and(addr, v) \
bf33aaea PB	605	UATOMIC_COMPAT(and(addr, v))
42e83919 MD	606	#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
42e83919 MD	607	#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
2812a2d2	608
985b35b1 PB	609	#define uatomic_or(addr, v) \
985b35b1 PB	610	UATOMIC_COMPAT(or(addr, v))
42e83919 MD	611	#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
42e83919 MD	612	#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
2812a2d2	613
bf9de1b7 MD	614	#define uatomic_add_return(addr, v) \
bf9de1b7 MD	615	UATOMIC_COMPAT(add_return(addr, v))
8760d94e	616
bf9de1b7	617	#define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
42e83919 MD	618	#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
42e83919 MD	619	#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
2812a2d2	620
bf9de1b7	621	#define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
42e83919 MD	622	#define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
42e83919 MD	623	#define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
2812a2d2	624
bf9de1b7	625	#define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
42e83919 MD	626	#define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
42e83919 MD	627	#define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
bf9de1b7	628
67ecffc0	629	#ifdef __cplusplus
36bc70a8 MD	630	}
	631	#endif
	632
a2e7bf9c	633	#include <urcu/uatomic/generic.h>
8760d94e	634
ec4e58a3	635	#endif /* _URCU_ARCH_UATOMIC_X86_H */