[urcu.git] / urcu / uatomic_arch_x86.h

#ifndef _URCU_ARCH_UATOMIC_X86_H
#define _URCU_ARCH_UATOMIC_X86_H

/* 
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <urcu/compiler.h>

#ifndef __SIZEOF_LONG__
#if defined(__x86_64__) || defined(__amd64__)
#define __SIZEOF_LONG__ 8
#else
#define __SIZEOF_LONG__ 4
#endif
#endif

#ifndef BITS_PER_LONG
#define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
#endif

/*
 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
 */

struct __uatomic_dummy {
	unsigned long v[10];
};
#define __hp(x)	((struct __uatomic_dummy *)(x))

#define uatomic_set(addr, v)				\
do {							\
	ACCESS_ONCE(*(addr)) = (v);			\
} while (0)

#define uatomic_read(addr)	ACCESS_ONCE(*(addr))

/* cmpxchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = old;

		__asm__ __volatile__(
		"lock; cmpxchgb %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "q"((unsigned char)_new)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result = old;

		__asm__ __volatile__(
		"lock; cmpxchgw %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned short)_new)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result = old;

		__asm__ __volatile__(
		"lock; cmpxchgl %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned int)_new)
			: "memory");
		return result;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = old;

		__asm__ __volatile__(
		"lock; cmpxchgq %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned long)_new)
			: "memory");
		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define uatomic_cmpxchg(addr, old, _new)				    \
	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
						(unsigned long)(_new), 	    \
						sizeof(*(addr))))

/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	switch (len) {
	case 1:
	{
		unsigned char result;
		__asm__ __volatile__(
		"xchgb %0, %1"
			: "=q"(result), "+m"(*__hp(addr))
			: "0" ((unsigned char)val)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result;
		__asm__ __volatile__(
		"xchgw %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned short)val)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result;
		__asm__ __volatile__(
		"xchgl %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned int)val)
			: "memory");
		return result;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result;
		__asm__ __volatile__(
		"xchgq %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned long)val)
			: "memory");
		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
						sizeof(*(addr))))

/* uatomic_add_return, uatomic_sub_return */

static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = val;

		__asm__ __volatile__(
		"lock; xaddb %1, %0"
			: "+m"(*__hp(addr)), "+q" (result)
			:
			: "memory");
		return result + (unsigned char)val;
	}
	case 2:
	{
		unsigned short result = val;

		__asm__ __volatile__(
		"lock; xaddw %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned short)val;
	}
	case 4:
	{
		unsigned int result = val;

		__asm__ __volatile__(
		"lock; xaddl %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned int)val;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = val;

		__asm__ __volatile__(
		"lock; xaddq %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned long)val;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define uatomic_add_return(addr, v)					\
	((__typeof__(*(addr))) _uatomic_add_return((addr),		\
						  (unsigned long)(v),	\
						  sizeof(*(addr))))

#define uatomic_sub_return(addr, v)	uatomic_add_return((addr), -(v))

/* uatomic_add, uatomic_sub */

static inline __attribute__((always_inline))
void _uatomic_add(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; addb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; addw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; addl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; addq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define uatomic_add(addr, v)						   \
	(_uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))

#define uatomic_sub(addr, v)	uatomic_add((addr), -(v))


/* uatomic_inc */

static inline __attribute__((always_inline))
void _uatomic_inc(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; incb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; incw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; incl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; incq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define uatomic_inc(addr)	(_uatomic_inc((addr), sizeof(*(addr))))

/* uatomic_dec */

static inline __attribute__((always_inline))
void _uatomic_dec(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; decb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; decw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; decl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; decq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define uatomic_dec(addr)	(_uatomic_dec((addr), sizeof(*(addr))))

#if (BITS_PER_LONG == 64)
#define URCU_CAS_AVAIL()	1
#define compat_uatomic_cmpxchg(ptr, old, _new)	uatomic_cmpxchg(ptr, old, _new)
#else
extern int __urcu_cas_avail;
extern int __urcu_cas_init(void);
#define URCU_CAS_AVAIL()						\
		((likely(__urcu_cas_avail > 0)) ?			\
			(1) :						\
			((unlikely(__urcu_cas_avail < 0) ? 		\
				(__urcu_cas_init()) :			\
				(0))))

extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len);

#define compat_uatomic_cmpxchg(addr, old, _new)				     \
	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
						(unsigned long)(_new), 	     \
						sizeof(*(addr))))
#endif

#endif /* _URCU_ARCH_UATOMIC_X86_H */
Commit	Line	Data
ec4e58a3 MD	1	#ifndef _URCU_ARCH_UATOMIC_X86_H
ec4e58a3 MD	2	#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f MD	3
	4	/*
	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	*
	10	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	11	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	12	*
	13	* Permission is hereby granted to use or copy this program
	14	* for any purpose, provided the above notices are retained on all copies.
	15	* Permission to modify the code and to distribute modified code is granted,
	16	* provided the above notices are retained, and a notice that the code was
	17	* modified is included with the above copyright notice.
	18	*
ec4e58a3	19	* Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f MD	20	* Boehm-Demers-Weiser conservative garbage collector.
	21	*/
	22
ec4e58a3	23	#include <urcu/compiler.h>
0fad128b	24
618c4ae1 MD	25	#ifndef __SIZEOF_LONG__
	26	#if defined(__x86_64__) \|\| defined(__amd64__)
	27	#define __SIZEOF_LONG__ 8
	28	#else
	29	#define __SIZEOF_LONG__ 4
	30	#endif
	31	#endif
	32
0114ba7f MD	33	#ifndef BITS_PER_LONG
	34	#define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
	35	#endif
	36
0114ba7f	37	/*
0114ba7f MD	38	* Derived from AO_compare_and_swap() and AO_test_and_set_full().
	39	*/
	40
ec4e58a3	41	struct __uatomic_dummy {
cc1be41b MD	42	unsigned long v[10];
cc1be41b MD	43	};
ec4e58a3	44	#define __hp(x) ((struct __uatomic_dummy *)(x))
cc1be41b	45
ec4e58a3	46	#define uatomic_set(addr, v) \
0fad128b MD	47	do { \
	48	ACCESS_ONCE(*(addr)) = (v); \
	49	} while (0)
	50
ec4e58a3	51	#define uatomic_read(addr) ACCESS_ONCE(*(addr))
0fad128b	52
cc1be41b MD	53	/* cmpxchg */
cc1be41b MD	54
5dba80f9	55	static inline __attribute__((always_inline))
ec4e58a3	56	unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b	57	unsigned long _new, int len)
0114ba7f	58	{
cc1be41b MD	59	switch (len) {
	60	case 1:
	61	{
	62	unsigned char result = old;
0fad128b	63
cc1be41b MD	64	__asm__ __volatile__(
	65	"lock; cmpxchgb %2, %1"
	66	: "+a"(result), "+m"(*__hp(addr))
	67	: "q"((unsigned char)_new)
0114ba7f	68	: "memory");
cc1be41b MD	69	return result;
	70	}
	71	case 2:
	72	{
	73	unsigned short result = old;
0fad128b	74
cc1be41b MD	75	__asm__ __volatile__(
	76	"lock; cmpxchgw %2, %1"
	77	: "+a"(result), "+m"(*__hp(addr))
	78	: "r"((unsigned short)_new)
	79	: "memory");
	80	return result;
	81	}
	82	case 4:
	83	{
	84	unsigned int result = old;
0fad128b	85
cc1be41b MD	86	__asm__ __volatile__(
	87	"lock; cmpxchgl %2, %1"
	88	: "+a"(result), "+m"(*__hp(addr))
	89	: "r"((unsigned int)_new)
	90	: "memory");
	91	return result;
	92	}
	93	#if (BITS_PER_LONG == 64)
	94	case 8:
	95	{
6edb297e	96	unsigned long result = old;
0fad128b	97
cc1be41b	98	__asm__ __volatile__(
2c5e5fb3	99	"lock; cmpxchgq %2, %1"
cc1be41b MD	100	: "+a"(result), "+m"(*__hp(addr))
	101	: "r"((unsigned long)_new)
	102	: "memory");
	103	return result;
	104	}
	105	#endif
	106	}
	107	/* generate an illegal instruction. Cannot catch this with linker tricks
	108	* when optimizations are disabled. */
	109	__asm__ __volatile__("ud2");
	110	return 0;
0114ba7f MD	111	}
0114ba7f MD	112
ec4e58a3 MD	113	#define uatomic_cmpxchg(addr, old, _new) \
ec4e58a3 MD	114	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
cc1be41b MD	115	(unsigned long)(_new), \
	116	sizeof(*(addr))))
	117
	118	/* xchg */
0114ba7f	119
5dba80f9	120	static inline __attribute__((always_inline))
ec4e58a3	121	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f	122	{
cc1be41b MD	123	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	124	switch (len) {
	125	case 1:
	126	{
	127	unsigned char result;
	128	__asm__ __volatile__(
	129	"xchgb %0, %1"
	130	: "=q"(result), "+m"(*__hp(addr))
	131	: "0" ((unsigned char)val)
	132	: "memory");
	133	return result;
	134	}
	135	case 2:
	136	{
	137	unsigned short result;
	138	__asm__ __volatile__(
	139	"xchgw %0, %1"
	140	: "=r"(result), "+m"(*__hp(addr))
	141	: "0" ((unsigned short)val)
	142	: "memory");
	143	return result;
	144	}
	145	case 4:
	146	{
	147	unsigned int result;
	148	__asm__ __volatile__(
	149	"xchgl %0, %1"
	150	: "=r"(result), "+m"(*__hp(addr))
	151	: "0" ((unsigned int)val)
	152	: "memory");
	153	return result;
	154	}
	155	#if (BITS_PER_LONG == 64)
	156	case 8:
	157	{
	158	unsigned long result;
	159	__asm__ __volatile__(
0114ba7f	160	"xchgq %0, %1"
cc1be41b MD	161	: "=r"(result), "+m"(*__hp(addr))
cc1be41b MD	162	: "0" ((unsigned long)val)
0114ba7f	163	: "memory");
cc1be41b MD	164	return result;
	165	}
	166	#endif
	167	}
	168	/* generate an illegal instruction. Cannot catch this with linker tricks
	169	* when optimizations are disabled. */
	170	__asm__ __volatile__("ud2");
	171	return 0;
0114ba7f MD	172	}
0114ba7f MD	173
ec4e58a3 MD	174	#define uatomic_xchg(addr, v) \
ec4e58a3 MD	175	((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
cc1be41b MD	176	sizeof(*(addr))))
cc1be41b MD	177
ec4e58a3	178	/* uatomic_add_return, uatomic_sub_return */
0fad128b MD	179
0fad128b MD	180	static inline __attribute__((always_inline))
ec4e58a3	181	unsigned long _uatomic_add_return(void *addr, unsigned long val,
0fad128b MD	182	int len)
	183	{
	184	switch (len) {
	185	case 1:
	186	{
	187	unsigned char result = val;
	188
	189	__asm__ __volatile__(
	190	"lock; xaddb %1, %0"
	191	: "+m"(*__hp(addr)), "+q" (result)
	192	:
	193	: "memory");
	194	return result + (unsigned char)val;
	195	}
	196	case 2:
	197	{
	198	unsigned short result = val;
	199
	200	__asm__ __volatile__(
	201	"lock; xaddw %1, %0"
	202	: "+m"(*__hp(addr)), "+r" (result)
	203	:
	204	: "memory");
	205	return result + (unsigned short)val;
	206	}
	207	case 4:
	208	{
	209	unsigned int result = val;
	210
	211	__asm__ __volatile__(
	212	"lock; xaddl %1, %0"
	213	: "+m"(*__hp(addr)), "+r" (result)
	214	:
	215	: "memory");
	216	return result + (unsigned int)val;
	217	}
	218	#if (BITS_PER_LONG == 64)
	219	case 8:
	220	{
	221	unsigned long result = val;
	222
	223	__asm__ __volatile__(
	224	"lock; xaddq %1, %0"
	225	: "+m"(*__hp(addr)), "+r" (result)
	226	:
	227	: "memory");
	228	return result + (unsigned long)val;
	229	}
	230	#endif
	231	}
	232	/* generate an illegal instruction. Cannot catch this with linker tricks
	233	* when optimizations are disabled. */
	234	__asm__ __volatile__("ud2");
	235	return 0;
	236	}
	237
ec4e58a3 MD	238	#define uatomic_add_return(addr, v) \
ec4e58a3 MD	239	((__typeof__(*(addr))) _uatomic_add_return((addr), \
0fad128b MD	240	(unsigned long)(v), \
	241	sizeof(*(addr))))
	242
ec4e58a3	243	#define uatomic_sub_return(addr, v) uatomic_add_return((addr), -(v))
0fad128b	244
ec4e58a3	245	/* uatomic_add, uatomic_sub */
0114ba7f	246
5dba80f9	247	static inline __attribute__((always_inline))
ec4e58a3	248	void _uatomic_add(void *addr, unsigned long val, int len)
0114ba7f MD	249	{
0114ba7f MD	250	switch (len) {
cc1be41b MD	251	case 1:
	252	{
	253	__asm__ __volatile__(
	254	"lock; addb %1, %0"
	255	: "=m"(*__hp(addr))
87322fe8 MD	256	: "iq" ((unsigned char)val)
87322fe8 MD	257	: "memory");
cc1be41b MD	258	return;
	259	}
	260	case 2:
	261	{
	262	__asm__ __volatile__(
	263	"lock; addw %1, %0"
	264	: "=m"(*__hp(addr))
87322fe8 MD	265	: "ir" ((unsigned short)val)
87322fe8 MD	266	: "memory");
cc1be41b MD	267	return;
	268	}
	269	case 4:
	270	{
	271	__asm__ __volatile__(
	272	"lock; addl %1, %0"
	273	: "=m"(*__hp(addr))
87322fe8 MD	274	: "ir" ((unsigned int)val)
87322fe8 MD	275	: "memory");
cc1be41b MD	276	return;
cc1be41b MD	277	}
0114ba7f	278	#if (BITS_PER_LONG == 64)
cc1be41b MD	279	case 8:
	280	{
	281	__asm__ __volatile__(
	282	"lock; addq %1, %0"
	283	: "=m"(*__hp(addr))
87322fe8 MD	284	: "er" ((unsigned long)val)
87322fe8 MD	285	: "memory");
cc1be41b MD	286	return;
cc1be41b MD	287	}
0114ba7f MD	288	#endif
	289	}
	290	/* generate an illegal instruction. Cannot catch this with linker tricks
	291	* when optimizations are disabled. */
	292	__asm__ __volatile__("ud2");
a81b8e5e	293	return;
0114ba7f MD	294	}
0114ba7f MD	295
ec4e58a3 MD	296	#define uatomic_add(addr, v) \
ec4e58a3 MD	297	(_uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
0114ba7f	298
ec4e58a3	299	#define uatomic_sub(addr, v) uatomic_add((addr), -(v))
2c5e5fb3 MD	300
2c5e5fb3 MD	301
ec4e58a3	302	/* uatomic_inc */
2c5e5fb3 MD	303
2c5e5fb3 MD	304	static inline __attribute__((always_inline))
ec4e58a3	305	void _uatomic_inc(void *addr, int len)
2c5e5fb3 MD	306	{
	307	switch (len) {
	308	case 1:
	309	{
	310	__asm__ __volatile__(
	311	"lock; incb %0"
	312	: "=m"(*__hp(addr))
	313	:
	314	: "memory");
	315	return;
	316	}
	317	case 2:
	318	{
	319	__asm__ __volatile__(
	320	"lock; incw %0"
	321	: "=m"(*__hp(addr))
	322	:
	323	: "memory");
	324	return;
	325	}
	326	case 4:
	327	{
	328	__asm__ __volatile__(
	329	"lock; incl %0"
	330	: "=m"(*__hp(addr))
	331	:
	332	: "memory");
	333	return;
	334	}
	335	#if (BITS_PER_LONG == 64)
	336	case 8:
	337	{
	338	__asm__ __volatile__(
	339	"lock; incq %0"
	340	: "=m"(*__hp(addr))
	341	:
	342	: "memory");
	343	return;
	344	}
	345	#endif
	346	}
	347	/* generate an illegal instruction. Cannot catch this with linker tricks
	348	* when optimizations are disabled. */
	349	__asm__ __volatile__("ud2");
	350	return;
	351	}
	352
ec4e58a3	353	#define uatomic_inc(addr) (_uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3	354
ec4e58a3	355	/* uatomic_dec */
2c5e5fb3 MD	356
2c5e5fb3 MD	357	static inline __attribute__((always_inline))
ec4e58a3	358	void _uatomic_dec(void *addr, int len)
2c5e5fb3 MD	359	{
	360	switch (len) {
	361	case 1:
	362	{
	363	__asm__ __volatile__(
	364	"lock; decb %0"
	365	: "=m"(*__hp(addr))
	366	:
	367	: "memory");
	368	return;
	369	}
	370	case 2:
	371	{
	372	__asm__ __volatile__(
	373	"lock; decw %0"
	374	: "=m"(*__hp(addr))
	375	:
	376	: "memory");
	377	return;
	378	}
	379	case 4:
	380	{
	381	__asm__ __volatile__(
	382	"lock; decl %0"
	383	: "=m"(*__hp(addr))
	384	:
	385	: "memory");
	386	return;
	387	}
	388	#if (BITS_PER_LONG == 64)
	389	case 8:
	390	{
	391	__asm__ __volatile__(
	392	"lock; decq %0"
	393	: "=m"(*__hp(addr))
	394	:
	395	: "memory");
	396	return;
	397	}
	398	#endif
	399	}
	400	/* generate an illegal instruction. Cannot catch this with linker tricks
	401	* when optimizations are disabled. */
	402	__asm__ __volatile__("ud2");
	403	return;
	404	}
	405
ec4e58a3	406	#define uatomic_dec(addr) (_uatomic_dec((addr), sizeof(*(addr))))
0114ba7f	407
7d413817 MD	408	#if (BITS_PER_LONG == 64)
	409	#define URCU_CAS_AVAIL() 1
	410	#define compat_uatomic_cmpxchg(ptr, old, _new) uatomic_cmpxchg(ptr, old, _new)
	411	#else
	412	extern int __urcu_cas_avail;
e1562850 MD	413	extern int __urcu_cas_init(void);
	414	#define URCU_CAS_AVAIL() \
	415	((likely(__urcu_cas_avail > 0)) ? \
	416	(1) : \
	417	((unlikely(__urcu_cas_avail < 0) ? \
	418	(__urcu_cas_init()) : \
	419	(0))))
7d413817 MD	420
	421	extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
	422	unsigned long _new, int len);
	423
	424	#define compat_uatomic_cmpxchg(addr, old, _new) \
	425	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
	426	(unsigned long)(_new), \
	427	sizeof(*(addr))))
	428	#endif
	429
ec4e58a3	430	#endif /* _URCU_ARCH_UATOMIC_X86_H */