[urcu.git] / urcu / uatomic_arch_x86.h

#ifndef _URCU_ARCH_UATOMIC_X86_H
#define _URCU_ARCH_UATOMIC_X86_H

/* 
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <urcu/compiler.h>

#ifndef BITS_PER_LONG
#define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
#endif

/*
 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
 */

struct __uatomic_dummy {
	unsigned long v[10];
};
#define __hp(x)	((struct __uatomic_dummy *)(x))

#define uatomic_set(addr, v)				\
do {							\
	ACCESS_ONCE(*(addr)) = (v);			\
} while (0)

#define uatomic_read(addr)	ACCESS_ONCE(*(addr))

/* cmpxchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = old;

		__asm__ __volatile__(
		"lock; cmpxchgb %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "q"((unsigned char)_new)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result = old;

		__asm__ __volatile__(
		"lock; cmpxchgw %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned short)_new)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result = old;

		__asm__ __volatile__(
		"lock; cmpxchgl %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned int)_new)
			: "memory");
		return result;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = old;

		__asm__ __volatile__(
		"lock; cmpxchgq %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned long)_new)
			: "memory");
		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define uatomic_cmpxchg(addr, old, _new)				    \
	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
						(unsigned long)(_new), 	    \
						sizeof(*(addr))))

/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	switch (len) {
	case 1:
	{
		unsigned char result;
		__asm__ __volatile__(
		"xchgb %0, %1"
			: "=q"(result), "+m"(*__hp(addr))
			: "0" ((unsigned char)val)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result;
		__asm__ __volatile__(
		"xchgw %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned short)val)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result;
		__asm__ __volatile__(
		"xchgl %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned int)val)
			: "memory");
		return result;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result;
		__asm__ __volatile__(
		"xchgq %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned long)val)
			: "memory");
		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
						sizeof(*(addr))))

/* uatomic_add_return, uatomic_sub_return */

static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = val;

		__asm__ __volatile__(
		"lock; xaddb %1, %0"
			: "+m"(*__hp(addr)), "+q" (result)
			:
			: "memory");
		return result + (unsigned char)val;
	}
	case 2:
	{
		unsigned short result = val;

		__asm__ __volatile__(
		"lock; xaddw %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned short)val;
	}
	case 4:
	{
		unsigned int result = val;

		__asm__ __volatile__(
		"lock; xaddl %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned int)val;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = val;

		__asm__ __volatile__(
		"lock; xaddq %1, %0"
			: "+m"(*__hp(addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned long)val;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define uatomic_add_return(addr, v)					\
	((__typeof__(*(addr))) _uatomic_add_return((addr),		\
						  (unsigned long)(v),	\
						  sizeof(*(addr))))

#define uatomic_sub_return(addr, v)	uatomic_add_return((addr), -(v))

/* uatomic_add, uatomic_sub */

static inline __attribute__((always_inline))
void _uatomic_add(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; addb %1, %0"
			: "=m"(*__hp(addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; addw %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; addl %1, %0"
			: "=m"(*__hp(addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; addq %1, %0"
			: "=m"(*__hp(addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define uatomic_add(addr, v)						   \
	(_uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))

#define uatomic_sub(addr, v)	uatomic_add((addr), -(v))


/* uatomic_inc */

static inline __attribute__((always_inline))
void _uatomic_inc(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; incb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; incw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; incl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; incq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define uatomic_inc(addr)	(_uatomic_inc((addr), sizeof(*(addr))))

/* uatomic_dec */

static inline __attribute__((always_inline))
void _uatomic_dec(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; decb %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; decw %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; decl %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; decq %0"
			: "=m"(*__hp(addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define uatomic_dec(addr)	(_uatomic_dec((addr), sizeof(*(addr))))

#if (BITS_PER_LONG == 64)
#define URCU_CAS_AVAIL()	1
#define compat_uatomic_cmpxchg(ptr, old, _new)	uatomic_cmpxchg(ptr, old, _new)
#else
extern int __urcu_cas_avail;
extern int __urcu_cas_init(void);
#define URCU_CAS_AVAIL()						\
		((likely(__urcu_cas_avail > 0)) ?			\
			(1) :						\
			((unlikely(__urcu_cas_avail < 0) ? 		\
				(__urcu_cas_init()) :			\
				(0))))

extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len);

#define compat_uatomic_cmpxchg(addr, old, _new)				     \
	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
						(unsigned long)(_new), 	     \
						sizeof(*(addr))))
#endif

#endif /* _URCU_ARCH_UATOMIC_X86_H */
Commit	Line	Data
ec4e58a3 MD	1	#ifndef _URCU_ARCH_UATOMIC_X86_H
ec4e58a3 MD	2	#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f MD	3
	4	/*
	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	*
	10	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	11	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	12	*
	13	* Permission is hereby granted to use or copy this program
	14	* for any purpose, provided the above notices are retained on all copies.
	15	* Permission to modify the code and to distribute modified code is granted,
	16	* provided the above notices are retained, and a notice that the code was
	17	* modified is included with the above copyright notice.
	18	*
ec4e58a3	19	* Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f MD	20	* Boehm-Demers-Weiser conservative garbage collector.
	21	*/
	22
ec4e58a3	23	#include <urcu/compiler.h>
0fad128b	24
0114ba7f MD	25	#ifndef BITS_PER_LONG
	26	#define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
	27	#endif
	28
0114ba7f	29	/*
0114ba7f MD	30	* Derived from AO_compare_and_swap() and AO_test_and_set_full().
	31	*/
	32
ec4e58a3	33	struct __uatomic_dummy {
cc1be41b MD	34	unsigned long v[10];
cc1be41b MD	35	};
ec4e58a3	36	#define __hp(x) ((struct __uatomic_dummy *)(x))
cc1be41b	37
ec4e58a3	38	#define uatomic_set(addr, v) \
0fad128b MD	39	do { \
	40	ACCESS_ONCE(*(addr)) = (v); \
	41	} while (0)
	42
ec4e58a3	43	#define uatomic_read(addr) ACCESS_ONCE(*(addr))
0fad128b	44
cc1be41b MD	45	/* cmpxchg */
cc1be41b MD	46
5dba80f9	47	static inline __attribute__((always_inline))
ec4e58a3	48	unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b	49	unsigned long _new, int len)
0114ba7f	50	{
cc1be41b MD	51	switch (len) {
	52	case 1:
	53	{
	54	unsigned char result = old;
0fad128b	55
cc1be41b MD	56	__asm__ __volatile__(
	57	"lock; cmpxchgb %2, %1"
	58	: "+a"(result), "+m"(*__hp(addr))
	59	: "q"((unsigned char)_new)
0114ba7f	60	: "memory");
cc1be41b MD	61	return result;
	62	}
	63	case 2:
	64	{
	65	unsigned short result = old;
0fad128b	66
cc1be41b MD	67	__asm__ __volatile__(
	68	"lock; cmpxchgw %2, %1"
	69	: "+a"(result), "+m"(*__hp(addr))
	70	: "r"((unsigned short)_new)
	71	: "memory");
	72	return result;
	73	}
	74	case 4:
	75	{
	76	unsigned int result = old;
0fad128b	77
cc1be41b MD	78	__asm__ __volatile__(
	79	"lock; cmpxchgl %2, %1"
	80	: "+a"(result), "+m"(*__hp(addr))
	81	: "r"((unsigned int)_new)
	82	: "memory");
	83	return result;
	84	}
	85	#if (BITS_PER_LONG == 64)
	86	case 8:
	87	{
6edb297e	88	unsigned long result = old;
0fad128b	89
cc1be41b	90	__asm__ __volatile__(
2c5e5fb3	91	"lock; cmpxchgq %2, %1"
cc1be41b MD	92	: "+a"(result), "+m"(*__hp(addr))
	93	: "r"((unsigned long)_new)
	94	: "memory");
	95	return result;
	96	}
	97	#endif
	98	}
	99	/* generate an illegal instruction. Cannot catch this with linker tricks
	100	* when optimizations are disabled. */
	101	__asm__ __volatile__("ud2");
	102	return 0;
0114ba7f MD	103	}
0114ba7f MD	104
ec4e58a3 MD	105	#define uatomic_cmpxchg(addr, old, _new) \
ec4e58a3 MD	106	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
cc1be41b MD	107	(unsigned long)(_new), \
	108	sizeof(*(addr))))
	109
	110	/* xchg */
0114ba7f	111
5dba80f9	112	static inline __attribute__((always_inline))
ec4e58a3	113	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f	114	{
cc1be41b MD	115	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	116	switch (len) {
	117	case 1:
	118	{
	119	unsigned char result;
	120	__asm__ __volatile__(
	121	"xchgb %0, %1"
	122	: "=q"(result), "+m"(*__hp(addr))
	123	: "0" ((unsigned char)val)
	124	: "memory");
	125	return result;
	126	}
	127	case 2:
	128	{
	129	unsigned short result;
	130	__asm__ __volatile__(
	131	"xchgw %0, %1"
	132	: "=r"(result), "+m"(*__hp(addr))
	133	: "0" ((unsigned short)val)
	134	: "memory");
	135	return result;
	136	}
	137	case 4:
	138	{
	139	unsigned int result;
	140	__asm__ __volatile__(
	141	"xchgl %0, %1"
	142	: "=r"(result), "+m"(*__hp(addr))
	143	: "0" ((unsigned int)val)
	144	: "memory");
	145	return result;
	146	}
	147	#if (BITS_PER_LONG == 64)
	148	case 8:
	149	{
	150	unsigned long result;
	151	__asm__ __volatile__(
0114ba7f	152	"xchgq %0, %1"
cc1be41b MD	153	: "=r"(result), "+m"(*__hp(addr))
cc1be41b MD	154	: "0" ((unsigned long)val)
0114ba7f	155	: "memory");
cc1be41b MD	156	return result;
	157	}
	158	#endif
	159	}
	160	/* generate an illegal instruction. Cannot catch this with linker tricks
	161	* when optimizations are disabled. */
	162	__asm__ __volatile__("ud2");
	163	return 0;
0114ba7f MD	164	}
0114ba7f MD	165
ec4e58a3 MD	166	#define uatomic_xchg(addr, v) \
ec4e58a3 MD	167	((__typeof__(*(addr))) _uatomic_exchange((addr), (unsigned long)(v), \
cc1be41b MD	168	sizeof(*(addr))))
cc1be41b MD	169
ec4e58a3	170	/* uatomic_add_return, uatomic_sub_return */
0fad128b MD	171
0fad128b MD	172	static inline __attribute__((always_inline))
ec4e58a3	173	unsigned long _uatomic_add_return(void *addr, unsigned long val,
0fad128b MD	174	int len)
	175	{
	176	switch (len) {
	177	case 1:
	178	{
	179	unsigned char result = val;
	180
	181	__asm__ __volatile__(
	182	"lock; xaddb %1, %0"
	183	: "+m"(*__hp(addr)), "+q" (result)
	184	:
	185	: "memory");
	186	return result + (unsigned char)val;
	187	}
	188	case 2:
	189	{
	190	unsigned short result = val;
	191
	192	__asm__ __volatile__(
	193	"lock; xaddw %1, %0"
	194	: "+m"(*__hp(addr)), "+r" (result)
	195	:
	196	: "memory");
	197	return result + (unsigned short)val;
	198	}
	199	case 4:
	200	{
	201	unsigned int result = val;
	202
	203	__asm__ __volatile__(
	204	"lock; xaddl %1, %0"
	205	: "+m"(*__hp(addr)), "+r" (result)
	206	:
	207	: "memory");
	208	return result + (unsigned int)val;
	209	}
	210	#if (BITS_PER_LONG == 64)
	211	case 8:
	212	{
	213	unsigned long result = val;
	214
	215	__asm__ __volatile__(
	216	"lock; xaddq %1, %0"
	217	: "+m"(*__hp(addr)), "+r" (result)
	218	:
	219	: "memory");
	220	return result + (unsigned long)val;
	221	}
	222	#endif
	223	}
	224	/* generate an illegal instruction. Cannot catch this with linker tricks
	225	* when optimizations are disabled. */
	226	__asm__ __volatile__("ud2");
	227	return 0;
	228	}
	229
ec4e58a3 MD	230	#define uatomic_add_return(addr, v) \
ec4e58a3 MD	231	((__typeof__(*(addr))) _uatomic_add_return((addr), \
0fad128b MD	232	(unsigned long)(v), \
	233	sizeof(*(addr))))
	234
ec4e58a3	235	#define uatomic_sub_return(addr, v) uatomic_add_return((addr), -(v))
0fad128b	236
ec4e58a3	237	/* uatomic_add, uatomic_sub */
0114ba7f	238
5dba80f9	239	static inline __attribute__((always_inline))
ec4e58a3	240	void _uatomic_add(void *addr, unsigned long val, int len)
0114ba7f MD	241	{
0114ba7f MD	242	switch (len) {
cc1be41b MD	243	case 1:
	244	{
	245	__asm__ __volatile__(
	246	"lock; addb %1, %0"
	247	: "=m"(*__hp(addr))
87322fe8 MD	248	: "iq" ((unsigned char)val)
87322fe8 MD	249	: "memory");
cc1be41b MD	250	return;
	251	}
	252	case 2:
	253	{
	254	__asm__ __volatile__(
	255	"lock; addw %1, %0"
	256	: "=m"(*__hp(addr))
87322fe8 MD	257	: "ir" ((unsigned short)val)
87322fe8 MD	258	: "memory");
cc1be41b MD	259	return;
	260	}
	261	case 4:
	262	{
	263	__asm__ __volatile__(
	264	"lock; addl %1, %0"
	265	: "=m"(*__hp(addr))
87322fe8 MD	266	: "ir" ((unsigned int)val)
87322fe8 MD	267	: "memory");
cc1be41b MD	268	return;
cc1be41b MD	269	}
0114ba7f	270	#if (BITS_PER_LONG == 64)
cc1be41b MD	271	case 8:
	272	{
	273	__asm__ __volatile__(
	274	"lock; addq %1, %0"
	275	: "=m"(*__hp(addr))
87322fe8 MD	276	: "er" ((unsigned long)val)
87322fe8 MD	277	: "memory");
cc1be41b MD	278	return;
cc1be41b MD	279	}
0114ba7f MD	280	#endif
	281	}
	282	/* generate an illegal instruction. Cannot catch this with linker tricks
	283	* when optimizations are disabled. */
	284	__asm__ __volatile__("ud2");
a81b8e5e	285	return;
0114ba7f MD	286	}
0114ba7f MD	287
ec4e58a3 MD	288	#define uatomic_add(addr, v) \
ec4e58a3 MD	289	(_uatomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
0114ba7f	290
ec4e58a3	291	#define uatomic_sub(addr, v) uatomic_add((addr), -(v))
2c5e5fb3 MD	292
2c5e5fb3 MD	293
ec4e58a3	294	/* uatomic_inc */
2c5e5fb3 MD	295
2c5e5fb3 MD	296	static inline __attribute__((always_inline))
ec4e58a3	297	void _uatomic_inc(void *addr, int len)
2c5e5fb3 MD	298	{
	299	switch (len) {
	300	case 1:
	301	{
	302	__asm__ __volatile__(
	303	"lock; incb %0"
	304	: "=m"(*__hp(addr))
	305	:
	306	: "memory");
	307	return;
	308	}
	309	case 2:
	310	{
	311	__asm__ __volatile__(
	312	"lock; incw %0"
	313	: "=m"(*__hp(addr))
	314	:
	315	: "memory");
	316	return;
	317	}
	318	case 4:
	319	{
	320	__asm__ __volatile__(
	321	"lock; incl %0"
	322	: "=m"(*__hp(addr))
	323	:
	324	: "memory");
	325	return;
	326	}
	327	#if (BITS_PER_LONG == 64)
	328	case 8:
	329	{
	330	__asm__ __volatile__(
	331	"lock; incq %0"
	332	: "=m"(*__hp(addr))
	333	:
	334	: "memory");
	335	return;
	336	}
	337	#endif
	338	}
	339	/* generate an illegal instruction. Cannot catch this with linker tricks
	340	* when optimizations are disabled. */
	341	__asm__ __volatile__("ud2");
	342	return;
	343	}
	344
ec4e58a3	345	#define uatomic_inc(addr) (_uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3	346
ec4e58a3	347	/* uatomic_dec */
2c5e5fb3 MD	348
2c5e5fb3 MD	349	static inline __attribute__((always_inline))
ec4e58a3	350	void _uatomic_dec(void *addr, int len)
2c5e5fb3 MD	351	{
	352	switch (len) {
	353	case 1:
	354	{
	355	__asm__ __volatile__(
	356	"lock; decb %0"
	357	: "=m"(*__hp(addr))
	358	:
	359	: "memory");
	360	return;
	361	}
	362	case 2:
	363	{
	364	__asm__ __volatile__(
	365	"lock; decw %0"
	366	: "=m"(*__hp(addr))
	367	:
	368	: "memory");
	369	return;
	370	}
	371	case 4:
	372	{
	373	__asm__ __volatile__(
	374	"lock; decl %0"
	375	: "=m"(*__hp(addr))
	376	:
	377	: "memory");
	378	return;
	379	}
	380	#if (BITS_PER_LONG == 64)
	381	case 8:
	382	{
	383	__asm__ __volatile__(
	384	"lock; decq %0"
	385	: "=m"(*__hp(addr))
	386	:
	387	: "memory");
	388	return;
	389	}
	390	#endif
	391	}
	392	/* generate an illegal instruction. Cannot catch this with linker tricks
	393	* when optimizations are disabled. */
	394	__asm__ __volatile__("ud2");
	395	return;
	396	}
	397
ec4e58a3	398	#define uatomic_dec(addr) (_uatomic_dec((addr), sizeof(*(addr))))
0114ba7f	399
7d413817 MD	400	#if (BITS_PER_LONG == 64)
	401	#define URCU_CAS_AVAIL() 1
	402	#define compat_uatomic_cmpxchg(ptr, old, _new) uatomic_cmpxchg(ptr, old, _new)
	403	#else
	404	extern int __urcu_cas_avail;
e1562850 MD	405	extern int __urcu_cas_init(void);
	406	#define URCU_CAS_AVAIL() \
	407	((likely(__urcu_cas_avail > 0)) ? \
	408	(1) : \
	409	((unlikely(__urcu_cas_avail < 0) ? \
	410	(__urcu_cas_init()) : \
	411	(0))))
7d413817 MD	412
	413	extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
	414	unsigned long _new, int len);
	415
	416	#define compat_uatomic_cmpxchg(addr, old, _new) \
	417	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old),\
	418	(unsigned long)(_new), \
	419	sizeof(*(addr))))
	420	#endif
	421
ec4e58a3	422	#endif /* _URCU_ARCH_UATOMIC_X86_H */