[urcu.git] / arch_atomic_x86.h

#ifndef _ARCH_ATOMIC_X86_H
#define _ARCH_ATOMIC_X86_H

/* 
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#ifndef BITS_PER_LONG
#define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
#endif

#ifndef _INCLUDE_API_H

/*
 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
 */

struct __atomic_dummy {
	unsigned long v[10];
};
#define __hp(x)	((struct __atomic_dummy *)(x))

/* cmpxchg */

static inline __attribute__((always_inline))
unsigned long _atomic_cmpxchg(volatile void *addr, unsigned long old,
		unsigned long _new, int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = old;
		__asm__ __volatile__(
		"lock; cmpxchgb %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "q"((unsigned char)_new)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result = old;
		__asm__ __volatile__(
		"lock; cmpxchgw %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned short)_new)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result = old;
		__asm__ __volatile__(
		"lock; cmpxchgl %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned int)_new)
			: "memory");
		return result;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned int result = old;
		__asm__ __volatile__(
		"lock; cmpxchgl %2, %1"
			: "+a"(result), "+m"(*__hp(addr))
			: "r"((unsigned long)_new)
			: "memory");
		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define cmpxchg(addr, old, _new)					    \
	((__typeof__(*(addr))) _atomic_cmpxchg((addr), (unsigned long)(old),\
						(unsigned long)(_new), 	    \
						sizeof(*(addr))))

/* xchg */

static inline __attribute__((always_inline))
unsigned long _atomic_exchange(volatile void *addr, unsigned long val, int len)
{
	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	switch (len) {
	case 1:
	{
		unsigned char result;
		__asm__ __volatile__(
		"xchgb %0, %1"
			: "=q"(result), "+m"(*__hp(addr))
			: "0" ((unsigned char)val)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result;
		__asm__ __volatile__(
		"xchgw %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned short)val)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result;
		__asm__ __volatile__(
		"xchgl %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned int)val)
			: "memory");
		return result;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result;
		__asm__ __volatile__(
		"xchgq %0, %1"
			: "=r"(result), "+m"(*__hp(addr))
			: "0" ((unsigned long)val)
			: "memory");
		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return 0;
}

#define xchg(addr, v)							    \
	((__typeof__(*(addr))) _atomic_exchange((addr), (unsigned long)(v), \
						sizeof(*(addr))))

/* atomic_add */

static inline __attribute__((always_inline))
void _atomic_add(volatile void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; addb %1, %0"
			: "=m"(*__hp(addr))
			: "q" ((unsigned char)val));
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; addw %1, %0"
			: "=m"(*__hp(addr))
			: "r" ((unsigned short)val));
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; addl %1, %0"
			: "=m"(*__hp(addr))
			: "r" ((unsigned int)val));
		return;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; addq %1, %0"
			: "=m"(*__hp(addr))
			: "r" ((unsigned long)val));
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define atomic_add(addr, v)						   \
	(_atomic_add((addr), (unsigned long)(v), sizeof(*(addr))))

#endif /* #ifndef _INCLUDE_API_H */

#endif /* ARCH_ATOMIC_X86_H */
Commit	Line	Data
0114ba7f MD	1	#ifndef _ARCH_ATOMIC_X86_H
	2	#define _ARCH_ATOMIC_X86_H
	3
	4	/*
	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	*
	10	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	11	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	12	*
	13	* Permission is hereby granted to use or copy this program
	14	* for any purpose, provided the above notices are retained on all copies.
	15	* Permission to modify the code and to distribute modified code is granted,
	16	* provided the above notices are retained, and a notice that the code was
	17	* modified is included with the above copyright notice.
	18	*
	19	* Code inspired from libatomic_ops-1.2, inherited in part from the
	20	* Boehm-Demers-Weiser conservative garbage collector.
	21	*/
	22
	23	#ifndef BITS_PER_LONG
	24	#define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
	25	#endif
	26
	27	#ifndef _INCLUDE_API_H
	28
	29	/*
0114ba7f MD	30	* Derived from AO_compare_and_swap() and AO_test_and_set_full().
	31	*/
	32
cc1be41b MD	33	struct __atomic_dummy {
	34	unsigned long v[10];
	35	};
	36	#define __hp(x) ((struct __atomic_dummy *)(x))
	37
	38	/* cmpxchg */
	39
5dba80f9	40	static inline __attribute__((always_inline))
cc1be41b MD	41	unsigned long _atomic_cmpxchg(volatile void *addr, unsigned long old,
cc1be41b MD	42	unsigned long _new, int len)
0114ba7f	43	{
cc1be41b MD	44	switch (len) {
	45	case 1:
	46	{
	47	unsigned char result = old;
	48	__asm__ __volatile__(
	49	"lock; cmpxchgb %2, %1"
	50	: "+a"(result), "+m"(*__hp(addr))
	51	: "q"((unsigned char)_new)
0114ba7f	52	: "memory");
cc1be41b MD	53	return result;
	54	}
	55	case 2:
	56	{
	57	unsigned short result = old;
	58	__asm__ __volatile__(
	59	"lock; cmpxchgw %2, %1"
	60	: "+a"(result), "+m"(*__hp(addr))
	61	: "r"((unsigned short)_new)
	62	: "memory");
	63	return result;
	64	}
	65	case 4:
	66	{
	67	unsigned int result = old;
	68	__asm__ __volatile__(
	69	"lock; cmpxchgl %2, %1"
	70	: "+a"(result), "+m"(*__hp(addr))
	71	: "r"((unsigned int)_new)
	72	: "memory");
	73	return result;
	74	}
	75	#if (BITS_PER_LONG == 64)
	76	case 8:
	77	{
	78	unsigned int result = old;
	79	__asm__ __volatile__(
	80	"lock; cmpxchgl %2, %1"
	81	: "+a"(result), "+m"(*__hp(addr))
	82	: "r"((unsigned long)_new)
	83	: "memory");
	84	return result;
	85	}
	86	#endif
	87	}
	88	/* generate an illegal instruction. Cannot catch this with linker tricks
	89	* when optimizations are disabled. */
	90	__asm__ __volatile__("ud2");
	91	return 0;
0114ba7f MD	92	}
0114ba7f MD	93
cc1be41b MD	94	#define cmpxchg(addr, old, _new) \
	95	((__typeof__(*(addr))) _atomic_cmpxchg((addr), (unsigned long)(old),\
	96	(unsigned long)(_new), \
	97	sizeof(*(addr))))
	98
	99	/* xchg */
0114ba7f	100
5dba80f9	101	static inline __attribute__((always_inline))
cc1be41b	102	unsigned long _atomic_exchange(volatile void *addr, unsigned long val, int len)
0114ba7f	103	{
cc1be41b MD	104	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	105	switch (len) {
	106	case 1:
	107	{
	108	unsigned char result;
	109	__asm__ __volatile__(
	110	"xchgb %0, %1"
	111	: "=q"(result), "+m"(*__hp(addr))
	112	: "0" ((unsigned char)val)
	113	: "memory");
	114	return result;
	115	}
	116	case 2:
	117	{
	118	unsigned short result;
	119	__asm__ __volatile__(
	120	"xchgw %0, %1"
	121	: "=r"(result), "+m"(*__hp(addr))
	122	: "0" ((unsigned short)val)
	123	: "memory");
	124	return result;
	125	}
	126	case 4:
	127	{
	128	unsigned int result;
	129	__asm__ __volatile__(
	130	"xchgl %0, %1"
	131	: "=r"(result), "+m"(*__hp(addr))
	132	: "0" ((unsigned int)val)
	133	: "memory");
	134	return result;
	135	}
	136	#if (BITS_PER_LONG == 64)
	137	case 8:
	138	{
	139	unsigned long result;
	140	__asm__ __volatile__(
0114ba7f	141	"xchgq %0, %1"
cc1be41b MD	142	: "=r"(result), "+m"(*__hp(addr))
cc1be41b MD	143	: "0" ((unsigned long)val)
0114ba7f	144	: "memory");
cc1be41b MD	145	return result;
	146	}
	147	#endif
	148	}
	149	/* generate an illegal instruction. Cannot catch this with linker tricks
	150	* when optimizations are disabled. */
	151	__asm__ __volatile__("ud2");
	152	return 0;
0114ba7f MD	153	}
0114ba7f MD	154
cc1be41b MD	155	#define xchg(addr, v) \
	156	((__typeof__(*(addr))) _atomic_exchange((addr), (unsigned long)(v), \
	157	sizeof(*(addr))))
	158
	159	/* atomic_add */
0114ba7f	160
5dba80f9	161	static inline __attribute__((always_inline))
a81b8e5e	162	void _atomic_add(volatile void *addr, unsigned long val, int len)
0114ba7f MD	163	{
0114ba7f MD	164	switch (len) {
cc1be41b MD	165	case 1:
	166	{
	167	__asm__ __volatile__(
	168	"lock; addb %1, %0"
	169	: "=m"(*__hp(addr))
	170	: "q" ((unsigned char)val));
	171	return;
	172	}
	173	case 2:
	174	{
	175	__asm__ __volatile__(
	176	"lock; addw %1, %0"
	177	: "=m"(*__hp(addr))
	178	: "r" ((unsigned short)val));
	179	return;
	180	}
	181	case 4:
	182	{
	183	__asm__ __volatile__(
	184	"lock; addl %1, %0"
	185	: "=m"(*__hp(addr))
	186	: "r" ((unsigned int)val));
	187	return;
	188	}
0114ba7f	189	#if (BITS_PER_LONG == 64)
cc1be41b MD	190	case 8:
	191	{
	192	__asm__ __volatile__(
	193	"lock; addq %1, %0"
	194	: "=m"(*__hp(addr))
	195	: "r" ((unsigned long)val));
	196	return;
	197	}
0114ba7f MD	198	#endif
	199	}
	200	/* generate an illegal instruction. Cannot catch this with linker tricks
	201	* when optimizations are disabled. */
	202	__asm__ __volatile__("ud2");
a81b8e5e	203	return;
0114ba7f MD	204	}
0114ba7f MD	205
cc1be41b MD	206	#define atomic_add(addr, v) \
cc1be41b MD	207	(_atomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
0114ba7f MD	208
	209	#endif /* #ifndef _INCLUDE_API_H */
	210
	211	#endif /* ARCH_ATOMIC_X86_H */