[urcu.git] / arch_atomic_ppc.h

#ifndef _ARCH_ATOMIC_PPC_H
#define _ARCH_ATOMIC_PPC_H

/* 
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009      Mathieu Desnoyers
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 * Code inspired from libatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <compiler.h>

#ifndef __SIZEOF_LONG__
#ifdef __powerpc64__
#define __SIZEOF_LONG__ 8
#else
#define __SIZEOF_LONG__ 4
#endif
#endif

#ifndef BITS_PER_LONG
#define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
#endif

#define ILLEGAL_INSTR	".long	0xd00d00"

#ifndef _INCLUDE_API_H

#define atomic_set(addr, v)				\
do {							\
	ACCESS_ONCE(*(addr)) = (v);			\
} while (0)

#define atomic_read(addr)	ACCESS_ONCE(*(addr))

/*
 * Using a isync as second barrier for exchange to provide acquire semantic.
 * According to atomic_ops/sysdeps/gcc/powerpc.h, the documentation is "fairly
 * explicit that this also has acquire semantics."
 * Derived from AO_compare_and_swap(), but removed the comparison.
 */

/* xchg */

static inline __attribute__((always_inline))
unsigned long _atomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 4:
	{
		unsigned int result;

		__asm__ __volatile__(
			"lwsync\n"
		"1:\t"	"lwarx %0,0,%1\n"	/* load and reserve */
			"stwcx. %2,0,%1\n"	/* else store conditional */
			"bne- 1b\n"	 	/* retry if lost reservation */
			"isync\n"
				: "=&r"(result)
				: "r"(addr), "r"(val)
				: "memory", "cc");

		return result;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result;

		__asm__ __volatile__(
			"lwsync\n"
		"1:\t"	"ldarx %0,0,%1\n"	/* load and reserve */
			"stdcx. %2,0,%1\n"	/* else store conditional */
			"bne- 1b\n"	 	/* retry if lost reservation */
			"isync\n"
				: "=&r"(result)
				: "r"(addr), "r"(val)
				: "memory", "cc");

		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__(ILLEGAL_INSTR);
	return 0;
}

#define xchg(addr, v)							    \
	((__typeof__(*(addr))) _atomic_exchange((addr), (unsigned long)(v), \
						sizeof(*(addr))))
/* cmpxchg */

static inline __attribute__((always_inline))
unsigned long _atomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
	case 4:
	{
		unsigned int old_val;

		__asm__ __volatile__(
			"lwsync\n"
		"1:\t"	"lwarx %0,0,%1\n"	/* load and reserve */
			"cmpd %0,%3\n"		/* if load is not equal to */
			"bne 2f\n"		/* old, fail */
			"stwcx. %2,0,%1\n"	/* else store conditional */
			"bne- 1b\n"	 	/* retry if lost reservation */
			"isync\n"
		"2:\n"
				: "=&r"(old_val)
				: "r"(addr), "r"((unsigned int)_new),
				  "r"((unsigned int)old)
				: "memory", "cc");

		return old_val;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long old_val;

		__asm__ __volatile__(
			"lwsync\n"
		"1:\t"	"ldarx %0,0,%1\n"	/* load and reserve */
			"cmpd %0,%3\n"		/* if load is not equal to */
			"bne 2f\n"		/* old, fail */
			"stdcx. %2,0,%1\n"	/* else store conditional */
			"bne- 1b\n"	 	/* retry if lost reservation */
			"isync\n"
		"2:\n"
				: "=&r"(old_val),
				: "r"(addr), "r"((unsigned long)_new),
				  "r"((unsigned long)old)
				: "memory", "cc");

		return old_val;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__(ILLEGAL_INSTR);
	return 0;
}


#define cmpxchg(addr, old, _new)					    \
	((__typeof__(*(addr))) _atomic_cmpxchg((addr), (unsigned long)(old),\
						(unsigned long)(_new), 	    \
						sizeof(*(addr))))

/* atomic_add_return */

static inline __attribute__((always_inline))
unsigned long _atomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
	case 4:
	{
		unsigned int result;

		__asm__ __volatile__(
			"lwsync\n"
		"1:\t"	"lwarx %0,0,%1\n"	/* load and reserve */
			"add %0,%2,%0\n"	/* add val to value loaded */
			"stwcx. %0,0,%1\n"	/* store conditional */
			"bne- 1b\n"	 	/* retry if lost reservation */
			"isync\n"
				: "=&r"(result)
				: "r"(addr), "r"(val)
				: "memory", "cc");

		return result;
	}
#if (BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result;

		__asm__ __volatile__(
			"lwsync\n"
		"1:\t"	"ldarx %0,0,%1\n"	/* load and reserve */
			"add %0,%2,%0\n"	/* add val to value loaded */
			"stdcx. %0,0,%1\n"	/* store conditional */
			"bne- 1b\n"	 	/* retry if lost reservation */
			"isync\n"
				: "=&r"(result)
				: "r"(addr), "r"(val)
				: "memory", "cc");

		return result;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__(ILLEGAL_INSTR);
	return 0;
}


#define atomic_add_return(addr, v)					\
	((__typeof__(*(addr))) _atomic_add_return((addr),		\
						  (unsigned long)(v),	\
						  sizeof(*(addr))))

/* atomic_sub_return, atomic_add, atomic_sub, atomic_inc, atomic_dec */

#define atomic_sub_return(addr, v)	atomic_add_return((addr), -(v))

#define atomic_add(addr, v)		(void)atomic_add_return((addr), (v))
#define atomic_sub(addr, v)		(void)atomic_sub_return((addr), (v))

#define atomic_inc(addr)		atomic_add((addr), 1)
#define atomic_dec(addr)		atomic_add((addr), -1)

#endif /* #ifndef _INCLUDE_API_H */

#endif /* ARCH_ATOMIC_PPC_H */
Commit	Line	Data
0114ba7f MD	1	#ifndef _ARCH_ATOMIC_PPC_H
	2	#define _ARCH_ATOMIC_PPC_H
	3
	4	/*
	5	* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
	6	* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
	7	* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
	8	* Copyright (c) 2009 Mathieu Desnoyers
	9	*
	10	* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
	11	* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
	12	*
	13	* Permission is hereby granted to use or copy this program
	14	* for any purpose, provided the above notices are retained on all copies.
	15	* Permission to modify the code and to distribute modified code is granted,
	16	* provided the above notices are retained, and a notice that the code was
	17	* modified is included with the above copyright notice.
	18	*
	19	* Code inspired from libatomic_ops-1.2, inherited in part from the
	20	* Boehm-Demers-Weiser conservative garbage collector.
	21	*/
	22
1315d277 MD	23	#include <compiler.h>
1315d277 MD	24
bcf80111 SM	25	#ifndef __SIZEOF_LONG__
	26	#ifdef __powerpc64__
	27	#define __SIZEOF_LONG__ 8
	28	#else
	29	#define __SIZEOF_LONG__ 4
	30	#endif
	31	#endif
	32
0114ba7f MD	33	#ifndef BITS_PER_LONG
	34	#define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
	35	#endif
	36
e7061ad2	37	#define ILLEGAL_INSTR ".long 0xd00d00"
0114ba7f MD	38
	39	#ifndef _INCLUDE_API_H
	40
5f8052b4 MD	41	#define atomic_set(addr, v) \
	42	do { \
	43	ACCESS_ONCE(*(addr)) = (v); \
	44	} while (0)
	45
	46	#define atomic_read(addr) ACCESS_ONCE(*(addr))
	47
0114ba7f MD	48	/*
	49	* Using a isync as second barrier for exchange to provide acquire semantic.
	50	* According to atomic_ops/sysdeps/gcc/powerpc.h, the documentation is "fairly
	51	* explicit that this also has acquire semantics."
	52	* Derived from AO_compare_and_swap(), but removed the comparison.
	53	*/
	54
f689dcbc MD	55	/* xchg */
f689dcbc MD	56
da1c1635	57	static inline __attribute__((always_inline))
5eb3201d	58	unsigned long _atomic_exchange(void *addr, unsigned long val, int len)
0114ba7f	59	{
f689dcbc MD	60	switch (len) {
	61	case 4:
	62	{
	63	unsigned int result;
	64
	65	__asm__ __volatile__(
	66	"lwsync\n"
	67	"1:\t" "lwarx %0,0,%1\n" /* load and reserve */
	68	"stwcx. %2,0,%1\n" /* else store conditional */
	69	"bne- 1b\n" /* retry if lost reservation */
	70	"isync\n"
	71	: "=&r"(result)
	72	: "r"(addr), "r"(val)
	73	: "memory", "cc");
	74
	75	return result;
	76	}
	77	#if (BITS_PER_LONG == 64)
	78	case 8:
	79	{
	80	unsigned long result;
	81
	82	__asm__ __volatile__(
	83	"lwsync\n"
	84	"1:\t" "ldarx %0,0,%1\n" /* load and reserve */
	85	"stdcx. %2,0,%1\n" /* else store conditional */
	86	"bne- 1b\n" /* retry if lost reservation */
	87	"isync\n"
	88	: "=&r"(result)
	89	: "r"(addr), "r"(val)
	90	: "memory", "cc");
	91
	92	return result;
	93	}
	94	#endif
	95	}
	96	/* generate an illegal instruction. Cannot catch this with linker tricks
	97	* when optimizations are disabled. */
	98	__asm__ __volatile__(ILLEGAL_INSTR);
	99	return 0;
0114ba7f MD	100	}
0114ba7f MD	101
da1c1635 MD	102	#define xchg(addr, v) \
	103	((__typeof__(*(addr))) _atomic_exchange((addr), (unsigned long)(v), \
	104	sizeof(*(addr))))
f689dcbc	105	/* cmpxchg */
0114ba7f	106
da1c1635	107	static inline __attribute__((always_inline))
5eb3201d	108	unsigned long _atomic_cmpxchg(void *addr, unsigned long old,
f689dcbc	109	unsigned long _new, int len)
0114ba7f	110	{
f689dcbc MD	111	switch (len) {
	112	case 4:
	113	{
	114	unsigned int old_val;
	115
	116	__asm__ __volatile__(
	117	"lwsync\n"
	118	"1:\t" "lwarx %0,0,%1\n" /* load and reserve */
	119	"cmpd %0,%3\n" /* if load is not equal to */
	120	"bne 2f\n" /* old, fail */
	121	"stwcx. %2,0,%1\n" /* else store conditional */
	122	"bne- 1b\n" /* retry if lost reservation */
	123	"isync\n"
	124	"2:\n"
e72f4937	125	: "=&r"(old_val)
f689dcbc MD	126	: "r"(addr), "r"((unsigned int)_new),
	127	"r"((unsigned int)old)
	128	: "memory", "cc");
	129
	130	return old_val;
	131	}
	132	#if (BITS_PER_LONG == 64)
	133	case 8:
	134	{
	135	unsigned long old_val;
	136
	137	__asm__ __volatile__(
	138	"lwsync\n"
	139	"1:\t" "ldarx %0,0,%1\n" /* load and reserve */
	140	"cmpd %0,%3\n" /* if load is not equal to */
	141	"bne 2f\n" /* old, fail */
	142	"stdcx. %2,0,%1\n" /* else store conditional */
	143	"bne- 1b\n" /* retry if lost reservation */
	144	"isync\n"
	145	"2:\n"
	146	: "=&r"(old_val),
	147	: "r"(addr), "r"((unsigned long)_new),
	148	"r"((unsigned long)old)
	149	: "memory", "cc");
	150
	151	return old_val;
	152	}
	153	#endif
	154	}
	155	/* generate an illegal instruction. Cannot catch this with linker tricks
	156	* when optimizations are disabled. */
	157	__asm__ __volatile__(ILLEGAL_INSTR);
	158	return 0;
0114ba7f MD	159	}
0114ba7f MD	160
da1c1635 MD	161
	162	#define cmpxchg(addr, old, _new) \
	163	((__typeof__(*(addr))) _atomic_cmpxchg((addr), (unsigned long)(old),\
	164	(unsigned long)(_new), \
	165	sizeof(*(addr))))
f689dcbc MD	166
f689dcbc MD	167	/* atomic_add_return */
0114ba7f	168
da1c1635	169	static inline __attribute__((always_inline))
5eb3201d	170	unsigned long _atomic_add_return(void *addr, unsigned long val,
f689dcbc	171	int len)
0114ba7f MD	172	{
0114ba7f MD	173	switch (len) {
f689dcbc MD	174	case 4:
	175	{
	176	unsigned int result;
	177
	178	__asm__ __volatile__(
	179	"lwsync\n"
	180	"1:\t" "lwarx %0,0,%1\n" /* load and reserve */
	181	"add %0,%2,%0\n" /* add val to value loaded */
	182	"stwcx. %0,0,%1\n" /* store conditional */
	183	"bne- 1b\n" /* retry if lost reservation */
	184	"isync\n"
	185	: "=&r"(result)
	186	: "r"(addr), "r"(val)
	187	: "memory", "cc");
	188
	189	return result;
	190	}
0114ba7f	191	#if (BITS_PER_LONG == 64)
f689dcbc MD	192	case 8:
	193	{
	194	unsigned long result;
	195
	196	__asm__ __volatile__(
	197	"lwsync\n"
	198	"1:\t" "ldarx %0,0,%1\n" /* load and reserve */
	199	"add %0,%2,%0\n" /* add val to value loaded */
	200	"stdcx. %0,0,%1\n" /* store conditional */
	201	"bne- 1b\n" /* retry if lost reservation */
	202	"isync\n"
	203	: "=&r"(result)
	204	: "r"(addr), "r"(val)
	205	: "memory", "cc");
	206
	207	return result;
	208	}
0114ba7f MD	209	#endif
	210	}
	211	/* generate an illegal instruction. Cannot catch this with linker tricks
	212	* when optimizations are disabled. */
	213	__asm__ __volatile__(ILLEGAL_INSTR);
	214	return 0;
	215	}
	216
da1c1635 MD	217
	218	#define atomic_add_return(addr, v) \
	219	((__typeof__(*(addr))) _atomic_add_return((addr), \
	220	(unsigned long)(v), \
	221	sizeof(*(addr))))
f689dcbc MD	222
	223	/* atomic_sub_return, atomic_add, atomic_sub, atomic_inc, atomic_dec */
	224
	225	#define atomic_sub_return(addr, v) atomic_add_return((addr), -(v))
	226
	227	#define atomic_add(addr, v) (void)atomic_add_return((addr), (v))
	228	#define atomic_sub(addr, v) (void)atomic_sub_return((addr), (v))
	229
8f41d19a MD	230	#define atomic_inc(addr) atomic_add((addr), 1)
8f41d19a MD	231	#define atomic_dec(addr) atomic_add((addr), -1)
0114ba7f MD	232
	233	#endif /* #ifndef _INCLUDE_API_H */
	234
	235	#endif /* ARCH_ATOMIC_PPC_H */