X-Git-Url: http://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=arch_atomic_x86.h;h=3eedc3f5e0fba51f46105e10f5d60a59fbe0aa7f;hp=3422cb4734a0a5e2dd57ff2147f86cd783b58c97;hb=ba59a0c7b244a0939a2298fc76a9002436ef9674;hpb=a81b8e5e402188a088c28e27c5e718b691109cf2

diff --git a/arch_atomic_x86.h b/arch_atomic_x86.h
index 3422cb4..3eedc3f 100644
--- a/arch_atomic_x86.h
+++ b/arch_atomic_x86.h
@@ -20,6 +20,8 @@
  * Boehm-Demers-Weiser conservative garbage collector.
  */
 
+#include <compiler.h>
+
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
@@ -35,16 +37,24 @@ struct __atomic_dummy {
 };
 #define __hp(x)	((struct __atomic_dummy *)(x))
 
+#define atomic_set(addr, v)				\
+do {							\
+	ACCESS_ONCE(*(addr)) = (v);			\
+} while (0)
+
+#define atomic_read(addr)	ACCESS_ONCE(*(addr))
+
 /* cmpxchg */
 
 static inline __attribute__((always_inline))
-unsigned long _atomic_cmpxchg(volatile void *addr, unsigned long old,
-		unsigned long _new, int len)
+unsigned long _atomic_cmpxchg(void *addr, unsigned long old,
+			      unsigned long _new, int len)
 {
 	switch (len) {
 	case 1:
 	{
 		unsigned char result = old;
+
 		__asm__ __volatile__(
 		"lock; cmpxchgb %2, %1"
 			: "+a"(result), "+m"(*__hp(addr))
@@ -55,6 +65,7 @@ unsigned long _atomic_cmpxchg(volatile void *addr, unsigned long old,
 	case 2:
 	{
 		unsigned short result = old;
+
 		__asm__ __volatile__(
 		"lock; cmpxchgw %2, %1"
 			: "+a"(result), "+m"(*__hp(addr))
@@ -65,6 +76,7 @@ unsigned long _atomic_cmpxchg(volatile void *addr, unsigned long old,
 	case 4:
 	{
 		unsigned int result = old;
+
 		__asm__ __volatile__(
 		"lock; cmpxchgl %2, %1"
 			: "+a"(result), "+m"(*__hp(addr))
@@ -75,9 +87,10 @@ unsigned long _atomic_cmpxchg(volatile void *addr, unsigned long old,
 #if (BITS_PER_LONG == 64)
 	case 8:
 	{
-		unsigned int result = old;
+		unsigned long result = old;
+
 		__asm__ __volatile__(
-		"lock; cmpxchgl %2, %1"
+		"lock; cmpxchgq %2, %1"
 			: "+a"(result), "+m"(*__hp(addr))
 			: "r"((unsigned long)_new)
 			: "memory");
@@ -99,7 +112,7 @@ unsigned long _atomic_cmpxchg(volatile void *addr, unsigned long old,
 /* xchg */
 
 static inline __attribute__((always_inline))
-unsigned long _atomic_exchange(volatile void *addr, unsigned long val, int len)
+unsigned long _atomic_exchange(void *addr, unsigned long val, int len)
 {
 	/* Note: the "xchg" instruction does not need a "lock" prefix. */
 	switch (len) {
@@ -156,10 +169,77 @@ unsigned long _atomic_exchange(volatile void *addr, unsigned long val, int len)
 	((__typeof__(*(addr))) _atomic_exchange((addr), (unsigned long)(v), \
 						sizeof(*(addr))))
 
-/* atomic_add */
+/* atomic_add_return, atomic_sub_return */
 
 static inline __attribute__((always_inline))
-void _atomic_add(volatile void *addr, unsigned long val, int len)
+unsigned long _atomic_add_return(void *addr, unsigned long val,
+				 int len)
+{
+	switch (len) {
+	case 1:
+	{
+		unsigned char result = val;
+
+		__asm__ __volatile__(
+		"lock; xaddb %1, %0"
+			: "+m"(*__hp(addr)), "+q" (result)
+			:
+			: "memory");
+		return result + (unsigned char)val;
+	}
+	case 2:
+	{
+		unsigned short result = val;
+
+		__asm__ __volatile__(
+		"lock; xaddw %1, %0"
+			: "+m"(*__hp(addr)), "+r" (result)
+			:
+			: "memory");
+		return result + (unsigned short)val;
+	}
+	case 4:
+	{
+		unsigned int result = val;
+
+		__asm__ __volatile__(
+		"lock; xaddl %1, %0"
+			: "+m"(*__hp(addr)), "+r" (result)
+			:
+			: "memory");
+		return result + (unsigned int)val;
+	}
+#if (BITS_PER_LONG == 64)
+	case 8:
+	{
+		unsigned long result = val;
+
+		__asm__ __volatile__(
+		"lock; xaddq %1, %0"
+			: "+m"(*__hp(addr)), "+r" (result)
+			:
+			: "memory");
+		return result + (unsigned long)val;
+	}
+#endif
+	}
+	/* generate an illegal instruction. Cannot catch this with linker tricks
+	 * when optimizations are disabled. */
+	__asm__ __volatile__("ud2");
+	return 0;
+}
+
+#define atomic_add_return(addr, v)					\
+	((__typeof__(*(addr))) _atomic_add_return((addr),		\
+						  (unsigned long)(v),	\
+						  sizeof(*(addr))))
+
+#define atomic_sub_return(addr, v)	atomic_add_return((addr), -(v))
+
+/* atomic_add, atomic_sub */
+
+static inline __attribute__((always_inline))
+void _atomic_add(void *addr, unsigned long val, int len)
 {
 	switch (len) {
 	case 1:
@@ -167,7 +247,8 @@ void _atomic_add(volatile void *addr, unsigned long val, int len)
 		__asm__ __volatile__(
 		"lock; addb %1, %0"
 			: "=m"(*__hp(addr))
-			: "q" ((unsigned char)val));
+			: "iq" ((unsigned char)val)
+			: "memory");
 		return;
 	}
 	case 2:
@@ -175,7 +256,8 @@ void _atomic_add(volatile void *addr, unsigned long val, int len)
 		__asm__ __volatile__(
 		"lock; addw %1, %0"
 			: "=m"(*__hp(addr))
-			: "r" ((unsigned short)val));
+			: "ir" ((unsigned short)val)
+			: "memory");
 		return;
 	}
 	case 4:
@@ -183,7 +265,8 @@ void _atomic_add(volatile void *addr, unsigned long val, int len)
 		__asm__ __volatile__(
 		"lock; addl %1, %0"
 			: "=m"(*__hp(addr))
-			: "r" ((unsigned int)val));
+			: "ir" ((unsigned int)val)
+			: "memory");
 		return;
 	}
 #if (BITS_PER_LONG == 64)
@@ -192,7 +275,8 @@ void _atomic_add(volatile void *addr, unsigned long val, int len)
 		__asm__ __volatile__(
 		"lock; addq %1, %0"
 			: "=m"(*__hp(addr))
-			: "r" ((unsigned long)val));
+			: "er" ((unsigned long)val)
+			: "memory");
 		return;
 	}
 #endif
@@ -206,6 +290,115 @@ void _atomic_add(volatile void *addr, unsigned long val, int len)
 #define atomic_add(addr, v)						   \
 	(_atomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
 
+#define atomic_sub(addr, v)	atomic_add((addr), -(v))
+
+
+/* atomic_inc */
+
+static inline __attribute__((always_inline))
+void _atomic_inc(void *addr, int len)
+{
+	switch (len) {
+	case 1:
+	{
+		__asm__ __volatile__(
+		"lock; incb %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+	case 2:
+	{
+		__asm__ __volatile__(
+		"lock; incw %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+	case 4:
+	{
+		__asm__ __volatile__(
+		"lock; incl %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+#if (BITS_PER_LONG == 64)
+	case 8:
+	{
+		__asm__ __volatile__(
+		"lock; incq %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+#endif
+	}
+	/* generate an illegal instruction. Cannot catch this with linker tricks
+	 * when optimizations are disabled. */
+	__asm__ __volatile__("ud2");
+	return;
+}
+
+#define atomic_inc(addr)	(_atomic_inc((addr), sizeof(*(addr))))
+
+/* atomic_dec */
+
+static inline __attribute__((always_inline))
+void _atomic_dec(void *addr, int len)
+{
+	switch (len) {
+	case 1:
+	{
+		__asm__ __volatile__(
+		"lock; decb %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+	case 2:
+	{
+		__asm__ __volatile__(
+		"lock; decw %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+	case 4:
+	{
+		__asm__ __volatile__(
+		"lock; decl %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+#if (BITS_PER_LONG == 64)
+	case 8:
+	{
+		__asm__ __volatile__(
+		"lock; decq %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+#endif
+	}
+	/* generate an illegal instruction. Cannot catch this with linker tricks
+	 * when optimizations are disabled. */
+	__asm__ __volatile__("ud2");
+	return;
+}
+
+#define atomic_dec(addr)	(_atomic_dec((addr), sizeof(*(addr))))
+
 #endif /* #ifndef _INCLUDE_API_H */
 
 #endif /* ARCH_ATOMIC_X86_H */