X-Git-Url: https://git.liburcu.org/?a=blobdiff_plain;f=arch_atomic_x86.h;h=6b33d453285f2e11535f30a9f9f7621b31a5c184;hb=f689dcbc075f17e1382bc560ce41b273173a9f46;hp=e9a0b3e851e13a3d17f893573cd2de1d89772044;hpb=0114ba7f23f86623c237baeb28ec8e4b39b9bb84;p=urcu.git

diff --git a/arch_atomic_x86.h b/arch_atomic_x86.h
index e9a0b3e..6b33d45 100644
--- a/arch_atomic_x86.h
+++ b/arch_atomic_x86.h
@@ -27,65 +27,297 @@
 #ifndef _INCLUDE_API_H
 
 /*
- * Using a isync as second barrier for exchange to provide acquire semantic.
- * According to atomic_ops/sysdeps/gcc/powerpc.h, the documentation is "fairly
- * explicit that this also has acquire semantics."
  * Derived from AO_compare_and_swap() and AO_test_and_set_full().
  */
 
-static __attribute__((always_inline))
-unsigned int atomic_exchange_32(volatile unsigned int *addr, unsigned int val)
-{
-	unsigned int result;
+struct __atomic_dummy {
+	unsigned long v[10];
+};
+#define __hp(x)	((struct __atomic_dummy *)(x))
 
-	__asm__ __volatile__(
-		/* Note: the "xchg" instruction does not need a "lock" prefix */
-		"xchgl %0, %1"
-			: "=&r"(result), "=m"(*addr)
-			: "0" (val), "m"(*addr)
-			: "memory");
+/* cmpxchg */
 
-	return result;
+static inline __attribute__((always_inline))
+unsigned long _atomic_cmpxchg(volatile void *addr, unsigned long old,
+		unsigned long _new, int len)
+{
+	switch (len) {
+	case 1:
+	{
+		unsigned char result = old;
+		__asm__ __volatile__(
+		"lock; cmpxchgb %2, %1"
+			: "+a"(result), "+m"(*__hp(addr))
+			: "q"((unsigned char)_new)
+			: "memory");
+		return result;
+	}
+	case 2:
+	{
+		unsigned short result = old;
+		__asm__ __volatile__(
+		"lock; cmpxchgw %2, %1"
+			: "+a"(result), "+m"(*__hp(addr))
+			: "r"((unsigned short)_new)
+			: "memory");
+		return result;
+	}
+	case 4:
+	{
+		unsigned int result = old;
+		__asm__ __volatile__(
+		"lock; cmpxchgl %2, %1"
+			: "+a"(result), "+m"(*__hp(addr))
+			: "r"((unsigned int)_new)
+			: "memory");
+		return result;
+	}
+#if (BITS_PER_LONG == 64)
+	case 8:
+	{
+		unsigned long result = old;
+		__asm__ __volatile__(
+		"lock; cmpxchgq %2, %1"
+			: "+a"(result), "+m"(*__hp(addr))
+			: "r"((unsigned long)_new)
+			: "memory");
+		return result;
+	}
+#endif
+	}
+	/* generate an illegal instruction. Cannot catch this with linker tricks
+	 * when optimizations are disabled. */
+	__asm__ __volatile__("ud2");
+	return 0;
 }
 
-#if (BITS_PER_LONG == 64)
+#define cmpxchg(addr, old, _new)					    \
+	((__typeof__(*(addr))) _atomic_cmpxchg((addr), (unsigned long)(old),\
+						(unsigned long)(_new), 	    \
+						sizeof(*(addr))))
 
-static __attribute__((always_inline))
-unsigned long atomic_exchange_64(volatile unsigned long *addr,
-				 unsigned long val)
-{
-	unsigned long result;
+/* xchg */
 
-	__asm__ __volatile__(
-		/* Note: the "xchg" instruction does not need a "lock" prefix */
+static inline __attribute__((always_inline))
+unsigned long _atomic_exchange(volatile void *addr, unsigned long val, int len)
+{
+	/* Note: the "xchg" instruction does not need a "lock" prefix. */
+	switch (len) {
+	case 1:
+	{
+		unsigned char result;
+		__asm__ __volatile__(
+		"xchgb %0, %1"
+			: "=q"(result), "+m"(*__hp(addr))
+			: "0" ((unsigned char)val)
+			: "memory");
+		return result;
+	}
+	case 2:
+	{
+		unsigned short result;
+		__asm__ __volatile__(
+		"xchgw %0, %1"
+			: "=r"(result), "+m"(*__hp(addr))
+			: "0" ((unsigned short)val)
+			: "memory");
+		return result;
+	}
+	case 4:
+	{
+		unsigned int result;
+		__asm__ __volatile__(
+		"xchgl %0, %1"
+			: "=r"(result), "+m"(*__hp(addr))
+			: "0" ((unsigned int)val)
+			: "memory");
+		return result;
+	}
+#if (BITS_PER_LONG == 64)
+	case 8:
+	{
+		unsigned long result;
+		__asm__ __volatile__(
 		"xchgq %0, %1"
-			: "=&r"(result), "=m"(*addr)
-			: "0" (val), "m"(*addr)
+			: "=r"(result), "+m"(*__hp(addr))
+			: "0" ((unsigned long)val)
 			: "memory");
+		return result;
+	}
+#endif
+	}
+	/* generate an illegal instruction. Cannot catch this with linker tricks
+	 * when optimizations are disabled. */
+	__asm__ __volatile__("ud2");
+	return 0;
+}
+
+#define xchg(addr, v)							    \
+	((__typeof__(*(addr))) _atomic_exchange((addr), (unsigned long)(v), \
+						sizeof(*(addr))))
 
-	return result;
+/* atomic_add, atomic_sub */
+
+static inline __attribute__((always_inline))
+void _atomic_add(volatile void *addr, unsigned long val, int len)
+{
+	switch (len) {
+	case 1:
+	{
+		__asm__ __volatile__(
+		"lock; addb %1, %0"
+			: "=m"(*__hp(addr))
+			: "iq" ((unsigned char)val)
+			: "memory");
+		return;
+	}
+	case 2:
+	{
+		__asm__ __volatile__(
+		"lock; addw %1, %0"
+			: "=m"(*__hp(addr))
+			: "ir" ((unsigned short)val)
+			: "memory");
+		return;
+	}
+	case 4:
+	{
+		__asm__ __volatile__(
+		"lock; addl %1, %0"
+			: "=m"(*__hp(addr))
+			: "ir" ((unsigned int)val)
+			: "memory");
+		return;
+	}
+#if (BITS_PER_LONG == 64)
+	case 8:
+	{
+		__asm__ __volatile__(
+		"lock; addq %1, %0"
+			: "=m"(*__hp(addr))
+			: "er" ((unsigned long)val)
+			: "memory");
+		return;
+	}
+#endif
+	}
+	/* generate an illegal instruction. Cannot catch this with linker tricks
+	 * when optimizations are disabled. */
+	__asm__ __volatile__("ud2");
+	return;
 }
 
+#define atomic_add(addr, v)						   \
+	(_atomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
+
+#define atomic_sub(addr, v)	atomic_add((addr), -(v))
+
+
+/* atomic_inc */
+
+static inline __attribute__((always_inline))
+void _atomic_inc(volatile void *addr, int len)
+{
+	switch (len) {
+	case 1:
+	{
+		__asm__ __volatile__(
+		"lock; incb %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+	case 2:
+	{
+		__asm__ __volatile__(
+		"lock; incw %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+	case 4:
+	{
+		__asm__ __volatile__(
+		"lock; incl %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+#if (BITS_PER_LONG == 64)
+	case 8:
+	{
+		__asm__ __volatile__(
+		"lock; incq %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
 #endif
+	}
+	/* generate an illegal instruction. Cannot catch this with linker tricks
+	 * when optimizations are disabled. */
+	__asm__ __volatile__("ud2");
+	return;
+}
 
-static __attribute__((always_inline))
-unsigned long _atomic_exchange(volatile void *addr, unsigned long val, int len)
+#define atomic_inc(addr)	(_atomic_inc((addr), sizeof(*(addr))))
+
+/* atomic_dec */
+
+static inline __attribute__((always_inline))
+void _atomic_dec(volatile void *addr, int len)
 {
 	switch (len) {
-	case 4:	return atomic_exchange_32(addr, val);
+	case 1:
+	{
+		__asm__ __volatile__(
+		"lock; decb %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+	case 2:
+	{
+		__asm__ __volatile__(
+		"lock; decw %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
+	case 4:
+	{
+		__asm__ __volatile__(
+		"lock; decl %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
 #if (BITS_PER_LONG == 64)
-	case 8:	return atomic_exchange_64(addr, val);
+	case 8:
+	{
+		__asm__ __volatile__(
+		"lock; decq %0"
+			: "=m"(*__hp(addr))
+			:
+			: "memory");
+		return;
+	}
 #endif
 	}
 	/* generate an illegal instruction. Cannot catch this with linker tricks
 	 * when optimizations are disabled. */
 	__asm__ __volatile__("ud2");
-	return 0;
+	return;
 }
 
-#define xchg(addr, v)							    \
-	((__typeof__(*(addr))) _atomic_exchange((addr), (unsigned long)(v), \
-						sizeof(*(addr))))
+#define atomic_dec(addr)	(_atomic_dec((addr), sizeof(*(addr))))
 
 #endif /* #ifndef _INCLUDE_API_H */