arch_atomic_x86.h

   1 #ifndef _ARCH_ATOMIC_X86_H
   2 #define _ARCH_ATOMIC_X86_H
   3
   4 /*
   5  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
   6  * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
   7  * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
   8  * Copyright (c) 2009      Mathieu Desnoyers
   9  *
  10  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  11  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  12  *
  13  * Permission is hereby granted to use or copy this program
  14  * for any purpose,  provided the above notices are retained on all copies.
  15  * Permission to modify the code and to distribute modified code is granted,
  16  * provided the above notices are retained, and a notice that the code was
  17  * modified is included with the above copyright notice.
  18  *
  19  * Code inspired from libatomic_ops-1.2, inherited in part from the
  20  * Boehm-Demers-Weiser conservative garbage collector.
  21  */
  22
  23 #ifndef BITS_PER_LONG
  24 #define BITS_PER_LONG   (__SIZEOF_LONG__ * 8)
  25 #endif
  26
  27 #ifndef _INCLUDE_API_H
  28
  29 /*
  30  * Derived from AO_compare_and_swap() and AO_test_and_set_full().
  31  */
  32
  33 struct __atomic_dummy {
  34         unsigned long v[10];
  35 };
  36 #define __hp(x) ((struct __atomic_dummy *)(x))
  37
  38 /* cmpxchg */
  39
  40 static inline __attribute__((always_inline))
  41 unsigned long _atomic_cmpxchg(volatile void *addr, unsigned long old,
  42                 unsigned long _new, int len)
  43 {
  44         switch (len) {
  45         case 1:
  46         {
  47                 unsigned char result = old;
  48                 __asm__ __volatile__(
  49                 "lock; cmpxchgb %2, %1"
  50                         : "+a"(result), "+m"(*__hp(addr))
  51                         : "q"((unsigned char)_new)
  52                         : "memory");
  53                 return result;
  54         }
  55         case 2:
  56         {
  57                 unsigned short result = old;
  58                 __asm__ __volatile__(
  59                 "lock; cmpxchgw %2, %1"
  60                         : "+a"(result), "+m"(*__hp(addr))
  61                         : "r"((unsigned short)_new)
  62                         : "memory");
  63                 return result;
  64         }
  65         case 4:
  66         {
  67                 unsigned int result = old;
  68                 __asm__ __volatile__(
  69                 "lock; cmpxchgl %2, %1"
  70                         : "+a"(result), "+m"(*__hp(addr))
  71                         : "r"((unsigned int)_new)
  72                         : "memory");
  73                 return result;
  74         }
  75 #if (BITS_PER_LONG == 64)
  76         case 8:
  77         {
  78                 unsigned int result = old;
  79                 __asm__ __volatile__(
  80                 "lock; cmpxchgl %2, %1"
  81                         : "+a"(result), "+m"(*__hp(addr))
  82                         : "r"((unsigned long)_new)
  83                         : "memory");
  84                 return result;
  85         }
  86 #endif
  87         }
  88         /* generate an illegal instruction. Cannot catch this with linker tricks
  89          * when optimizations are disabled. */
  90         __asm__ __volatile__("ud2");
  91         return 0;
  92 }
  93
  94 #define cmpxchg(addr, old, _new)                                            \
  95         ((__typeof__(*(addr))) _atomic_cmpxchg((addr), (unsigned long)(old),\
  96                                                 (unsigned long)(_new),      \
  97                                                 sizeof(*(addr))))
  98
  99 /* xchg */
 100
 101 static inline __attribute__((always_inline))
 102 unsigned long _atomic_exchange(volatile void *addr, unsigned long val, int len)
 103 {
 104         /* Note: the "xchg" instruction does not need a "lock" prefix. */
 105         switch (len) {
 106         case 1:
 107         {
 108                 unsigned char result;
 109                 __asm__ __volatile__(
 110                 "xchgb %0, %1"
 111                         : "=q"(result), "+m"(*__hp(addr))
 112                         : "0" ((unsigned char)val)
 113                         : "memory");
 114                 return result;
 115         }
 116         case 2:
 117         {
 118                 unsigned short result;
 119                 __asm__ __volatile__(
 120                 "xchgw %0, %1"
 121                         : "=r"(result), "+m"(*__hp(addr))
 122                         : "0" ((unsigned short)val)
 123                         : "memory");
 124                 return result;
 125         }
 126         case 4:
 127         {
 128                 unsigned int result;
 129                 __asm__ __volatile__(
 130                 "xchgl %0, %1"
 131                         : "=r"(result), "+m"(*__hp(addr))
 132                         : "0" ((unsigned int)val)
 133                         : "memory");
 134                 return result;
 135         }
 136 #if (BITS_PER_LONG == 64)
 137         case 8:
 138         {
 139                 unsigned long result;
 140                 __asm__ __volatile__(
 141                 "xchgq %0, %1"
 142                         : "=r"(result), "+m"(*__hp(addr))
 143                         : "0" ((unsigned long)val)
 144                         : "memory");
 145                 return result;
 146         }
 147 #endif
 148         }
 149         /* generate an illegal instruction. Cannot catch this with linker tricks
 150          * when optimizations are disabled. */
 151         __asm__ __volatile__("ud2");
 152         return 0;
 153 }
 154
 155 #define xchg(addr, v)                                                       \
 156         ((__typeof__(*(addr))) _atomic_exchange((addr), (unsigned long)(v), \
 157                                                 sizeof(*(addr))))
 158
 159 /* atomic_add */
 160
 161 static inline __attribute__((always_inline))
 162 unsigned long _atomic_add(volatile void *addr, unsigned long val, int len)
 163 {
 164         switch (len) {
 165         case 1:
 166         {
 167                 __asm__ __volatile__(
 168                 "lock; addb %1, %0"
 169                         : "=m"(*__hp(addr))
 170                         : "q" ((unsigned char)val));
 171                 return;
 172         }
 173         case 2:
 174         {
 175                 __asm__ __volatile__(
 176                 "lock; addw %1, %0"
 177                         : "=m"(*__hp(addr))
 178                         : "r" ((unsigned short)val));
 179                 return;
 180         }
 181         case 4:
 182         {
 183                 __asm__ __volatile__(
 184                 "lock; addl %1, %0"
 185                         : "=m"(*__hp(addr))
 186                         : "r" ((unsigned int)val));
 187                 return;
 188         }
 189 #if (BITS_PER_LONG == 64)
 190         case 8:
 191         {
 192                 __asm__ __volatile__(
 193                 "lock; addq %1, %0"
 194                         : "=m"(*__hp(addr))
 195                         : "r" ((unsigned long)val));
 196                 return;
 197         }
 198 #endif
 199         }
 200         /* generate an illegal instruction. Cannot catch this with linker tricks
 201          * when optimizations are disabled. */
 202         __asm__ __volatile__("ud2");
 203         return 0;
 204 }
 205
 206 #define atomic_add(addr, v)                                                \
 207         (_atomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
 208
 209 #endif /* #ifndef _INCLUDE_API_H */
 210
 211 #endif /* ARCH_ATOMIC_X86_H */