WebSVN – Kolibri OS – Diff – /drivers/include/linux/asm/cmpxchg_32.h

 #define __xchg(x, ptr, size)						\
 ({									\
 	__typeof(*(ptr)) __x = (x);					\
 	switch (size) {							\
 	case 1:								\
+	{								\
+		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
-		asm volatile("xchgb %b0,%1"				\
+		asm volatile("xchgb %0,%1"				\
-			     : "=q" (__x)				\
+			     : "=q" (__x), "+m" (*__ptr)		\
-			     : "m" (*__xg(ptr)), "0" (__x)		\
+			     : "0" (__x)				\
 			     : "memory");				\
 		break;							\
+	}								\
 	case 2:								\
+	{								\
+		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
-		asm volatile("xchgw %w0,%1"				\
+		asm volatile("xchgw %0,%1"				\
-			     : "=r" (__x)				\
+			     : "=r" (__x), "+m" (*__ptr)		\
-			     : "m" (*__xg(ptr)), "0" (__x)		\
+			     : "0" (__x)				\
 			     : "memory");				\
 		break;							\
+	}								\
 	case 4:								\
+	{								\
+		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
 		asm volatile("xchgl %0,%1"				\
-			     : "=r" (__x)				\
+			     : "=r" (__x), "+m" (*__ptr)		\
-			     : "m" (*__xg(ptr)), "0" (__x)		\
+			     : "0" (__x)				\
 			     : "memory");				\
 		break;							\
+	}								\
 	default:							\
 		__xchg_wrong_size();					\
 	}								\
 	__x;								\
 })
 #define xchg(ptr, v)							\
 	__xchg((v), (ptr), sizeof(*ptr))
 /*
- * The semantics of XCHGCMP8B are a bit strange, this is why
+ * CMPXCHG8B only writes to the target if we had the previous
- * there is a loop and the loading of %%eax and %%edx has to
+ * value in registers, otherwise it acts as a read and gives us the
- * be inside. This inlines well in most cases, the cached
- * cost is around ~38 cycles. (in the future we might want
- * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that
+ * "new previous" value.  That is why there is a loop.  Preloading
- * might have an implicit FPU-save as a cost, so it's not
+ * EDX:EAX is a performance optimization: in the common case it means
+ * we need only one locked operation.
+ *
+ * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
- * clear which path to go.)
+ * least an FPU save and/or %cr0.ts manipulation.
  *
- * cmpxchg8b must be used with the lock prefix here to allow
- * the instruction to be executed atomically, see page 3-102
+ * cmpxchg8b must be used with the lock prefix here to allow the
- * of the instruction set reference 24319102.pdf. We need
+ * instruction to be executed atomically.  We need to have the reader
- * the reader side to see the coherent 64bit value.
+ * side to see the coherent 64bit value.
- */
-static inline void __set_64bit(unsigned long long *ptr,
+ */
+static inline void set_64bit(volatile u64 *ptr, u64 value)
+{
+	u32 low  = value;
+	u32 high = value >> 32;
-			       unsigned int low, unsigned int high)
+	u64 prev = *ptr;
-{
-	asm volatile("\n1:\t"
-		     "movl (%0), %%eax\n\t"
-		     "movl 4(%0), %%edx\n\t"
+	asm volatile("\n1:\t"
-		     LOCK_PREFIX "cmpxchg8b (%0)\n\t"
-		     "jnz 1b"
+		     LOCK_PREFIX "cmpxchg8b %0\n\t"
-		     : /* no outputs */
-		     : "D"(ptr),
+		     "jnz 1b"
-		       "b"(low),
+		     : "=m" (*ptr), "+A" (prev)
-		       "c"(high)
-		     : "ax", "dx", "memory");
-}
-static inline void __set_64bit_constant(unsigned long long *ptr,
-					unsigned long long value)
-{
-	__set_64bit(ptr, (unsigned int)value, (unsigned int)(value >> 32));
-}
-#define ll_low(x)	*(((unsigned int *)&(x)) + 0)
-#define ll_high(x)	*(((unsigned int *)&(x)) + 1)
-static inline void __set_64bit_var(unsigned long long *ptr,
-				   unsigned long long value)
 		     : "b" (low), "c" (high)
-	__set_64bit(ptr, ll_low(value), ll_high(value));
-}
-#define set_64bit(ptr, value)			\
-	(__builtin_constant_p((value))		\
-	 ? __set_64bit_constant((ptr), (value))	\
-	 : __set_64bit_var((ptr), (value)))
-#define _set_64bit(ptr, value)						\
-	(__builtin_constant_p(value)					\
-	 ? __set_64bit(ptr, (unsigned int)(value),			\
-		       (unsigned int)((value) >> 32))			\
+		     : "memory");
 	 : __set_64bit(ptr, ll_low((value)), ll_high((value))))
 extern void __cmpxchg_wrong_size(void);
 	__typeof__(*(ptr)) __ret;					\
 	__typeof__(*(ptr)) __old = (old);				\
 	__typeof__(*(ptr)) __new = (new);				\
 	switch (size) {							\
 	case 1:								\
+	{								\
+		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
-		asm volatile(lock "cmpxchgb %b1,%2"			\
+		asm volatile(lock "cmpxchgb %2,%1"			\
-			     : "=a"(__ret)				\
+			     : "=a" (__ret), "+m" (*__ptr)		\
-			     : "q"(__new), "m"(*__xg(ptr)), "0"(__old)	\
+			     : "q" (__new), "0" (__old)			\
 			     : "memory");				\
 		break;							\
+	}								\
 	case 2:								\
+	{								\
+		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
-		asm volatile(lock "cmpxchgw %w1,%2"			\
+		asm volatile(lock "cmpxchgw %2,%1"			\
-			     : "=a"(__ret)				\
+			     : "=a" (__ret), "+m" (*__ptr)		\
-			     : "r"(__new), "m"(*__xg(ptr)), "0"(__old)	\
+			     : "r" (__new), "0" (__old)			\
 			     : "memory");				\
 		break;							\
+	}								\
 	case 4:								\
+	{								\
+		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
-		asm volatile(lock "cmpxchgl %1,%2"			\
+		asm volatile(lock "cmpxchgl %2,%1"			\
-			     : "=a"(__ret)				\
+			     : "=a" (__ret), "+m" (*__ptr)		\
-			     : "r"(__new), "m"(*__xg(ptr)), "0"(__old)	\
+			     : "r" (__new), "0" (__old)			\
 			     : "memory");				\
 		break;							\
+	}								\
 	default:							\
 		__cmpxchg_wrong_size();					\
 	}								\
 	__ret;								\
 })
 #define cmpxchg64_local(ptr, o, n)					\
 	((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
 					       (unsigned long long)(n)))
 #endif
-static inline unsigned long long __cmpxchg64(volatile void *ptr,
-					     unsigned long long old,
-					     unsigned long long new)
+static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
 {
-	unsigned long long prev;
+	u64 prev;
-	asm volatile(LOCK_PREFIX "cmpxchg8b %3"
+	asm volatile(LOCK_PREFIX "cmpxchg8b %1"
-		     : "=A"(prev)
+		     : "=A" (prev),
-		     : "b"((unsigned long)new),
+		       "+m" (*ptr)
-		       "c"((unsigned long)(new >> 32)),
+		     : "b" ((u32)new),
-		       "m"(*__xg(ptr)),
+		       "c" ((u32)(new >> 32)),
-		       "0"(old)
+		       "0" (old)
 		     : "memory");
 	return prev;
 }
-static inline unsigned long long __cmpxchg64_local(volatile void *ptr,
 						   unsigned long long old,
-						   unsigned long long new)
+static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
 {
-	unsigned long long prev;
+	u64 prev;
-	asm volatile("cmpxchg8b %3"
+	asm volatile("cmpxchg8b %1"
-		     : "=A"(prev)
+		     : "=A" (prev),
-		     : "b"((unsigned long)new),
+		       "+m" (*ptr)
-		       "c"((unsigned long)(new >> 32)),
+		     : "b" ((u32)new),
-		       "m"(*__xg(ptr)),
+		       "c" ((u32)(new >> 32)),
-		       "0"(old)
+		       "0" (old)
 		     : "memory");
  * Building a kernel capable running on 80386. It may be necessary to
  * simulate the cmpxchg on the 80386 CPU. For that purpose we define
  * a function for each of the sizes we support.
  */
+extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
+extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
+extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
+static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
+					unsigned long new, int size)
+{
+	switch (size) {
+	case 1:
+		return cmpxchg_386_u8(ptr, old, new);
+	case 2:
+		return cmpxchg_386_u16(ptr, old, new);
+	case 4:
+		return cmpxchg_386_u32(ptr, old, new);
+	}
+	return old;
+}
 #define cmpxchg(ptr, o, n)						\
 ({									\
 	__typeof__(*(ptr)) __ret;					\
-    __ret = (__typeof__(*(ptr)))__cmpxchg((ptr),        \
+		__ret = (__typeof__(*(ptr)))__cmpxchg((ptr),		\
 /*
  * Building a kernel capable running on 80386 and 80486. It may be necessary
  * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
  */
-extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
 #define cmpxchg64(ptr, o, n)					\
 ({								\
 	__typeof__(*(ptr)) __ret;				\
 	__typeof__(*(ptr)) __old = (o);				\
+	__typeof__(*(ptr)) __new = (n);				\
-	__typeof__(*(ptr)) __new = (n);				\
+	alternative_io(LOCK_PREFIX_HERE				\
-	alternative_io("call cmpxchg8b_emu",			\
+			"call cmpxchg8b_emu",			\
 			"lock; cmpxchg8b (%%esi)" ,		\
 		       X86_FEATURE_CX8,				\
 		       "=A" (__ret),				\
 		       "S" ((ptr)), "0" (__old),		\
 		       "b" ((unsigned int)__new),		\
 		       "c" ((unsigned int)(__new>>32))		\
 		       : "memory");				\
-	__ret; })
 	__ret; })
-#define cmpxchg64_local(ptr, o, n)					\
+#define cmpxchg64_local(ptr, o, n)				\
-({									\
+({								\
+	__typeof__(*(ptr)) __ret;				\
-	__typeof__(*(ptr)) __ret;					\
+	__typeof__(*(ptr)) __old = (o);				\
-	if (likely(boot_cpu_data.x86 > 4))				\
+	__typeof__(*(ptr)) __new = (n);				\
-		__ret = (__typeof__(*(ptr)))__cmpxchg64_local((ptr),	\
+	alternative_io("call cmpxchg8b_emu",			\
-				(unsigned long long)(o),		\
+		       "cmpxchg8b (%%esi)" ,			\
-				(unsigned long long)(n));		\
+		       X86_FEATURE_CX8,				\
-	else								\
+		       "=A" (__ret),				\
-		__ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr),	\
+		       "S" ((ptr)), "0" (__old),		\
-				(unsigned long long)(o),		\
+		       "b" ((unsigned int)__new),		\
-				(unsigned long long)(n));		\
+		       "c" ((unsigned int)(__new>>32))		\
-	__ret;								\
+		       : "memory");				\

Subversion Repositories Kolibri OS

(root)/drivers/include/linux/asm/cmpxchg_32.h – Rev 1408 → 1631