[PATCH] x86/uaccess: Implement get_kernel()

[Ingo Molnar]

Implement an optimized get_kernel() primitive on x86: avoid the
STAC/CLAC overhead. Only offer a 64-bit variant for now, 32-bit
will fall back to get_user().

Not-Yet-Signed-off-by: Ingo Molnar <mingo@xxxxxxxxxx>
---
 arch/x86/include/asm/uaccess_64.h | 32 ++++++++++++++++++++++++++++++++
 include/linux/uaccess.h           |  8 ++------
 kernel/locking/mutex.c            |  3 +--
 3 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index f2f9b39b274a..3ef75f0addac 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -207,6 +207,38 @@ __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
 	return __copy_from_user_nocheck(dst, src, size);
 }
 
+
+#define __get_kernel_asm_ex(dst, src, int_type, reg_type, lh_type)	\
+	asm volatile("1: mov"int_type" %1,%"reg_type"0\n"		\
+		     "2:\n"						\
+		     _ASM_EXTABLE_EX(1b, 2b)				\
+		     : lh_type(dst) : "m" (__m(src)))
+
+extern void __get_kernel_BUILD_ERROR(void);
+
+/*
+ * Simple copy-from-possibly-faulting-kernel-addresses method that
+ * avoids the STAC/CLAC SMAP overhead.
+ *
+ * NOTE: this does not propagate the error code of faulting kernel
+ *	 addresses properly. You can recover it via uaccess_catch()
+ *       if you really need to.
+ */
+#define get_kernel(dst, src)						\
+do {									\
+	typeof(*(src)) __val;						\
+									\
+	switch (sizeof(__val)) {					\
+	case 1:	__get_kernel_asm_ex(__val, src, "b", "b", "=q"); break;	\
+	case 2: __get_kernel_asm_ex(__val, src, "w", "w", "=r"); break;	\
+	case 4: __get_kernel_asm_ex(__val, src, "l", "k", "=r"); break;	\
+	case 8: __get_kernel_asm_ex(__val, src, "q", " ", "=r"); break;	\
+	default: __get_kernel_BUILD_ERROR();				\
+	}								\
+	(dst) = __val;							\
+} while (0)
+#define ARCH_HAS_GET_KERNEL
+
 static __must_check __always_inline int
 __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
 {
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index 885eea43b69f..03a025b98d2e 100644
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -111,12 +111,8 @@ extern long notrace __probe_kernel_write(void *dst, const void *src, size_t size
  * Generic wrapper, most architectures can just use __copy_from_user_inatomic()
  * to implement __copy_from_kernel_inatomic():
  */
-#ifndef ARCH_HAS_COPY_FROM_KERNEL_INATOMIC
-static __must_check __always_inline int
-__copy_from_kernel_inatomic(void *dst, const void __user *src, unsigned size)
-{
-	return __copy_from_user_inatomic(dst, src, size);
-}
+#ifndef ARCH_HAS_GET_KERNEL
+# define get_kernel(dst, src) get_user(dst, src)
 #endif
 
 #endif		/* __LINUX_UACCESS_H__ */
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index a4f74cda9fc4..68c750f4e8e8 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -226,7 +226,6 @@ static noinline
 bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
 {
 	int on_cpu;
-	int ret;
 
 	while (lock->owner == owner) {
 		/*
@@ -252,7 +251,7 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
 		 * NOTE2: We ignore failed copies, as the next iteration will clean
 		 *        up after us. This saves an extra branch in the common case.
 		 */
-		ret = __copy_from_kernel_inatomic(&on_cpu, &owner->on_cpu, sizeof(on_cpu));
+		get_kernel(on_cpu, &owner->on_cpu);
 
 		if (!on_cpu || need_resched())
 			return false;
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/