[PATCH 3/3] Fix copy_user on x86_64

[Vitaly Mayatskikh]

Added copy_user_64.c instead of copy_user_64.S and
copy_user_nocache_64.S

diff --git a/arch/x86/lib/copy_user_64.c b/arch/x86/lib/copy_user_64.c
new file mode 100644
index 0000000..7317bf5
--- /dev/null
+++ b/arch/x86/lib/copy_user_64.c
@@ -0,0 +1,295 @@
+/*
+ * Copyright 2008 Vitaly Mayatskikh <vmayatsk@xxxxxxxxxx>
+ * Copyright 2002 Andi Kleen, SuSE Labs.
+ * Subject to the GNU Public License v2.
+ *
+ * Functions to copy from and to user space.
+ */
+
+#include <linux/module.h>
+#include <linux/uaccess.h>
+
+/*
+ * Try to copy last bytes and clear rest if needed.
+ * Since protection fault in copy_from/to_user is not a normal situation,
+ * it is not necessary to optimize tail handling .
+ */
+unsigned long
+copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest)
+{
+	char c;
+	unsigned zero_len;
+
+	for (; len; --len) {
+		if (__get_user_nocheck(c, from++, sizeof(char)))
+			break;
+		if (__put_user_nocheck(c, to++, sizeof(char)))
+			break;
+	}
+
+	for (c = 0, zero_len = len; zerorest && zero_len; --zero_len)
+		if (__put_user_nocheck(c, to++, sizeof(char)))
+			break;
+	return len;
+}
+
+/* Some CPUs run faster using the string copy instructions.
+ * This is also a lot simpler. Use them when possible.
+ *
+ * Only 4GB of copy is supported. This shouldn't be a problem
+ * because the kernel normally only writes from/to page sized chunks
+ * even if user space passed a longer buffer.
+ * And more would be dangerous because both Intel and AMD have
+ * errata with rep movsq > 4GB. If someone feels the need to fix
+ * this please consider this.
+ */
+inline unsigned long
+copy_user_generic_string(void *to, const void *from, unsigned len)
+{
+	unsigned long ret;
+	asm volatile (
+		"	movl %%ecx,%%edx\n"
+		"	shrl $3,%%ecx\n"
+		"	andl $7,%%edx\n"
+		"1:	rep; movsq\n"
+		"	movl %%edx,%%ecx\n"
+		"2:	rep; movsb\n"
+		"3:\n"
+		".section .fixup,\"ax\"\n"
+		"12:	xorl %%ecx,%%ecx\n"
+		"11:	leal (%%edx,%%ecx,8),%%ecx\n"
+		"	movl %%ecx,%%edx\n"	/* ecx is zerorest also */
+		"	call copy_user_handle_tail\n"
+		"	movl %%eax,%%ecx\n"
+		"	jmp 3b\n"
+		".previous\n"
+		".section __ex_table,\"a\"\n"
+		"	.quad 1b,11b\n"
+		"	.quad 2b,12b\n"
+		".previous"
+		: "=c"(ret)
+		: "D"(to), "S"(from), "c"(len)
+		: "eax", "edx", "memory"
+		);
+	return ret;
+}
+
+/*
+ * copy_user_generic_unrolled - memory copy with exception handling.
+ * This version is for CPUs like P4 that don't have efficient micro
+ * code for rep movsq
+ */
+inline unsigned long
+copy_user_generic_unrolled(void *to, const void *from, unsigned len)
+{
+	unsigned long ret;
+	asm volatile (
+		"	movl %%ecx,%%edx\n"
+		"	andl $63,%%edx\n"
+		"	shrl $6,%%ecx\n"
+		"	jz 17f\n"
+		"1:	movq (%%rsi),%%r8\n"
+		"2:	movq 1*8(%%rsi),%%r9\n"
+		"3:	movq 2*8(%%rsi),%%r10\n"
+		"4:	movq 3*8(%%rsi),%%r11\n"
+		"5:	movq %%r8,(%%rdi)\n"
+		"6:	movq %%r9,1*8(%%rdi)\n"
+		"7:	movq %%r10,2*8(%%rdi)\n"
+		"8:	movq %%r11,3*8(%%rdi)\n"
+		"9:	movq 4*8(%%rsi),%%r8\n"
+		"10:	movq 5*8(%%rsi),%%r9\n"
+		"11:	movq 6*8(%%rsi),%%r10\n"
+		"12:	movq 7*8(%%rsi),%%r11\n"
+		"13:	movq %%r8,4*8(%%rdi)\n"
+		"14:	movq %%r9,5*8(%%rdi)\n"
+		"15:	movq %%r10,6*8(%%rdi)\n"
+		"16:	movq %%r11,7*8(%%rdi)\n"
+		"	leaq 64(%%rsi),%%rsi\n"
+		"	leaq 64(%%rdi),%%rdi\n"
+		"	decl %%ecx\n"
+		"	jnz 1b\n"
+		"17:	movl %%edx,%%ecx\n"
+		"	andl $7,%%edx\n"
+		"	shrl $3,%%ecx\n"
+		"	jz 20f\n"
+		"18:	movq (%%rsi),%%r8\n"
+		"19:	movq %%r8,(%%rdi)\n"
+		"	leaq 8(%%rsi),%%rsi\n"
+		"	leaq 8(%%rdi),%%rdi\n"
+		"	decl %%ecx\n"
+		"	jnz 18b\n"
+		"20:	andl %%edx,%%edx\n"
+		"	jz 23f\n"
+		"	movl %%edx,%%ecx\n"
+		"21:	movb (%%rsi),%%al\n"
+		"22:	movb %%al,(%%rdi)\n"
+		"	incq %%rsi\n"
+		"	incq %%rdi\n"
+		"	decl %%ecx\n"
+		"	jnz 21b\n"
+		"23:\n"
+		".section .fixup,\"ax\"\n"
+		"30:	shll $6,%%ecx\n"
+		"	addl %%ecx,%%edx\n"
+		"	jmp 60f\n"
+		"40:	leal (%%edx,%%ecx,8),%%edx\n"
+		"	jmp 60f\n"
+		"50:	movl %%ecx,%%edx\n"
+		"60:\n"				/* ecx is zerorest also */
+		"	call copy_user_handle_tail\n"
+		"	movl %%eax,%%ecx\n"
+		"	jmp  23b\n"
+		".previous\n"
+		".section __ex_table,\"a\"\n"
+		"	.quad 1b,30b\n"
+		"	.quad 2b,30b\n"
+		"	.quad 3b,30b\n"
+		"	.quad 4b,30b\n"
+		"	.quad 5b,30b\n"
+		"	.quad 6b,30b\n"
+		"	.quad 7b,30b\n"
+		"	.quad 8b,30b\n"
+		"	.quad 9b,30b\n"
+		"	.quad 10b,30b\n"
+		"	.quad 11b,30b\n"
+		"	.quad 12b,30b\n"
+		"	.quad 13b,30b\n"
+		"	.quad 14b,30b\n"
+		"	.quad 15b,30b\n"
+		"	.quad 16b,30b\n"
+		"	.quad 18b,40b\n"
+		"	.quad 19b,40b\n"
+		"	.quad 21b,50b\n"
+		"	.quad 22b,50b\n"
+		".previous"
+		: "=c"(ret)
+		: "D"(to), "S"(from), "c"(len)
+		: "eax", "edx", "r8", "r9", "r10", "r11", "memory"
+		);
+	return ret;
+}
+
+/*
+ * copy_user_nocache - Uncached memory copy with exception handling
+ * This will force destination/source out of cache for more performance.
+ */
+long __copy_user_nocache(void *to, const void *from, unsigned len, int zerorest)
+{
+	unsigned long ret;
+	asm volatile (
+		"	movl %%ecx,%%edx\n"
+		"	andl $63,%%edx\n"
+		"	shrl $6,%%ecx\n"
+		"	jz 17f\n"
+		"1:	movq (%%rsi),%%r8\n"
+		"2:	movq 1*8(%%rsi),%%r9\n"
+		"3:	movq 2*8(%%rsi),%%r10\n"
+		"4:	movq 3*8(%%rsi),%%r11\n"
+		"5:	movnti %%r8,(%%rdi)\n"
+		"6:	movnti %%r9,1*8(%%rdi)\n"
+		"7:	movnti %%r10,2*8(%%rdi)\n"
+		"8:	movnti %%r11,3*8(%%rdi)\n"
+		"9:	movq 4*8(%%rsi),%%r8\n"
+		"10:	movq 5*8(%%rsi),%%r9\n"
+		"11:	movq 6*8(%%rsi),%%r10\n"
+		"12:	movq 7*8(%%rsi),%%r11\n"
+		"13:	movnti %%r8,4*8(%%rdi)\n"
+		"14:	movnti %%r9,5*8(%%rdi)\n"
+		"15:	movnti %%r10,6*8(%%rdi)\n"
+		"16:	movnti %%r11,7*8(%%rdi)\n"
+		"	leaq 64(%%rsi),%%rsi\n"
+		"	leaq 64(%%rdi),%%rdi\n"
+		"	decl %%ecx\n"
+		"	jnz 1b\n"
+		"17:	movl %%edx,%%ecx\n"
+		"	andl $7,%%edx\n"
+		"	shrl $3,%%ecx\n"
+		"	jz 20f\n"
+		"18:	movq (%%rsi),%%r8\n"
+		"19:	movnti %%r8,(%%rdi)\n"
+		"	leaq 8(%%rsi),%%rsi\n"
+		"	leaq 8(%%rdi),%%rdi\n"
+		"	decl %%ecx\n"
+		"	jnz 18b\n"
+		"20:	andl %%edx,%%edx\n"
+		"	jz 23f\n"
+		"	movl %%edx,%%ecx\n"
+		"21:	movb (%%rsi),%%al\n"
+		"22:	movb %%al,(%%rdi)\n"
+		"	incq %%rsi\n"
+		"	incq %%rdi\n"
+		"	decl %%ecx\n"
+		"	jnz 21b\n"
+		"23:	sfence\n"
+		".section .fixup,\"ax\"\n"
+		"30:	shll $6,%%ecx\n"
+		"	addl %%ecx,%%edx\n"
+		"	jmp 60f\n"
+		"40:	leal (%%edx,%%ecx,8),%%edx\n"
+		"	jmp 60f\n"
+		"50:	movl %%ecx,%%edx\n"
+		"60:	sfence\n"
+		"	movl %%ebx,%%ecx\n"
+		"	call copy_user_handle_tail\n"
+		"	movl %%eax,%%ecx\n"
+		"	jmp  23b\n"
+		".previous\n"
+		".section __ex_table,\"a\"\n"
+		"	.quad 1b,30b\n"
+		"	.quad 2b,30b\n"
+		"	.quad 3b,30b\n"
+		"	.quad 4b,30b\n"
+		"	.quad 5b,30b\n"
+		"	.quad 6b,30b\n"
+		"	.quad 7b,30b\n"
+		"	.quad 8b,30b\n"
+		"	.quad 9b,30b\n"
+		"	.quad 10b,30b\n"
+		"	.quad 11b,30b\n"
+		"	.quad 12b,30b\n"
+		"	.quad 13b,30b\n"
+		"	.quad 14b,30b\n"
+		"	.quad 15b,30b\n"
+		"	.quad 16b,30b\n"
+		"	.quad 18b,40b\n"
+		"	.quad 19b,40b\n"
+		"	.quad 21b,50b\n"
+		"	.quad 22b,50b\n"
+		".previous"
+		: "=c"(ret)
+		: "D"(to), "S"(from), "c"(len), "b"(zerorest)
+		: "eax", "edx", "r8", "r9", "r10", "r11", "memory"
+		);
+	return ret;
+}
+
+unsigned long copy_user_generic(void *to, const void *from, unsigned len)
+{
+	if (cpu_has(&boot_cpu_data, X86_FEATURE_REP_GOOD))
+		return copy_user_generic_string(to, from, len);
+	else
+		return copy_user_generic_unrolled(to, from, len);
+}
+
+/* Standard copy_to_user with segment limit checking */
+unsigned long copy_to_user(void __user *to, const void *from, unsigned len)
+{
+	if (access_ok(VERIFY_WRITE, to, len))
+		return copy_user_generic(to, from, len);
+	return len;
+}
+
+/* Standard copy_from_user with segment limit checking */
+unsigned long copy_from_user(void *to, const void __user *from, unsigned len)
+{
+	if (access_ok(VERIFY_READ, from, len))
+		return copy_user_generic(to, from, len);
+	else
+		memset(to, 0, len);
+	return len;
+}
+
+long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
+{
+	return copy_user_generic(dst, src, size);
+}

Signed-off-by: Vitaly Mayatskikh <v.mayatskih@xxxxxxxxx>

-- 
wbr, Vitaly