From 3a8b8e63190961db76718cd96640ac535bb6917a Mon Sep 17 00:00:00 2001
From: Linus Torvalds <torvalds@linux-foundation.org>
Date: Thu, 3 May 2012 10:16:43 -0700
Subject: [PATCH] vfs: make word-at-a-time accesses handle a non-existing page

It turns out that there are more cases than CONFIG_DEBUG_PAGEALLOC that
can have holes in the kernel address space: it seems to happen easily
with Xen, and it looks like the AMD gart64 code will also punch holes
dynamically.

Actually hitting that case is still very unlikely, so just do the
access, and take an exception and fix it up for the very unlikely case
of it being a page-crosser with no next page.

And hey, this abstraction might even help other architectures that have
other issues with unaligned word accesses than the possible missing next
page.  IOW, this could do the byte order magic too.

Peter Anvin fixed a thinko in the shifting for the exception case.

Cc:  Peter Anvin <hpa@zytor.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 arch/x86/Kconfig                      |    2 +-
 arch/x86/include/asm/word-at-a-time.h |   37 +++++++++++++++++++++++++++++++++
 fs/namei.c                            |    4 ++--
 3 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 1d14cc6b79ad..c9866b0b77d8 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -81,7 +81,7 @@ config X86
 	select CLKEVT_I8253
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
 	select GENERIC_IOMAP
-	select DCACHE_WORD_ACCESS if !DEBUG_PAGEALLOC
+	select DCACHE_WORD_ACCESS
 
 config INSTRUCTION_DECODER
 	def_bool (KPROBES || PERF_EVENTS)
diff --git a/arch/x86/include/asm/word-at-a-time.h b/arch/x86/include/asm/word-at-a-time.h
index 6fe6767b7124..f609f07e6b93 100644
--- a/arch/x86/include/asm/word-at-a-time.h
+++ b/arch/x86/include/asm/word-at-a-time.h
@@ -22,6 +22,8 @@ static inline long count_masked_bytes(unsigned long mask)
 	return mask*0x0001020304050608ul >> 56;
 }
 
+#define __WORDSUFFIX "q"
+
 #else	/* 32-bit case */
 
 /* Carl Chatfield / Jan Achrenius G+ version for 32-bit */
@@ -33,6 +35,8 @@ static inline long count_masked_bytes(long mask)
 	return a & mask;
 }
 
+#define __WORDSUFFIX "l"
+
 #endif
 
 #define REPEAT_BYTE(x)	((~0ul / 0xff) * (x))
@@ -43,4 +47,37 @@ static inline unsigned long has_zero(unsigned long a)
 	return ((a - REPEAT_BYTE(0x01)) & ~a) & REPEAT_BYTE(0x80);
 }
 
+/*
+ * Load an unaligned word from kernel space.
+ *
+ * In the (very unlikely) case of the word being a page-crosser
+ * and the next page not being mapped, take the exception and
+ * return zeroes in the non-existing part.
+ */
+static inline unsigned long load_unaligned_zeropad(const void *addr)
+{
+	unsigned long ret, dummy;
+
+	asm(
+		"1:\tmov" __WORDSUFFIX " %2,%0\n"
+		"2:\n"
+		".section .fixup,\"ax\"\n"
+		"3:\t"
+		"lea" __WORDSUFFIX " %2,%1\n\t"
+		"and" __WORDSUFFIX " %3,%1\n\t"
+		"mov" __WORDSUFFIX " (%1),%0\n\t"
+		"lea" __WORDSUFFIX " %2,%1\n\t"
+		"and" __WORDSUFFIX " %4,%1\n\t"
+		"shl" __WORDSUFFIX " $3,%1\n\t"
+		"shr" __WORDSUFFIX " %b1,%0\n\t"
+		"jmp 2b\n"
+		".previous\n"
+		_ASM_EXTABLE(1b, 3b)
+		:"=&r" (ret),"=&c" (dummy)
+		:"m" (*(unsigned long *)addr),
+		 "i" (-sizeof(unsigned long)),
+		 "i" (sizeof(unsigned long)-1));
+	return ret;
+}
+
 #endif /* _ASM_WORD_AT_A_TIME_H */
diff --git a/fs/namei.c b/fs/namei.c
index 0062dd17eb55..c42791914f82 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -1429,7 +1429,7 @@ unsigned int full_name_hash(const unsigned char *name, unsigned int len)
 	unsigned long hash = 0;
 
 	for (;;) {
-		a = *(unsigned long *)name;
+		a = load_unaligned_zeropad(name);
 		if (len < sizeof(unsigned long))
 			break;
 		hash += a;
@@ -1459,7 +1459,7 @@ static inline unsigned long hash_name(const char *name, unsigned int *hashp)
 	do {
 		hash = (hash + a) * 9;
 		len += sizeof(unsigned long);
-		a = *(unsigned long *)(name+len);
+		a = load_unaligned_zeropad(name+len);
 		/* Do we have any NUL or '/' bytes in this word? */
 		mask = has_zero(a) | has_zero(a ^ REPEAT_BYTE('/'));
 	} while (!mask);