[PATCH 13/24] huge tmpfs: extend get_user_pages_fast to shmem pmd

[Hugh Dickins]

Factor out one small part of the shmem pmd handling: the arch-specific
get_user_pages_fast() has special code to cope with the peculiar
refcounting on anonymous THP tail pages (and on hugetlbfs tail pages):
which must be avoided in the straightforward shmem pmd case.

Signed-off-by: Hugh Dickins <hughd@xxxxxxxxxx>
---
 arch/mips/mm/gup.c  |   17 ++++++++++++-----
 arch/s390/mm/gup.c  |   22 +++++++++++++++++++++-
 arch/sparc/mm/gup.c |   22 +++++++++++++++++++++-
 arch/x86/mm/gup.c   |   17 ++++++++++++-----
 mm/gup.c            |   22 +++++++++++++++++++++-
 5 files changed, 87 insertions(+), 13 deletions(-)

--- thpfs.orig/arch/mips/mm/gup.c	2015-02-08 18:54:22.000000000 -0800
+++ thpfs/arch/mips/mm/gup.c	2015-02-20 19:34:26.971957306 -0800
@@ -64,7 +64,8 @@ static inline void get_head_page_multipl
 {
 	VM_BUG_ON(page != compound_head(page));
 	VM_BUG_ON(page_count(page) == 0);
-	atomic_add(nr, &page->_count);
+	if (nr)
+		atomic_add(nr, &page->_count);
 	SetPageReferenced(page);
 }
 
@@ -85,13 +86,19 @@ static int gup_huge_pmd(pmd_t pmd, unsig
 	head = pte_page(pte);
 	page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
 	do {
-		VM_BUG_ON(compound_head(page) != head);
-		pages[*nr] = page;
-		if (PageTail(page))
+		if (PageTail(page)) {
+			VM_BUG_ON(compound_head(page) != head);
 			get_huge_page_tail(page);
+			refs++;
+		} else {
+			/*
+			 * Handle head or huge tmpfs with normal refcounting.
+			 */
+			get_page(page);
+		}
+		pages[*nr] = page;
 		(*nr)++;
 		page++;
-		refs++;
 	} while (addr += PAGE_SIZE, addr != end);
 
 	get_head_page_multiple(head, refs);
--- thpfs.orig/arch/s390/mm/gup.c	2014-01-19 18:40:07.000000000 -0800
+++ thpfs/arch/s390/mm/gup.c	2015-02-20 19:34:26.971957306 -0800
@@ -61,10 +61,30 @@ static inline int gup_huge_pmd(pmd_t *pm
 		return 0;
 	VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT));
 
-	refs = 0;
 	head = pmd_page(pmd);
 	page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
+
+	if (!PageHead(head)) {
+		/*
+		 * Handle a huge tmpfs team with normal refcounting.
+		 */
+		do {
+			if (!page_cache_get_speculative(page))
+				return 0;
+			if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
+				put_page(page);
+				return 0;
+			}
+			pages[*nr] = page;
+			(*nr)++;
+			page++;
+		} while (addr += PAGE_SIZE, addr != end);
+		return 1;
+	}
+
 	tail = page;
+	refs = 0;
+
 	do {
 		VM_BUG_ON(compound_head(page) != head);
 		pages[*nr] = page;
--- thpfs.orig/arch/sparc/mm/gup.c	2014-12-07 14:21:05.000000000 -0800
+++ thpfs/arch/sparc/mm/gup.c	2015-02-20 19:34:26.975957297 -0800
@@ -79,10 +79,30 @@ static int gup_huge_pmd(pmd_t *pmdp, pmd
 	if (write && !pmd_write(pmd))
 		return 0;
 
-	refs = 0;
 	head = pmd_page(pmd);
 	page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
+
+	if (!PageHead(head)) {
+		/*
+		 * Handle a huge tmpfs team with normal refcounting.
+		 */
+		do {
+			if (!page_cache_get_speculative(page))
+				return 0;
+			if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
+				put_page(page);
+				return 0;
+			}
+			pages[*nr] = page;
+			(*nr)++;
+			page++;
+		} while (addr += PAGE_SIZE, addr != end);
+		return 1;
+	}
+
 	tail = page;
+	refs = 0;
+
 	do {
 		VM_BUG_ON(compound_head(page) != head);
 		pages[*nr] = page;
--- thpfs.orig/arch/x86/mm/gup.c	2015-02-08 18:54:22.000000000 -0800
+++ thpfs/arch/x86/mm/gup.c	2015-02-20 19:34:26.975957297 -0800
@@ -110,7 +110,8 @@ static inline void get_head_page_multipl
 {
 	VM_BUG_ON_PAGE(page != compound_head(page), page);
 	VM_BUG_ON_PAGE(page_count(page) == 0, page);
-	atomic_add(nr, &page->_count);
+	if (nr)
+		atomic_add(nr, &page->_count);
 	SetPageReferenced(page);
 }
 
@@ -135,13 +136,19 @@ static noinline int gup_huge_pmd(pmd_t p
 	head = pte_page(pte);
 	page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
 	do {
-		VM_BUG_ON_PAGE(compound_head(page) != head, page);
-		pages[*nr] = page;
-		if (PageTail(page))
+		if (PageTail(page)) {
+			VM_BUG_ON_PAGE(compound_head(page) != head, page);
 			get_huge_page_tail(page);
+			refs++;
+		} else {
+			/*
+			 * Handle head or huge tmpfs with normal refcounting.
+			 */
+			get_page(page);
+		}
+		pages[*nr] = page;
 		(*nr)++;
 		page++;
-		refs++;
 	} while (addr += PAGE_SIZE, addr != end);
 	get_head_page_multiple(head, refs);
 
--- thpfs.orig/mm/gup.c	2015-02-08 18:54:22.000000000 -0800
+++ thpfs/mm/gup.c	2015-02-20 19:34:26.975957297 -0800
@@ -795,10 +795,30 @@ static int gup_huge_pmd(pmd_t orig, pmd_
 	if (write && !pmd_write(orig))
 		return 0;
 
-	refs = 0;
 	head = pmd_page(orig);
 	page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
+
+	if (!PageHead(head)) {
+		/*
+		 * Handle a huge tmpfs team with normal refcounting.
+		 */
+		do {
+			if (!page_cache_get_speculative(page))
+				return 0;
+			if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
+				put_page(page);
+				return 0;
+			}
+			pages[*nr] = page;
+			(*nr)++;
+			page++;
+		} while (addr += PAGE_SIZE, addr != end);
+		return 1;
+	}
+
 	tail = page;
+	refs = 0;
+
 	do {
 		VM_BUG_ON_PAGE(compound_head(page) != head, page);
 		pages[*nr] = page;
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/