[RFC PATCH v3 6/8] mm/gup: return a huge page's full count from follow_page_mask()
From: Rik van Riel
Date: Mon Aug 10 2026 - 23:07:06 EST
follow_huge_pud()/follow_huge_pmd() already know the huge page's full
size but report it via a separate *page_mask output; __get_user_pages()
does a second try_grab_folio() call and subpage loop for everything
past the first page.
Have the huge paths report their count as the return value instead,
clamped to the huge page's size and @end. The merged grab returns
whatever error try_grab_folio() gives, instead of forcing -EFAULT on the
second call's failure.
*page_mask and __get_user_pages()'s second-grab/subpage loop are now
dead; remove them. The old silent page_increm clamp becomes a
VM_WARN_ON_ONCE, since @end already bounds the count and refs/pages[]
are already committed by the time the caller sees it -- truncating here
would leak references, not just waste a comparison.
follow_page_pte() is unaffected, still returning at most 1 page.
The -EEXIST path needs an explicit nr = 1 when pages == NULL: it used to
get that from page_mask staying 0 for a PFN-special PTE, but nr holds
-EEXIST there, which would grow nr_pages instead of shrinking it.
mm/gup_test.c (PIN_LONGTERM_BENCHMARK) shows no measurable change; the
lock-hold-time improvement is by inspection, not measurement.
Suggested-by: David Hildenbrand <david@xxxxxxxxxx>
Assisted-by: Claude:claude-opus-4-8
Signed-off-by: Rik van Riel <riel@xxxxxxxxxxx>
---
mm/gup.c | 150 +++++++++++++++++++++----------------------------------
1 file changed, 58 insertions(+), 92 deletions(-)
diff --git a/mm/gup.c b/mm/gup.c
index 4036d3dc27df..ea2bb379183e 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -664,14 +664,14 @@ static inline bool can_follow_write_pud(pud_t pud, struct page *page,
}
static long follow_huge_pud(struct vm_area_struct *vma,
- unsigned long addr, pud_t *pudp,
- unsigned int flags, unsigned long *page_mask,
- struct page **pages)
+ unsigned long addr, unsigned long end, pud_t *pudp,
+ unsigned int flags, struct page **pages)
{
struct mm_struct *mm = vma->vm_mm;
struct page *page;
pud_t pud = *pudp;
unsigned long pfn = pud_pfn(pud);
+ unsigned long off, nr;
int ret;
assert_spin_locked(pud_lockptr(mm, pudp));
@@ -683,22 +683,23 @@ static long follow_huge_pud(struct vm_area_struct *vma,
!can_follow_write_pud(pud, pfn_to_page(pfn), vma, flags))
return 0;
- pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
+ off = PFN_DOWN(addr & ~PUD_MASK);
+ pfn += off;
page = pfn_to_page(pfn);
if (!pud_write(pud) && gup_must_unshare(vma, flags, page))
return -EMLINK;
- ret = try_grab_folio(page_folio(page), 1, flags);
+ nr = min(HPAGE_PUD_NR - off, PFN_DOWN(end - addr));
+
+ ret = try_grab_folio(page_folio(page), nr, flags);
if (ret)
return ret;
- *page_mask = HPAGE_PUD_NR - 1;
-
if (pages)
pages[0] = page;
- return 1;
+ return nr;
}
/* FOLL_FORCE can write to even unwritable PMDs in COW mappings. */
@@ -720,13 +721,13 @@ static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page,
}
static long follow_huge_pmd(struct vm_area_struct *vma,
- unsigned long addr, pmd_t *pmd,
- unsigned int flags, unsigned long *page_mask,
- struct page **pages)
+ unsigned long addr, unsigned long end, pmd_t *pmd,
+ unsigned int flags, struct page **pages)
{
struct mm_struct *mm = vma->vm_mm;
pmd_t pmdval = *pmd;
struct page *page;
+ unsigned long off, nr;
int ret;
assert_spin_locked(pmd_lockptr(mm, pmd));
@@ -749,7 +750,10 @@ static long follow_huge_pmd(struct vm_area_struct *vma,
VM_WARN_ON_ONCE_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
!PageAnonExclusive(page), page);
- ret = try_grab_folio(page_folio(page), 1, flags);
+ off = PFN_DOWN(addr & ~HPAGE_PMD_MASK);
+ nr = min(HPAGE_PMD_NR - off, PFN_DOWN(end - addr));
+
+ ret = try_grab_folio(page_folio(page), nr, flags);
if (ret)
return ret;
@@ -758,28 +762,25 @@ static long follow_huge_pmd(struct vm_area_struct *vma,
touch_pmd(vma, addr, pmd, flags & FOLL_WRITE);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
- page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
- *page_mask = HPAGE_PMD_NR - 1;
+ page += off;
if (pages)
pages[0] = page;
- return 1;
+ return nr;
}
#else /* CONFIG_PGTABLE_HAS_HUGE_LEAVES */
static long follow_huge_pud(struct vm_area_struct *vma,
- unsigned long addr, pud_t *pudp,
- unsigned int flags, unsigned long *page_mask,
- struct page **pages)
+ unsigned long addr, unsigned long end, pud_t *pudp,
+ unsigned int flags, struct page **pages)
{
return 0;
}
static long follow_huge_pmd(struct vm_area_struct *vma,
- unsigned long addr, pmd_t *pmd,
- unsigned int flags, unsigned long *page_mask,
- struct page **pages)
+ unsigned long addr, unsigned long end, pmd_t *pmd,
+ unsigned int flags, struct page **pages)
{
return 0;
}
@@ -953,9 +954,8 @@ static long follow_page_pte(struct vm_area_struct *vma,
}
static long follow_pmd_mask(struct vm_area_struct *vma,
- unsigned long address, pud_t *pudp,
- unsigned int flags, unsigned long *page_mask,
- struct page **pages)
+ unsigned long address, unsigned long end, pud_t *pudp,
+ unsigned int flags, struct page **pages)
{
pmd_t *pmd, pmdval;
spinlock_t *ptl;
@@ -991,7 +991,7 @@ static long follow_pmd_mask(struct vm_area_struct *vma,
return pte_alloc(mm, pmd) ? -ENOMEM :
follow_page_pte(vma, address, pmd, flags, pages);
}
- ret = follow_huge_pmd(vma, address, pmd, flags, page_mask, pages);
+ ret = follow_huge_pmd(vma, address, end, pmd, flags, pages);
spin_unlock(ptl);
/*
@@ -1005,9 +1005,8 @@ static long follow_pmd_mask(struct vm_area_struct *vma,
}
static long follow_pud_mask(struct vm_area_struct *vma,
- unsigned long address, p4d_t *p4dp,
- unsigned int flags, unsigned long *page_mask,
- struct page **pages)
+ unsigned long address, unsigned long end, p4d_t *p4dp,
+ unsigned int flags, struct page **pages)
{
pud_t *pudp, pud;
spinlock_t *ptl;
@@ -1020,11 +1019,13 @@ static long follow_pud_mask(struct vm_area_struct *vma,
return no_page_table(vma, flags, address);
if (pud_leaf(pud)) {
ptl = pud_lock(mm, pudp);
- ret = follow_huge_pud(vma, address, pudp, flags, page_mask, pages);
+ ret = follow_huge_pud(vma, address, end, pudp, flags, pages);
spin_unlock(ptl);
/*
* The ref is already held, so the page cannot go away: fill
- * the array and flush caches without the lock.
+ * the array and flush caches without the lock. A 1 GB folio
+ * can be up to HPAGE_PUD_NR pages, too long to flush under a
+ * spinlock.
*/
if (ret > 0 && pages)
gup_fill_pages(vma, address, pages[0], ret, pages);
@@ -1035,13 +1036,12 @@ static long follow_pud_mask(struct vm_area_struct *vma,
if (unlikely(pud_bad(pud)))
return no_page_table(vma, flags, address);
- return follow_pmd_mask(vma, address, pudp, flags, page_mask, pages);
+ return follow_pmd_mask(vma, address, end, pudp, flags, pages);
}
static long follow_p4d_mask(struct vm_area_struct *vma,
- unsigned long address, pgd_t *pgdp,
- unsigned int flags, unsigned long *page_mask,
- struct page **pages)
+ unsigned long address, unsigned long end, pgd_t *pgdp,
+ unsigned int flags, struct page **pages)
{
p4d_t *p4dp, p4d;
@@ -1052,18 +1052,18 @@ static long follow_p4d_mask(struct vm_area_struct *vma,
if (!p4d_present(p4d) || p4d_bad(p4d))
return no_page_table(vma, flags, address);
- return follow_pud_mask(vma, address, p4dp, flags, page_mask, pages);
+ return follow_pud_mask(vma, address, end, p4dp, flags, pages);
}
/**
- * follow_page_mask - look up a page descriptor from a user-virtual address
+ * follow_page_mask - look up pages at a user-virtual address
* @vma: vm_area_struct mapping @address
* @address: virtual address to look up
+ * @end: virtual address at which to stop batching contiguous pages
* @flags: flags modifying lookup behaviour
- * @page_mask: a pointer to output page_mask
- * @pages: array to receive the page found, refcounted per @flags, or NULL
- * to walk the page tables (e.g. to fault pages in) without
- * collecting or refcounting them
+ * @pages: array to receive the pages, refcounted per @flags, or NULL to
+ * walk the page tables (e.g. to fault pages in) without collecting
+ * or refcounting them
*
* @flags can have FOLL_ flags set, defined in <linux/mm.h>
*
@@ -1072,15 +1072,15 @@ static long follow_p4d_mask(struct vm_area_struct *vma,
* trigger a fault with FAULT_FLAG_UNSHARE set. Note that unsharing is only
* relevant with FOLL_PIN and !FOLL_WRITE.
*
- * On output, @page_mask is set according to the size of the page.
- *
- * Return: 1 with @pages[0] filled in if a page was found, 0 if no mapping
- * exists at @address, or a negative errno for a mapping to something not
- * represented by a page descriptor (see also vm_normal_page()).
+ * Return: the number of contiguous pages starting at @address that were
+ * placed into @pages (if non-NULL), which may be fewer than the pages
+ * requested via @end; 0 if no mapping exists at @address; or a negative
+ * errno for a mapping to something not represented by a page descriptor
+ * (see also vm_normal_page()).
*/
static long follow_page_mask(struct vm_area_struct *vma,
- unsigned long address, unsigned int flags,
- unsigned long *page_mask, struct page **pages)
+ unsigned long address, unsigned long end,
+ unsigned int flags, struct page **pages)
{
pgd_t *pgd;
struct mm_struct *mm = vma->vm_mm;
@@ -1088,13 +1088,12 @@ static long follow_page_mask(struct vm_area_struct *vma,
vma_pgtable_walk_begin(vma);
- *page_mask = 0;
pgd = pgd_offset(mm, address);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
ret = no_page_table(vma, flags, address);
else
- ret = follow_p4d_mask(vma, address, pgd, flags, page_mask, pages);
+ ret = follow_p4d_mask(vma, address, end, pgd, flags, pages);
vma_pgtable_walk_end(vma);
@@ -1432,7 +1431,6 @@ static long __get_user_pages(struct mm_struct *mm,
{
long ret = 0, i = 0;
struct vm_area_struct *vma = NULL;
- unsigned long page_mask = 0;
if (!nr_pages)
return 0;
@@ -1447,7 +1445,6 @@ static long __get_user_pages(struct mm_struct *mm,
do {
struct page *page;
- unsigned int page_increm;
long nr;
/* first iteration or cross vma bound */
@@ -1502,8 +1499,8 @@ static long __get_user_pages(struct mm_struct *mm,
}
cond_resched();
- nr = follow_page_mask(vma, start, gup_flags, &page_mask,
- pages ? &pages[i] : NULL);
+ nr = follow_page_mask(vma, start, start + nr_pages * PAGE_SIZE,
+ gup_flags, pages ? &pages[i] : NULL);
if (!nr || nr == -EMLINK) {
ret = faultin_page(vma, start, gup_flags,
nr == -EMLINK, locked);
@@ -1525,56 +1522,25 @@ static long __get_user_pages(struct mm_struct *mm,
* Proper page table entry exists, but no corresponding
* struct page. If the caller expects **pages to be
* filled in, bail out now, because that can't be done
- * for this page.
+ * for this page. Otherwise advance by the one page
+ * follow_page_mask() looked at.
*/
if (pages) {
ret = nr;
goto out;
}
+ nr = 1;
} else if (nr < 0) {
ret = nr;
goto out;
}
- page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask);
- if (page_increm > nr_pages)
- page_increm = nr_pages;
-
- /*
- * This must be a large folio (and doesn't need to
- * be the whole folio; it can be part of it), do
- * the refcount work for all the subpages too.
- *
- * NOTE: here the page may not be the head page
- * e.g. when start addr is not thp-size aligned.
- * try_grab_folio() should have taken care of tail
- * pages.
- */
- if (pages && page_increm > 1) {
- struct folio *folio = page_folio(pages[i]);
-
- /*
- * Since we already hold refcount on the
- * large folio, this should never fail.
- */
- if (try_grab_folio(folio, page_increm - 1,
- gup_flags)) {
- /*
- * Release the 1st page ref if the
- * folio is problematic, fail hard.
- */
- gup_put_folio(folio, 1, gup_flags);
- ret = -EFAULT;
- goto out;
- }
-
- gup_fill_pages(vma, start + PAGE_SIZE, pages[i] + 1,
- page_increm - 1, pages + i + 1);
- }
+ /* Check that we didn't pin more pages than the caller will free. */
+ VM_WARN_ON_ONCE(nr > nr_pages);
- i += page_increm;
- start += page_increm * PAGE_SIZE;
- nr_pages -= page_increm;
+ i += nr;
+ start += nr * PAGE_SIZE;
+ nr_pages -= nr;
} while (nr_pages);
out:
return i ? i : ret;
--
2.55.0