[PATCH RFC v4 10/12] mm/gup: pass an end address to follow_page_mask() and return a page count
From: Rik van Riel
Date: Fri Jul 24 2026 - 18:32:32 EST
follow_page_mask() reports how many pages the returned page covers with a
page_mask: a bitmask of the enclosing naturally-aligned huge page. Callers
turn that into a stride, which assumes the run is a power-of-two block
aligned to its own size.
That form cannot describe an arbitrary contiguous run, which a later patch
needs to batch PTE-mapped large folios.
Replace the page_mask output with nr_pages, the number of contiguous pages
the returned page covers starting at @address, and add an @end argument
bounding how far the walk looks, so a small request does not scan an entire
large folio.
The huge PMD and PUD paths report the same stride as before, now as a count
clamped to @end. __get_user_pages() uses the count directly.
get_user_page_vma() passes @address + PAGE_SIZE and still returns one page.
No functional change intended.
Assisted-by: Claude:claude-opus-4.8
Signed-off-by: Rik van Riel <riel@xxxxxxxxxxx>
---
mm/gup.c | 102 ++++++++++++++++++++++++++++++-------------------------
1 file changed, 56 insertions(+), 46 deletions(-)
diff --git a/mm/gup.c b/mm/gup.c
index 7f4107f7ff10..96b00ddf7e04 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -647,8 +647,9 @@ static inline bool can_follow_write_pud(pud_t pud, struct page *page,
}
static struct page *follow_huge_pud(struct vm_area_struct *vma,
- unsigned long addr, pud_t *pudp,
- int flags, unsigned long *page_mask)
+ unsigned long addr, unsigned long end,
+ pud_t *pudp, int flags,
+ unsigned long *nr_pages)
{
struct mm_struct *mm = vma->vm_mm;
struct page *page;
@@ -672,10 +673,13 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma,
return ERR_PTR(-EMLINK);
ret = try_grab_folio(page_folio(page), 1, flags);
- if (ret)
+ if (ret) {
page = ERR_PTR(ret);
- else
- *page_mask = HPAGE_PUD_NR - 1;
+ } else {
+ unsigned long off = (addr & ~PUD_MASK) >> PAGE_SHIFT;
+
+ *nr_pages = min(HPAGE_PUD_NR - off, (end - addr) >> PAGE_SHIFT);
+ }
return page;
}
@@ -699,9 +703,9 @@ static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page,
}
static struct page *follow_huge_pmd(struct vm_area_struct *vma,
- unsigned long addr, pmd_t *pmd,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long addr, unsigned long end,
+ pmd_t *pmd, unsigned int flags,
+ unsigned long *nr_pages)
{
struct mm_struct *mm = vma->vm_mm;
pmd_t pmdval = *pmd;
@@ -738,23 +742,25 @@ static struct page *follow_huge_pmd(struct vm_area_struct *vma,
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
- *page_mask = HPAGE_PMD_NR - 1;
+ *nr_pages = min(HPAGE_PMD_NR - ((addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT),
+ (end - addr) >> PAGE_SHIFT);
return page;
}
#else /* CONFIG_PGTABLE_HAS_HUGE_LEAVES */
static struct page *follow_huge_pud(struct vm_area_struct *vma,
- unsigned long addr, pud_t *pudp,
- int flags, unsigned long *page_mask)
+ unsigned long addr, unsigned long end,
+ pud_t *pudp, int flags,
+ unsigned long *nr_pages)
{
return NULL;
}
static struct page *follow_huge_pmd(struct vm_area_struct *vma,
- unsigned long addr, pmd_t *pmd,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long addr, unsigned long end,
+ pmd_t *pmd, unsigned int flags,
+ unsigned long *nr_pages)
{
return NULL;
}
@@ -800,7 +806,8 @@ static inline bool can_follow_write_pte(pte_t pte, struct page *page,
}
static struct page *follow_page_pte(struct vm_area_struct *vma,
- unsigned long address, pmd_t *pmd, unsigned int flags)
+ unsigned long address, unsigned long end, pmd_t *pmd,
+ unsigned int flags, unsigned long *nr_pages)
{
struct mm_struct *mm = vma->vm_mm;
struct folio *folio;
@@ -885,6 +892,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
*/
folio_mark_accessed(folio);
}
+
out:
pte_unmap_unlock(ptep, ptl);
return page;
@@ -896,9 +904,9 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
}
static struct page *follow_pmd_mask(struct vm_area_struct *vma,
- unsigned long address, pud_t *pudp,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long address, unsigned long end,
+ pud_t *pudp, unsigned int flags,
+ unsigned long *nr_pages)
{
pmd_t *pmd, pmdval;
spinlock_t *ptl;
@@ -912,7 +920,7 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
if (!pmd_present(pmdval))
return no_page_table(vma, flags, address);
if (likely(!pmd_leaf(pmdval)))
- return follow_page_pte(vma, address, pmd, flags);
+ return follow_page_pte(vma, address, end, pmd, flags, nr_pages);
if (pmd_protnone(pmdval) && !gup_can_follow_protnone(vma, flags))
return no_page_table(vma, flags, address);
@@ -925,24 +933,24 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
}
if (unlikely(!pmd_leaf(pmdval))) {
spin_unlock(ptl);
- return follow_page_pte(vma, address, pmd, flags);
+ return follow_page_pte(vma, address, end, pmd, flags, nr_pages);
}
if (pmd_trans_huge(pmdval) && (flags & FOLL_SPLIT_PMD)) {
spin_unlock(ptl);
split_huge_pmd(vma, pmd, address);
/* If pmd was left empty, stuff a page table in there quickly */
return pte_alloc(mm, pmd) ? ERR_PTR(-ENOMEM) :
- follow_page_pte(vma, address, pmd, flags);
+ follow_page_pte(vma, address, end, pmd, flags, nr_pages);
}
- page = follow_huge_pmd(vma, address, pmd, flags, page_mask);
+ page = follow_huge_pmd(vma, address, end, pmd, flags, nr_pages);
spin_unlock(ptl);
return page;
}
static struct page *follow_pud_mask(struct vm_area_struct *vma,
- unsigned long address, p4d_t *p4dp,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long address, unsigned long end,
+ p4d_t *p4dp, unsigned int flags,
+ unsigned long *nr_pages)
{
pud_t *pudp, pud;
spinlock_t *ptl;
@@ -955,7 +963,7 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
return no_page_table(vma, flags, address);
if (pud_leaf(pud)) {
ptl = pud_lock(mm, pudp);
- page = follow_huge_pud(vma, address, pudp, flags, page_mask);
+ page = follow_huge_pud(vma, address, end, pudp, flags, nr_pages);
spin_unlock(ptl);
if (page)
return page;
@@ -964,13 +972,13 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
if (unlikely(pud_bad(pud)))
return no_page_table(vma, flags, address);
- return follow_pmd_mask(vma, address, pudp, flags, page_mask);
+ return follow_pmd_mask(vma, address, end, pudp, flags, nr_pages);
}
static struct page *follow_p4d_mask(struct vm_area_struct *vma,
- unsigned long address, pgd_t *pgdp,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long address, unsigned long end,
+ pgd_t *pgdp, unsigned int flags,
+ unsigned long *nr_pages)
{
p4d_t *p4dp, p4d;
@@ -981,15 +989,16 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
if (!p4d_present(p4d) || p4d_bad(p4d))
return no_page_table(vma, flags, address);
- return follow_pud_mask(vma, address, p4dp, flags, page_mask);
+ return follow_pud_mask(vma, address, end, p4dp, flags, nr_pages);
}
/**
* follow_page_mask - look up a page descriptor from a user-virtual address
* @vma: vm_area_struct mapping @address
* @address: virtual address to look up
+ * @end: virtual address at which to stop batching contiguous pages
* @flags: flags modifying lookup behaviour
- * @page_mask: a pointer to output page_mask
+ * @nr_pages: output; number of contiguous pages the caller can read
*
* @flags can have FOLL_ flags set, defined in <linux/mm.h>
*
@@ -998,15 +1007,17 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
* trigger a fault with FAULT_FLAG_UNSHARE set. Note that unsharing is only
* relevant with FOLL_PIN and !FOLL_WRITE.
*
- * On output, @page_mask is set according to the size of the page.
+ * On output, @nr_pages holds how many contiguous pages the folio that includes
+ * the returned page has mapped into this process, so the caller can advance
+ * over a large folio in one step.
*
* Return: the mapped (struct page *), %NULL if no mapping exists, or
* an error pointer if there is a mapping to something not represented
* by a page descriptor (see also vm_normal_page()).
*/
static struct page *follow_page_mask(struct vm_area_struct *vma,
- unsigned long address, unsigned int flags,
- unsigned long *page_mask)
+ unsigned long address, unsigned long end,
+ unsigned int flags, unsigned long *nr_pages)
{
pgd_t *pgd;
struct mm_struct *mm = vma->vm_mm;
@@ -1014,13 +1025,13 @@ static struct page *follow_page_mask(struct vm_area_struct *vma,
vma_pgtable_walk_begin(vma);
- *page_mask = 0;
+ *nr_pages = 1;
pgd = pgd_offset(mm, address);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
page = no_page_table(vma, flags, address);
else
- page = follow_p4d_mask(vma, address, pgd, flags, page_mask);
+ page = follow_p4d_mask(vma, address, end, pgd, flags, nr_pages);
vma_pgtable_walk_end(vma);
@@ -1198,7 +1209,7 @@ struct page *get_user_page_vma(struct vm_area_struct *vma, unsigned long addr,
unsigned int gup_flags)
{
bool vma_locked = gup_flags & FOLL_VMA_LOCK;
- unsigned long page_mask;
+ unsigned long nr_pages;
struct page *page;
int locked = 1;
bool pfnmap;
@@ -1231,10 +1242,10 @@ struct page *get_user_page_vma(struct vm_area_struct *vma, unsigned long addr,
}
cond_resched();
- /* follow_page_mask() requires @page_mask; it is unused here. */
- page = follow_page_mask(vma, addr,
+ /* This helper hands back a single page; cap the batch at one. */
+ page = follow_page_mask(vma, addr, addr + PAGE_SIZE,
gup_flags | FOLL_TOUCH | FOLL_GET,
- &page_mask);
+ &nr_pages);
if (!IS_ERR_OR_NULL(page)) {
/* Match __get_user_pages(): flush for VIVT/aliasing caches. */
flush_anon_page(vma, page, addr);
@@ -1529,7 +1540,6 @@ static long __get_user_pages(struct mm_struct *mm,
{
long ret = 0, i = 0;
struct vm_area_struct *vma = NULL;
- unsigned long page_mask = 0;
if (!nr_pages)
return 0;
@@ -1544,7 +1554,7 @@ static long __get_user_pages(struct mm_struct *mm,
do {
struct page *page;
- unsigned int page_increm;
+ unsigned long page_increm;
/* first iteration or cross vma bound */
if (!vma || start >= vma->vm_end) {
@@ -1571,7 +1581,7 @@ static long __get_user_pages(struct mm_struct *mm,
pages ? &page : NULL);
if (ret)
goto out;
- page_mask = 0;
+ page_increm = 1;
goto next_page;
}
@@ -1594,7 +1604,8 @@ static long __get_user_pages(struct mm_struct *mm,
}
cond_resched();
- page = follow_page_mask(vma, start, gup_flags, &page_mask);
+ page = follow_page_mask(vma, start, start + nr_pages * PAGE_SIZE,
+ gup_flags, &page_increm);
if (!page || PTR_ERR(page) == -EMLINK) {
ret = faultin_page(vma, start, gup_flags,
PTR_ERR(page) == -EMLINK, locked);
@@ -1627,7 +1638,6 @@ static long __get_user_pages(struct mm_struct *mm,
goto out;
}
next_page:
- page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask);
if (page_increm > nr_pages)
page_increm = nr_pages;
--
2.53.0-Meta