[PATCH v5 4/8] mm/gup: split out folio grab-and-store helper

From: Rik van Riel

Date: Mon Nov 16 2026 - 16:02:03 EST


follow_page_pte() does two things once it has resolved a present PTE
to a page: run the per-PTE safety checks (write-fault, unshare), then
grab a folio reference and hand the page back: fault it in if pinning,
mark it dirty/accessed, and store the caller's array slot.

Split the second part into gup_grab_fill_folio_pages(), which stores the
pages with gup_fill_pages(). The cache flushes stay with the caller in
__get_user_pages(), which runs gup_flush_pages() without any page
table lock held. Storing one slot under the lock is cheap; flushing is
not, and needs no lock once the folio reference is held.

No functional changes intended.

Suggested-by: David Hildenbrand <david@xxxxxxxxxx>
Signed-off-by: Rik van Riel <riel@xxxxxxxxxxx>
---
mm/gup.c | 76 +++++++++++++++++++++++++++++++++++---------------------
1 file changed, 47 insertions(+), 29 deletions(-)

diff --git a/mm/gup.c b/mm/gup.c
index c59d436be7ad..efb2ad64be80 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -921,6 +921,50 @@ static inline bool can_follow_write_pte(pte_t pte, struct page *page,
return !userfaultfd_pte_wp(vma, pte);
}

+/*
+ * Grab a folio reference for @nr pages starting at @page, mark the folio
+ * accessed (and dirty for FOLL_WRITE), and store the pages for the caller.
+ * The caller (__get_user_pages()) flushes the caches with gup_flush_pages().
+ */
+static long gup_grab_fill_folio_pages(struct folio *folio,
+ struct page *page, unsigned long nr, bool dirty,
+ unsigned int flags, struct page **pages)
+{
+ long ret;
+
+ /* try_grab_folio() does nothing unless FOLL_GET or FOLL_PIN is set. */
+ ret = try_grab_folio(folio, nr, flags);
+ if (unlikely(ret))
+ return ret;
+
+ /*
+ * We need to make the page accessible if and only if we are going
+ * to access its content (the FOLL_PIN case). Please see
+ * Documentation/core-api/pin_user_pages.rst for details.
+ */
+ if (flags & FOLL_PIN) {
+ ret = arch_make_folio_accessible(folio);
+ if (ret) {
+ gup_put_folio(folio, nr, flags);
+ return ret;
+ }
+ }
+ if (flags & FOLL_TOUCH) {
+ if ((flags & FOLL_WRITE) && !dirty && !folio_test_dirty(folio))
+ folio_mark_dirty(folio);
+ /*
+ * pte_mkyoung() would be more correct here, but atomic care
+ * is needed to avoid losing the dirty bit: it is easier to use
+ * folio_mark_accessed().
+ */
+ folio_mark_accessed(folio);
+ }
+
+ gup_fill_pages(pages, page, nr);
+
+ return 0;
+}
+
static long follow_page_pte(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmd, unsigned int flags,
struct page **pages)
@@ -976,36 +1020,10 @@ static long follow_page_pte(struct vm_area_struct *vma,
VM_WARN_ON_ONCE_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
!PageAnonExclusive(page), page);

- /* try_grab_folio() does nothing unless FOLL_GET or FOLL_PIN is set. */
- ret = try_grab_folio(folio, 1, flags);
- if (unlikely(ret))
+ ret = gup_grab_fill_folio_pages(folio, page, 1,
+ pte_dirty(pte), flags, pages);
+ if (ret)
goto out;
-
- /*
- * We need to make the page accessible if and only if we are going
- * to access its content (the FOLL_PIN case). Please see
- * Documentation/core-api/pin_user_pages.rst for details.
- */
- if (flags & FOLL_PIN) {
- ret = arch_make_folio_accessible(folio);
- if (ret) {
- gup_put_folio(folio, 1, flags);
- goto out;
- }
- }
- if (flags & FOLL_TOUCH) {
- if ((flags & FOLL_WRITE) &&
- !pte_dirty(pte) && !folio_test_dirty(folio))
- folio_mark_dirty(folio);
- /*
- * pte_mkyoung() would be more correct here, but atomic care
- * is needed to avoid losing the dirty bit: it is easier to use
- * folio_mark_accessed().
- */
- folio_mark_accessed(folio);
- }
-
- gup_fill_pages(pages, page, 1);
ret = 1;
out:
pte_unmap_unlock(ptep, ptl);
--
2.56.0