[PATCH 5/5] iov_iter: extract bvecs of user-backed iters from folio ranges
From: Swarna Prabhu
Date: Wed Nov 18 2026 - 16:37:36 EST
From: Swarna Prabhu <s.prabhu@xxxxxxxxxxx>
iov_iter_extract_bvecs() pins a page array placed in the bvec array and
then walks it with get_contig_folio_len() to merge pages that turn out to
belong to the same folio back into one bvec.
For user-backed iterators, have GUP fill folio ranges straight into the
free part of the bvec array instead. A folio range has the same size as
a bvec and becomes exactly one bvec, so each range is converted in its own
slot and trimmed to the requested byte range on the way. This removes the
page array and the merging pass. Ranges that are not converted are still
folio ranges and are unpinned with their exact page count.
Since one range covers many pages, the number of pages pinned per call is
no longer limited to one page per bvec entry; it is bounded by the bvec
array, plus a cap on how many pages a single GUP-fast walk looks at.
Memory that is not backed by folios, such as pages a driver inserted
into a VM_MIXEDMAP vma, makes pin_user_folio_ranges_fast() fail with
-EOPNOTSUPP when no pages were pinned; such iterators fall back to the
page array path.
Each extraction requests at most 1024 pages, bounding the amount of work
performed by a single GUP call.
Block direct I/O picks this up through bio_iov_iter_get_pages() without
any change to the block layer.
Signed-off-by: Swarna Prabhu <s.prabhu@xxxxxxxxxxx>
---
lib/iov_iter.c | 100 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 100 insertions(+)
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index fa4fe2e92383..457e226c0251 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -1919,6 +1919,95 @@ static unsigned int get_contig_folio_len(struct page **pages,
#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))
+/*
+ * Upper limit on the number of pages a single GUP-fast walk is asked to look
+ * at to cap spending too much time in the GUP-fast path.
+ */
+#define EXTRACT_MAX_PAGES 1024
+
+/*
+ * Extract bvecs from a user-backed iterator. GUP fills folio ranges straight
+ * into the free part of the bvec array, one range per run of consecutive pages
+ * of a folio. A folio range has the same size as a bvec and turns into
+ * exactly one bvec, so each range is converted in its own slot.
+ *
+ * Returns -EOPNOTSUPP, with @iter left untouched, if the memory at the start of
+ * @iter is not backed by folios, so the caller can extract it as pages instead.
+ */
+static ssize_t iov_iter_extract_user_bvecs(struct iov_iter *iter,
+ struct bio_vec *bv, size_t max_size, unsigned short *nr_vecs,
+ unsigned short max_vecs, iov_iter_extraction_t extraction_flags)
+{
+ unsigned short entries_left = max_vecs - *nr_vecs;
+ unsigned int gup_flags = 0, nr_frs = 0, i;
+ unsigned long addr, nr_pages;
+ struct folio_range *frs;
+ size_t offset, left, len;
+ long res;
+
+ max_size = min_t(size_t, min_t(size_t, max_size, iter->count),
+ MAX_RW_COUNT);
+ if (unlikely(!max_size))
+ return -EFAULT;
+ if (WARN_ON_ONCE(!entries_left))
+ return -ENOMEM;
+
+ if (iter->data_source == ITER_DEST)
+ gup_flags |= FOLL_WRITE;
+ if (extraction_flags & ITER_ALLOW_P2PDMA)
+ gup_flags |= FOLL_PCI_P2PDMA;
+ if (iter->nofault)
+ gup_flags |= FOLL_NOFAULT;
+
+ addr = first_iovec_segment(iter, &max_size);
+ offset = addr % PAGE_SIZE;
+ addr &= PAGE_MASK;
+ nr_pages = min_t(unsigned long,
+ DIV_ROUND_UP(max_size + offset, PAGE_SIZE),
+ EXTRACT_MAX_PAGES);
+
+ /*
+ * Range i is converted into slot *nr_vecs + i, which is the memory it
+ * lives in, so converting cannot overwrite a range that has not been
+ * converted yet.
+ */
+ BUILD_BUG_ON(sizeof(struct folio_range) != sizeof(struct bio_vec));
+ frs = (struct folio_range *)(bv + *nr_vecs);
+
+ res = pin_user_folio_ranges_fast(addr, nr_pages, gup_flags, frs,
+ entries_left, &nr_frs);
+ if (unlikely(res <= 0))
+ return res ? res : -EFAULT;
+
+ /* We may have pinned fewer pages than asked for. */
+ max_size = min_t(size_t, max_size, res * PAGE_SIZE - offset);
+ iov_iter_advance(iter, max_size);
+
+ for (i = 0, left = max_size; left && i < nr_frs; i++) {
+ /* Copy it out first: bv[*nr_vecs] is the same memory as frs[i]. */
+ struct folio_range fr = frs[i];
+ struct page *page = folio_range_page(&fr);
+
+ if (*nr_vecs > 0 &&
+ !zone_device_pages_have_same_pgmap(bv[*nr_vecs - 1].bv_page,
+ page))
+ break;
+
+ len = min(folio_range_size(&fr) - offset, left);
+ bvec_set_folio(&bv[(*nr_vecs)++], fr.folio, len,
+ folio_range_offset(&fr) + offset);
+ left -= len;
+ offset = 0;
+ }
+
+ /* Not overwritten, so still folio ranges; no-op once i == nr_frs. */
+ if (unlikely(i < nr_frs))
+ unpin_user_folio_ranges(&frs[i], nr_frs - i);
+
+ iov_iter_revert(iter, left);
+ return max_size - left;
+}
+
/**
* iov_iter_extract_bvecs - Extract bvecs from an iterator
* @iter: the iterator to extract from
@@ -1966,6 +2055,17 @@ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv,
return -EINVAL;
}
+ if (likely(user_backed_iter(iter))) {
+ size = iov_iter_extract_user_bvecs(iter, bv, max_size, nr_vecs,
+ max_vecs, extraction_flags);
+ /*
+ * Memory that is not backed by folios cannot be pinned as folio
+ * ranges; fall back to pinning it as an array of pages.
+ */
+ if (size != -EOPNOTSUPP)
+ return size;
+ }
+
/*
* Move page array up in the allocated memory for the bio vecs as far as
* possible so that we can start filling biovecs from the beginning
--
2.53.0