[PATCH v2 5/8] mm/migrate: add a dedicated movable_ops migration pass

From: Shivank Garg

Date: Thu Nov 19 2026 - 02:13:06 EST


Pages with movable_ops transfer ownership through a driver callback and do
not need the unmap, TLB flush, copy, or LRU handling used for LRU folios.
These pages are expected to stop being represented as folios. Separating
their migration from the LRU folio path prepares for that conversion.

Move movable_ops pages to their own list in migrate_pages(), next to
hugetlb folios, and migrate them in migrate_movable_ops_pages() before the
LRU folios. The new pass handles allocation, locking, retries and cleanup
itself; pages that are not migrated are returned through the source list
as before. Remove the now-unreachable movable_ops branches from the LRU
path.

This changes three details of movable_ops migration:
- Pages are processed before LRU folios rather than in source-list order.
- Synchronous callers use the requested mode from the first attempt
instead of the LRU asynchronous pre-pass.
- A callback returning -EAGAIN ends the current attempt: both page locks
are dropped and the destination is released, so a later retry
allocates a fresh one.

The MIGRATE_SYNC_LIGHT !folio_test_uptodate() check is not carried over:
PG_movable_ops shares its bit with PG_uptodate, so it never fired for
these pages.

Suggested-by: Zi Yan <ziy@xxxxxxxxxx>
Suggested-by: David Hildenbrand (Arm) <david@xxxxxxxxxx>
Signed-off-by: Shivank Garg <shivankg@xxxxxxx>
---
mm/migrate.c | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------
1 file changed, 135 insertions(+), 17 deletions(-)

diff --git a/mm/migrate.c b/mm/migrate.c
index 3dfdf46681e3..bfd38e6339cf 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1233,7 +1233,7 @@ static void migrate_folio_undo_dst(struct folio *dst, bool locked,
static void migrate_folio_done(struct folio *src,
enum migrate_reason reason)
{
- if (likely(!page_has_movable_ops(&src->page)) && reason != MR_DEMOTION)
+ if (reason != MR_DEMOTION)
mod_node_page_state(folio_pgdat(src), NR_ISOLATED_ANON +
folio_is_file_lru(src), -folio_nr_pages(src));

@@ -1342,11 +1342,6 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
goto out;
dst_locked = true;

- if (unlikely(page_has_movable_ops(&src->page))) {
- __migrate_folio_record(dst, old_folio_state, anon_vma);
- return 0;
- }
-
/*
* Corner case handling:
* 1. When a new swap-cache page is read into, it is added to the LRU
@@ -1409,13 +1404,6 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
prev = dst->lru.prev;
list_del(&dst->lru);

- if (unlikely(page_has_movable_ops(&src->page))) {
- rc = migrate_movable_ops_page(&dst->page, &src->page, mode);
- if (rc)
- goto out;
- goto out_unlock_both;
- }
-
if (folio_order(src) > 1 &&
!data_race(list_empty(&src->_deferred_list))) {
src_deferred_split = true;
@@ -1451,7 +1439,6 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
if (old_folio_state & FOLIO_WAS_MAPPED)
remove_migration_ptes(src, dst, 0);

-out_unlock_both:
folio_unlock(dst);
folio_set_owner_migrate_reason(dst, reason);
/*
@@ -1844,6 +1831,124 @@ static void migrate_folios_undo(struct list_head *src_folios,
}
}

+/*
+ * Migrate one isolated movable_ops page. Keep @src on its list for retry and
+ * move it to @ret_folios on permanent failure.
+ *
+ * Return 0 on success or a negative error.
+ */
+static int move_movable_ops_page(struct folio *src,
+ new_folio_t get_new_folio, free_folio_t put_new_folio,
+ unsigned long private, enum migrate_mode mode,
+ enum migrate_reason reason, struct list_head *ret_folios)
+{
+ struct folio *dst;
+ int rc = -EAGAIN;
+
+ dst = get_new_folio(src, private);
+ if (!dst)
+ return -ENOMEM;
+
+ if (!folio_trylock(src)) {
+ if (mode == MIGRATE_ASYNC)
+ goto out_put_dst;
+ /*
+ * As in migrate_folio_unmap(), direct compaction must not
+ * sleep on the page lock.
+ */
+ if (current->flags & PF_MEMALLOC)
+ goto out_put_dst;
+ folio_lock(src);
+ }
+
+ if (unlikely(!folio_trylock(dst)))
+ goto out_unlock_src;
+
+ rc = migrate_movable_ops_page(&dst->page, &src->page, mode);
+ folio_unlock(dst);
+ if (rc)
+ goto out_unlock_src;
+
+ folio_set_owner_migrate_reason(dst, reason);
+ /* Drop migration's reference after transferring ownership to dst. */
+ folio_put(dst);
+
+ list_del(&src->lru);
+ folio_unlock(src);
+
+ if (reason != MR_MEMORY_FAILURE)
+ folio_put(src);
+
+ return 0;
+
+out_unlock_src:
+ folio_unlock(src);
+out_put_dst:
+ if (put_new_folio)
+ put_new_folio(dst, private);
+ else
+ folio_put(dst);
+
+ if (rc != -EAGAIN)
+ list_move_tail(&src->lru, ret_folios);
+
+ return rc;
+}
+
+/*
+ * Migrate the movable_ops pages on @from, trying each page up to
+ * NR_MAX_MIGRATE_PAGES_RETRY times. Permanent failures are moved to
+ * @ret_folios; any other page that was not migrated stays on @from.
+ * movable_ops pages are order-0, so each failure is one failed page.
+ *
+ * Return the number of failed pages, or a negative errno.
+ */
+static int migrate_movable_ops_pages(struct list_head *from,
+ new_folio_t get_new_folio, free_folio_t put_new_folio,
+ unsigned long private, enum migrate_mode mode,
+ enum migrate_reason reason, struct migrate_pages_stats *stats,
+ struct list_head *ret_folios)
+{
+ int retry = 1;
+ int nr_failed = 0;
+ int pass;
+ struct folio *folio, *folio2;
+ int rc;
+
+ for (pass = 0; pass < NR_MAX_MIGRATE_PAGES_RETRY && retry; pass++) {
+ retry = 0;
+
+ list_for_each_entry_safe(folio, folio2, from, lru) {
+ cond_resched();
+
+ rc = move_movable_ops_page(folio, get_new_folio,
+ put_new_folio, private,
+ mode, reason, ret_folios);
+ switch (rc) {
+ case -ENOMEM:
+ /* Count this page and those awaiting retry. */
+ stats->nr_failed_pages += 1 + retry;
+ return -ENOMEM;
+ case -EAGAIN:
+ retry++;
+ break;
+ case 0:
+ stats->nr_succeeded++;
+ break;
+ default:
+ nr_failed++;
+ stats->nr_failed_pages++;
+ break;
+ }
+ }
+ }
+ /* Count pages that exhausted the retry limit. */
+ nr_failed += retry;
+ stats->nr_failed_pages += retry;
+
+ return nr_failed;
+}
+
static int migrate_folio_split_on_alloc_fail(struct folio *folio,
struct list_head *split_folios, enum migrate_mode mode,
enum migrate_reason reason, struct migrate_pages_stats *stats)
@@ -1969,8 +2074,7 @@ static int migrate_folios_unmap(struct list_head *from,
* If we are holding the last folio reference, the folio
* was freed from under us, so just drop our reference.
*/
- if (likely(!page_has_movable_ops(&folio->page)) &&
- folio_ref_count(folio) == 1) {
+ if (folio_ref_count(folio) == 1) {
folio_clear_active(folio);
folio_clear_unevictable(folio);
list_del(&folio->lru);
@@ -2182,6 +2286,7 @@ int migrate_pages(struct list_head *from, new_folio_t get_new_folio,
struct folio *folio, *folio2;
LIST_HEAD(folios);
LIST_HEAD(hugetlb_folios);
+ LIST_HEAD(movable_ops_pages);
LIST_HEAD(ret_folios);
LIST_HEAD(split_folios);
struct migrate_pages_stats stats;
@@ -2190,9 +2295,12 @@ int migrate_pages(struct list_head *from, new_folio_t get_new_folio,

memset(&stats, 0, sizeof(stats));

- list_for_each_entry_safe(folio, folio2, from, lru)
+ list_for_each_entry_safe(folio, folio2, from, lru) {
if (folio_test_hugetlb(folio))
list_move_tail(&folio->lru, &hugetlb_folios);
+ else if (page_has_movable_ops(&folio->page))
+ list_move_tail(&folio->lru, &movable_ops_pages);
+ }

rc_gather = migrate_hugetlbs(&hugetlb_folios, get_new_folio,
put_new_folio, private, mode, reason,
@@ -2200,6 +2308,15 @@ int migrate_pages(struct list_head *from, new_folio_t get_new_folio,
if (rc_gather < 0)
goto out;

+ rc = migrate_movable_ops_pages(&movable_ops_pages, get_new_folio,
+ put_new_folio, private, mode, reason,
+ &stats, &ret_folios);
+ if (rc < 0) {
+ rc_gather = rc;
+ goto out;
+ }
+ rc_gather += rc;
+
again:
nr_pages = 0;
list_for_each_entry_safe(folio, folio2, from, lru) {
@@ -2246,6 +2363,7 @@ int migrate_pages(struct list_head *from, new_folio_t get_new_folio,
* will be put back to the right list by the caller.
*/
list_splice_tail(&hugetlb_folios, &ret_folios);
+ list_splice_tail(&movable_ops_pages, &ret_folios);
list_splice(&ret_folios, from);

/*

--
2.53.0