[PATCH 02/15] mm: convert anon_vma rmap APIs to anon_rmap
From: tao
Date: Wed May 27 2026 - 07:29:36 EST
Convert the rmap anon_vma interfaces to anon_rmap APIs to clarify the
semantics of anonymous rmap operations and prepare for upcoming
ANON_VMA_LAZY support and RCU-based lockless rmap traversal.
Replace folio_anon_vma(), folio_get_anon_vma(), folio_lock_anon_vma_read(),
anon_vma_trylock_read(), anon_vma_lock_read(), anon_vma_unlock_read(),
anon_vma_trylock_write(), anon_vma_lock_write(), anon_vma_unlock_write(),
and vma_interval_tree_foreach() with the anon_rmap APIs.
No functional change intended.
Signed-off-by: tao <tao.wangtao@xxxxxxxxx>
---
include/linux/rmap.h | 6 ++--
mm/damon/ops-common.c | 4 +--
mm/huge_memory.c | 16 +++++------
mm/ksm.c | 43 ++++++++++++++---------------
mm/memory-failure.c | 11 ++++----
mm/migrate.c | 64 +++++++++++++++++++++----------------------
mm/page_idle.c | 2 +-
mm/rmap.c | 51 ++++++++++++++++++----------------
8 files changed, 98 insertions(+), 99 deletions(-)
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index c42314ea4362..9802bce92695 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -997,15 +997,13 @@ struct rmap_walk_control {
bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma,
unsigned long addr, void *arg);
int (*done)(struct folio *folio);
- struct anon_vma *(*anon_lock)(const struct folio *folio,
- struct rmap_walk_control *rwc);
+ anon_rmap_t (*anon_lock)(const struct folio *folio,
+ struct rmap_walk_control *rwc);
bool (*invalid_vma)(struct vm_area_struct *vma, void *arg);
};
void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc);
void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc);
-struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
- struct rmap_walk_control *rwc);
bool folio_maybe_same_anon_vma(const struct folio *folio,
const struct vm_area_struct *vma);
diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c
index 8c6d613425c1..5788410965b8 100644
--- a/mm/damon/ops-common.c
+++ b/mm/damon/ops-common.c
@@ -172,7 +172,7 @@ void damon_folio_mkold(struct folio *folio)
{
struct rmap_walk_control rwc = {
.rmap_one = damon_folio_mkold_one,
- .anon_lock = folio_lock_anon_vma_read,
+ .anon_lock = folio_lock_anon_rmap_read,
};
if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
@@ -236,7 +236,7 @@ bool damon_folio_young(struct folio *folio)
struct rmap_walk_control rwc = {
.arg = &accessed,
.rmap_one = damon_folio_young_one,
- .anon_lock = folio_lock_anon_vma_read,
+ .anon_lock = folio_lock_anon_rmap_read,
};
if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 970e077019b7..ab3c2397449a 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -4051,7 +4051,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
struct folio *end_folio = folio_next(folio);
bool is_anon = folio_test_anon(folio);
struct address_space *mapping = NULL;
- struct anon_vma *anon_vma = NULL;
+ anon_rmap_t anon_rmap = ANON_RMAP_NULL;
int old_order = folio_order(folio);
struct folio *new_folio, *next;
int nr_shmem_dropped = 0;
@@ -4087,12 +4087,12 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
* is taken to serialise against parallel split or collapse
* operations.
*/
- anon_vma = folio_get_anon_vma(folio);
- if (!anon_vma) {
+ anon_rmap = folio_get_anon_rmap(folio);
+ if (!anon_rmap_value(anon_rmap)) {
ret = -EBUSY;
goto out;
}
- anon_vma_lock_write(anon_vma);
+ anon_rmap_lock_write(anon_rmap);
mapping = NULL;
} else {
unsigned int min_order;
@@ -4122,7 +4122,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
}
}
- anon_vma = NULL;
+ anon_rmap = ANON_RMAP_NULL;
i_mmap_lock_read(mapping);
/*
@@ -4200,9 +4200,9 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
}
out_unlock:
- if (anon_vma) {
- anon_vma_unlock_write(anon_vma);
- put_anon_vma(anon_vma);
+ if (anon_rmap_value(anon_rmap)) {
+ anon_rmap_unlock_write(anon_rmap);
+ put_anon_rmap(anon_rmap);
}
if (mapping)
i_mmap_unlock_read(mapping);
diff --git a/mm/ksm.c b/mm/ksm.c
index 7d5b76478f0b..f4c204a8a379 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -187,7 +187,7 @@ struct ksm_stable_node {
/**
* struct ksm_rmap_item - reverse mapping item for virtual addresses
* @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list
- * @anon_vma: pointer to anon_vma for this mm,address, when in stable tree
+ * @anon_rmap: anonymous folio rmap for this mm,address, when in stable tree
* @nid: NUMA node id of unstable tree in which linked (may not match page)
* @mm: the memory structure this rmap_item is pointing into
* @address: the virtual address this rmap_item tracks (+ flags in low bits)
@@ -201,7 +201,7 @@ struct ksm_stable_node {
struct ksm_rmap_item {
struct ksm_rmap_item *rmap_list;
union {
- struct anon_vma *anon_vma; /* when stable */
+ anon_rmap_t anon_rmap; /* when stable */
#ifdef CONFIG_NUMA
int nid; /* when node of unstable tree */
#endif
@@ -786,7 +786,7 @@ static void break_cow(struct ksm_rmap_item *rmap_item)
* It is not an accident that whenever we want to break COW
* to undo, we also need to drop a reference to the anon_vma.
*/
- put_anon_vma(rmap_item->anon_vma);
+ put_anon_rmap(rmap_item->anon_rmap);
mmap_read_lock(mm);
vma = find_mergeable_vma(mm, addr);
@@ -898,7 +898,7 @@ static void remove_node_from_stable_tree(struct ksm_stable_node *stable_node)
VM_BUG_ON(stable_node->rmap_hlist_len <= 0);
stable_node->rmap_hlist_len--;
- put_anon_vma(rmap_item->anon_vma);
+ put_anon_rmap(rmap_item->anon_rmap);
rmap_item->address &= PAGE_MASK;
cond_resched();
}
@@ -1051,7 +1051,7 @@ static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item)
VM_BUG_ON(stable_node->rmap_hlist_len <= 0);
stable_node->rmap_hlist_len--;
- put_anon_vma(rmap_item->anon_vma);
+ put_anon_rmap(rmap_item->anon_rmap);
rmap_item->head = NULL;
rmap_item->address &= PAGE_MASK;
@@ -1598,9 +1598,8 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item,
/* Unstable nid is in union with stable anon_vma: remove first */
remove_rmap_item_from_tree(rmap_item);
- /* Must get reference to anon_vma while still holding mmap_lock */
- rmap_item->anon_vma = vma->anon_vma;
- get_anon_vma(vma->anon_vma);
+ /* Must get reference to anon_rmap while still holding mmap_lock */
+ rmap_item->anon_rmap = vma_get_anon_rmap(vma);
out:
mmap_read_unlock(mm);
trace_ksm_merge_with_ksm_page(kpage, page_to_pfn(kpage ? kpage : page),
@@ -3108,7 +3107,6 @@ struct folio *ksm_might_need_to_copy(struct folio *folio,
struct vm_area_struct *vma, unsigned long addr)
{
struct page *page = folio_page(folio, 0);
- struct anon_vma *anon_vma = folio_anon_vma(folio);
struct folio *new_folio;
if (folio_test_large(folio))
@@ -3118,10 +3116,10 @@ struct folio *ksm_might_need_to_copy(struct folio *folio,
if (folio_stable_node(folio) &&
!(ksm_run & KSM_RUN_UNMERGE))
return folio; /* no need to copy it */
- } else if (!anon_vma) {
+ } else if (!folio_test_anon(folio)) {
return folio; /* no need to copy it */
} else if (folio->index == linear_page_index(vma, addr) &&
- anon_vma->root == vma->anon_vma->root) {
+ folio_maybe_same_anon_vma(folio, vma)) {
return folio; /* still no need to copy it */
}
if (PageHWPoison(page))
@@ -3173,20 +3171,20 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
/* Ignore the stable/unstable/sqnr flags */
const unsigned long addr = rmap_item->address & PAGE_MASK;
- struct anon_vma *anon_vma = rmap_item->anon_vma;
+ anon_rmap_t anon_rmap = rmap_item->anon_rmap;
struct anon_vma_chain *vmac;
struct vm_area_struct *vma;
cond_resched();
- if (!anon_vma_trylock_read(anon_vma)) {
+ if (!anon_rmap_trylock_read(anon_rmap)) {
if (rwc->try_lock) {
rwc->contended = true;
return;
}
- anon_vma_lock_read(anon_vma);
+ anon_rmap_lock_read(anon_rmap);
}
- anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
+ anon_rmap_foreach_vma(vma, vmac, anon_rmap,
0, ULONG_MAX) {
cond_resched();
@@ -3207,15 +3205,15 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
continue;
if (!rwc->rmap_one(folio, vma, addr, rwc->arg)) {
- anon_vma_unlock_read(anon_vma);
+ anon_rmap_unlock_read(anon_rmap);
return;
}
if (rwc->done && rwc->done(folio)) {
- anon_vma_unlock_read(anon_vma);
+ anon_rmap_unlock_read(anon_rmap);
return;
}
}
- anon_vma_unlock_read(anon_vma);
+ anon_rmap_unlock_read(anon_rmap);
}
if (!search_new_forks++)
goto again;
@@ -3237,9 +3235,9 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page,
if (!stable_node)
return;
hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
- struct anon_vma *av = rmap_item->anon_vma;
+ anon_rmap_t anon_rmap = rmap_item->anon_rmap;
- anon_vma_lock_read(av);
+ anon_rmap_lock_read(anon_rmap);
rcu_read_lock();
for_each_process(tsk) {
struct anon_vma_chain *vmac;
@@ -3248,10 +3246,9 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page,
task_early_kill(tsk, force_early);
if (!t)
continue;
- anon_vma_interval_tree_foreach(vmac, &av->rb_root, 0,
+ anon_rmap_foreach_vma(vma, vmac, anon_rmap, 0,
ULONG_MAX)
{
- vma = vmac->vma;
if (vma->vm_mm == t->mm) {
addr = rmap_item->address & PAGE_MASK;
add_to_kill_ksm(t, page, vma, to_kill,
@@ -3260,7 +3257,7 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page,
}
}
rcu_read_unlock();
- anon_vma_unlock_read(av);
+ anon_rmap_unlock_read(anon_rmap);
}
}
#endif
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index ee42d4361309..bc9abba75b5d 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -547,11 +547,11 @@ static void collect_procs_anon(const struct folio *folio,
int force_early)
{
struct task_struct *tsk;
- struct anon_vma *av;
+ anon_rmap_t anon_rmap;
pgoff_t pgoff;
- av = folio_lock_anon_vma_read(folio, NULL);
- if (av == NULL) /* Not actually mapped anymore */
+ anon_rmap = folio_lock_anon_rmap_read(folio, NULL);
+ if (!anon_rmap_value(anon_rmap)) /* Not actually mapped anymore */
return;
pgoff = page_pgoff(folio, page);
@@ -564,9 +564,8 @@ static void collect_procs_anon(const struct folio *folio,
if (!t)
continue;
- anon_vma_interval_tree_foreach(vmac, &av->rb_root,
+ anon_rmap_foreach_vma(vma, vmac, anon_rmap,
pgoff, pgoff) {
- vma = vmac->vma;
if (vma->vm_mm != t->mm)
continue;
addr = page_mapped_in_vma(page, vma);
@@ -574,7 +573,7 @@ static void collect_procs_anon(const struct folio *folio,
}
}
rcu_read_unlock();
- anon_vma_unlock_read(av);
+ anon_rmap_unlock_read(anon_rmap);
}
/*
diff --git a/mm/migrate.c b/mm/migrate.c
index 8a64291ab5b4..769983cf14e0 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1142,18 +1142,18 @@ enum {
static void __migrate_folio_record(struct folio *dst,
int old_page_state,
- struct anon_vma *anon_vma)
+ anon_rmap_t anon_rmap)
{
- dst->private = (void *)anon_vma + old_page_state;
+ dst->private = (void *)anon_rmap_to_anon_vma(anon_rmap) + old_page_state;
}
static void __migrate_folio_extract(struct folio *dst,
int *old_page_state,
- struct anon_vma **anon_vmap)
+ anon_rmap_t *anon_rmapp)
{
unsigned long private = (unsigned long)dst->private;
- *anon_vmap = (struct anon_vma *)(private & ~PAGE_OLD_STATES);
+ *anon_rmapp = anon_vma_to_anon_rmap((void *)(private & ~PAGE_OLD_STATES));
*old_page_state = private & PAGE_OLD_STATES;
dst->private = NULL;
}
@@ -1161,15 +1161,15 @@ static void __migrate_folio_extract(struct folio *dst,
/* Restore the source folio to the original state upon failure */
static void migrate_folio_undo_src(struct folio *src,
int page_was_mapped,
- struct anon_vma *anon_vma,
+ anon_rmap_t anon_rmap,
bool locked,
struct list_head *ret)
{
if (page_was_mapped)
remove_migration_ptes(src, src, 0);
- /* Drop an anon_vma reference if we took one */
- if (anon_vma)
- put_anon_vma(anon_vma);
+ /* Drop an anon_rmap reference if we took one */
+ if (anon_rmap_value(anon_rmap))
+ put_anon_rmap(anon_rmap);
if (locked)
folio_unlock(src);
if (ret)
@@ -1210,7 +1210,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
struct folio *dst;
int rc = -EAGAIN;
int old_page_state = 0;
- struct anon_vma *anon_vma = NULL;
+ anon_rmap_t anon_rmap = ANON_RMAP_NULL;
bool locked = false;
bool dst_locked = false;
@@ -1275,19 +1275,19 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
/*
* By try_to_migrate(), src->mapcount goes down to 0 here. In this case,
* we cannot notice that anon_vma is freed while we migrate a page.
- * This get_anon_vma() delays freeing anon_vma pointer until the end
+ * This get_anon_rmap() delays freeing anon_rmap pointer until the end
* of migration. File cache pages are no problem because of page_lock()
* File Caches may use write_page() or lock_page() in migration, then,
* just care Anon page here.
*
- * Only folio_get_anon_vma() understands the subtleties of
- * getting a hold on an anon_vma from outside one of its mms.
- * But if we cannot get anon_vma, then we won't need it anyway,
+ * Only folio_get_anon_rmap() understands the subtleties of
+ * getting a hold on an anon_rmap from outside one of its mms.
+ * But if we cannot get anon_rmap, then we won't need it anyway,
* because that implies that the anon page is no longer mapped
* (and cannot be remapped so long as we hold the page lock).
*/
if (folio_test_anon(src) && !folio_test_ksm(src))
- anon_vma = folio_get_anon_vma(src);
+ anon_rmap = folio_get_anon_rmap(src);
/*
* Block others from accessing the new page when we get around to
@@ -1302,7 +1302,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
dst_locked = true;
if (unlikely(page_has_movable_ops(&src->page))) {
- __migrate_folio_record(dst, old_page_state, anon_vma);
+ __migrate_folio_record(dst, old_page_state, anon_rmap);
return 0;
}
@@ -1326,13 +1326,13 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
} else if (folio_mapped(src)) {
/* Establish migration ptes */
VM_BUG_ON_FOLIO(folio_test_anon(src) &&
- !folio_test_ksm(src) && !anon_vma, src);
+ !folio_test_ksm(src) && !anon_rmap_value(anon_rmap), src);
try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH : 0);
old_page_state |= PAGE_WAS_MAPPED;
}
if (!folio_mapped(src)) {
- __migrate_folio_record(dst, old_page_state, anon_vma);
+ __migrate_folio_record(dst, old_page_state, anon_rmap);
return 0;
}
@@ -1345,7 +1345,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
ret = NULL;
migrate_folio_undo_src(src, old_page_state & PAGE_WAS_MAPPED,
- anon_vma, locked, ret);
+ anon_rmap, locked, ret);
migrate_folio_undo_dst(dst, dst_locked, put_new_folio, private);
return rc;
@@ -1359,12 +1359,12 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
{
int rc;
int old_page_state = 0;
- struct anon_vma *anon_vma = NULL;
+ anon_rmap_t anon_rmap = ANON_RMAP_NULL;
bool src_deferred_split = false;
bool src_partially_mapped = false;
struct list_head *prev;
- __migrate_folio_extract(dst, &old_page_state, &anon_vma);
+ __migrate_folio_extract(dst, &old_page_state, &anon_rmap);
prev = dst->lru.prev;
list_del(&dst->lru);
@@ -1425,9 +1425,9 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
* and will be freed.
*/
list_del(&src->lru);
- /* Drop an anon_vma reference if we took one */
- if (anon_vma)
- put_anon_vma(anon_vma);
+ /* Drop an anon_rmap reference if we took one */
+ if (anon_rmap_value(anon_rmap))
+ put_anon_rmap(anon_rmap);
folio_unlock(src);
migrate_folio_done(src, reason);
@@ -1439,12 +1439,12 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
*/
if (rc == -EAGAIN) {
list_add(&dst->lru, prev);
- __migrate_folio_record(dst, old_page_state, anon_vma);
+ __migrate_folio_record(dst, old_page_state, anon_rmap);
return rc;
}
migrate_folio_undo_src(src, old_page_state & PAGE_WAS_MAPPED,
- anon_vma, true, ret);
+ anon_rmap, true, ret);
migrate_folio_undo_dst(dst, true, put_new_folio, private);
return rc;
@@ -1476,7 +1476,7 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio,
struct folio *dst;
int rc = -EAGAIN;
int page_was_mapped = 0;
- struct anon_vma *anon_vma = NULL;
+ anon_rmap_t anon_rmap = ANON_RMAP_NULL;
struct address_space *mapping = NULL;
enum ttu_flags ttu = 0;
@@ -1513,7 +1513,7 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio,
}
if (folio_test_anon(src))
- anon_vma = folio_get_anon_vma(src);
+ anon_rmap = folio_get_anon_rmap(src);
if (unlikely(!folio_trylock(dst)))
goto put_anon;
@@ -1550,8 +1550,8 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio,
folio_unlock(dst);
put_anon:
- if (anon_vma)
- put_anon_vma(anon_vma);
+ if (anon_rmap_value(anon_rmap))
+ put_anon_rmap(anon_rmap);
if (!rc) {
move_hugetlb_state(src, dst, reason);
@@ -1778,11 +1778,11 @@ static void migrate_folios_undo(struct list_head *src_folios,
dst2 = list_next_entry(dst, lru);
list_for_each_entry_safe(folio, folio2, src_folios, lru) {
int old_page_state = 0;
- struct anon_vma *anon_vma = NULL;
+ anon_rmap_t anon_rmap = ANON_RMAP_NULL;
- __migrate_folio_extract(dst, &old_page_state, &anon_vma);
+ __migrate_folio_extract(dst, &old_page_state, &anon_rmap);
migrate_folio_undo_src(folio, old_page_state & PAGE_WAS_MAPPED,
- anon_vma, true, ret_folios);
+ anon_rmap, true, ret_folios);
list_del(&dst->lru);
migrate_folio_undo_dst(dst, true, put_new_folio, private);
dst = dst2;
diff --git a/mm/page_idle.c b/mm/page_idle.c
index 9c67cbac2965..d4103f20f526 100644
--- a/mm/page_idle.c
+++ b/mm/page_idle.c
@@ -102,7 +102,7 @@ static void page_idle_clear_pte_refs(struct folio *folio)
*/
static struct rmap_walk_control rwc = {
.rmap_one = page_idle_clear_pte_refs_one,
- .anon_lock = folio_lock_anon_vma_read,
+ .anon_lock = folio_lock_anon_rmap_read,
};
if (!folio_mapped(folio) || !folio_raw_mapping(folio))
diff --git a/mm/rmap.c b/mm/rmap.c
index 1b2dada71778..41607168e00e 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -630,8 +630,8 @@ struct anon_vma *folio_get_anon_vma(const struct folio *folio)
* reference like with folio_get_anon_vma() and then block on the mutex
* on !rwc->try_lock case.
*/
-struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
- struct rmap_walk_control *rwc)
+static struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
+ struct rmap_walk_control *rwc)
{
struct anon_vma *anon_vma = NULL;
struct anon_vma *root_anon_vma;
@@ -744,6 +744,14 @@ void anon_rmap_unlock_read(anon_rmap_t anon_rmap)
anon_vma_unlock_read(anon_rmap_to_anon_vma(anon_rmap));
}
+static anon_rmap_t folio_anon_rmap(const struct folio *folio)
+{
+ struct anon_vma *anon_vma;
+
+ anon_vma = folio_anon_vma(folio);
+ return anon_vma ? anon_vma_to_anon_rmap(anon_vma) : ANON_RMAP_NULL;
+}
+
bool folio_maybe_same_anon_vma(const struct folio *folio,
const struct vm_area_struct *vma)
{
@@ -930,13 +938,11 @@ unsigned long page_address_in_vma(const struct folio *folio,
const struct page *page, const struct vm_area_struct *vma)
{
if (folio_test_anon(folio)) {
- struct anon_vma *anon_vma = folio_anon_vma(folio);
/*
* Note: swapoff's unuse_vma() is more efficient with this
* check, and needs it to match anon_vma when KSM is active.
*/
- if (!vma->anon_vma || !anon_vma ||
- vma->anon_vma->root != anon_vma->root)
+ if (!vma->anon_vma || !folio_maybe_same_anon_vma(folio, vma))
return -EFAULT;
} else if (!vma->vm_file) {
return -EFAULT;
@@ -944,7 +950,7 @@ unsigned long page_address_in_vma(const struct folio *folio,
return -EFAULT;
}
- /* KSM folios don't reach here because of the !anon_vma check */
+ /* The !folio_maybe_same_anon_vma() above handles KSM folios */
return vma_address(vma, page_pgoff(folio, page), 1);
}
@@ -1145,7 +1151,7 @@ int folio_referenced(struct folio *folio, int is_locked,
struct rmap_walk_control rwc = {
.rmap_one = folio_referenced_one,
.arg = (void *)&pra,
- .anon_lock = folio_lock_anon_vma_read,
+ .anon_lock = folio_lock_anon_rmap_read,
.try_lock = true,
.invalid_vma = invalid_folio_referenced_vma,
};
@@ -1580,8 +1586,7 @@ static void __page_check_anon_rmap(const struct folio *folio,
* are initially only visible via the pagetables, and the pte is locked
* over the call to folio_add_new_anon_rmap.
*/
- VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,
- folio);
+ VM_BUG_ON_FOLIO(!folio_maybe_same_anon_vma(folio, vma), folio);
VM_BUG_ON_PAGE(page_pgoff(folio, page) != linear_page_index(vma, address),
page);
}
@@ -2468,7 +2473,7 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags)
.rmap_one = try_to_unmap_one,
.arg = (void *)flags,
.done = folio_not_mapped,
- .anon_lock = folio_lock_anon_vma_read,
+ .anon_lock = folio_lock_anon_rmap_read,
};
if (flags & TTU_RMAP_LOCKED)
@@ -2813,7 +2818,7 @@ void try_to_migrate(struct folio *folio, enum ttu_flags flags)
.rmap_one = try_to_migrate_one,
.arg = (void *)flags,
.done = folio_not_mapped,
- .anon_lock = folio_lock_anon_vma_read,
+ .anon_lock = folio_lock_anon_rmap_read,
};
/*
@@ -2990,8 +2995,8 @@ void __put_anon_vma(struct anon_vma *anon_vma)
anon_vma_free(root);
}
-static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio,
- struct rmap_walk_control *rwc)
+static anon_rmap_t rmap_walk_anon_lock(const struct folio *folio,
+ struct rmap_walk_control *rwc)
{
struct anon_vma *anon_vma;
@@ -3006,7 +3011,7 @@ static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio,
*/
anon_vma = folio_anon_vma(folio);
if (!anon_vma)
- return NULL;
+ return ANON_RMAP_NULL;
if (anon_vma_trylock_read(anon_vma))
goto out;
@@ -3019,7 +3024,7 @@ static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio,
anon_vma_lock_read(anon_vma);
out:
- return anon_vma;
+ return anon_vma ? anon_vma_to_anon_rmap(anon_vma) : ANON_RMAP_NULL;
}
/*
@@ -3035,9 +3040,10 @@ static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio,
static void rmap_walk_anon(struct folio *folio,
struct rmap_walk_control *rwc, bool locked)
{
- struct anon_vma *anon_vma;
+ anon_rmap_t anon_rmap;
pgoff_t pgoff_start, pgoff_end;
struct anon_vma_chain *avc;
+ struct vm_area_struct *vma;
/*
* The folio lock ensures that folio->mapping can't be changed under us
@@ -3046,20 +3052,19 @@ static void rmap_walk_anon(struct folio *folio,
VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
if (locked) {
- anon_vma = folio_anon_vma(folio);
+ anon_rmap = folio_anon_rmap(folio);
/* anon_vma disappear under us? */
- VM_BUG_ON_FOLIO(!anon_vma, folio);
+ VM_BUG_ON_FOLIO(!anon_rmap_value(anon_rmap), folio);
} else {
- anon_vma = rmap_walk_anon_lock(folio, rwc);
+ anon_rmap = rmap_walk_anon_lock(folio, rwc);
}
- if (!anon_vma)
+ if (!anon_rmap_value(anon_rmap))
return;
pgoff_start = folio_pgoff(folio);
pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
- anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
+ anon_rmap_foreach_vma(vma, avc, anon_rmap,
pgoff_start, pgoff_end) {
- struct vm_area_struct *vma = avc->vma;
unsigned long address = vma_address(vma, pgoff_start,
folio_nr_pages(folio));
@@ -3076,7 +3081,7 @@ static void rmap_walk_anon(struct folio *folio,
}
if (!locked)
- anon_vma_unlock_read(anon_vma);
+ anon_rmap_unlock_read(anon_rmap);
}
/**
--
2.17.1