[RFC PATCH v2 10/13] mm: replace anon_vma with anon_node for ANON_VMA_FRACTAL
From: tao
Date: Tue Jul 07 2026 - 03:05:06 EST
Replace anon_vma with anon_node while keeping existing APIs to
minimize code changes.
Remove anon_vma_chain from VMA to save 16 bytes on 64-bit.
Use ANON_RMAP_FOREACH_VMA for anonymous reverse mapping walks in
ksm.c, memory-failure.c, and rmap.c.
Keep CONFIG_ANON_VMA_FRACTAL enabling in a separate patch for
easier control.
Signed-off-by: tao <tao.wangtao@xxxxxxxxx>
---
include/linux/mm.h | 2 ++
include/linux/mm_types.h | 2 ++
include/linux/rmap.h | 5 +++-
mm/internal.h | 58 +++++++++++++++++++++++++++++++++++-----
mm/ksm.c | 19 ++++---------
mm/memory-failure.c | 8 ++----
mm/mmap.c | 7 ++---
mm/rmap.c | 43 +++++++++++++++--------------
mm/vma.c | 4 +++
mm/vma_init.c | 2 ++
10 files changed, 100 insertions(+), 50 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 485df9c2dbdd..3fc2f92040b0 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -970,7 +970,9 @@ static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm)
memset(vma, 0, sizeof(*vma));
vma->vm_mm = mm;
vma->vm_ops = &vma_dummy_vm_ops;
+#ifndef CONFIG_ANON_VMA_FRACTAL
INIT_LIST_HEAD(&vma->anon_vma_chain);
+#endif
vma_lock_init(vma, false);
}
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index b18c2b2e7d2c..b98090e6cfdd 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -970,8 +970,10 @@ struct vm_area_struct {
* can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack
* or brk vma (with NULL file) can only be in an anon_vma list.
*/
+#ifndef CONFIG_ANON_VMA_FRACTAL
struct list_head anon_vma_chain; /* Serialized by mmap_lock &
* page_table_lock */
+#endif
struct anon_vma *anon_vma; /* Serialized by page_table_lock */
/* Function pointers to deal with this struct. */
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 75738f4ede1f..81ad29f3c392 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -15,6 +15,7 @@
#include <linux/memremap.h>
#include <linux/bit_spinlock.h>
+#ifndef CONFIG_ANON_VMA_FRACTAL
/*
* The anon_vma heads a list of private "related" vmas, to scan if
* an anonymous page pointing to this anon_vma needs to be unmapped:
@@ -67,7 +68,9 @@ struct anon_vma {
struct rb_root_cached rb_root;
};
-#ifdef CONFIG_ANON_VMA_FRACTAL
+#else
+
+#define anon_node anon_vma
/* struct anon_node - Replace anon_vma for anonymous page reverse mapping. */
struct anon_node {
diff --git a/mm/internal.h b/mm/internal.h
index 6f0ff492a89c..205f1b5226aa 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -212,6 +212,7 @@ static inline unsigned long vma_rmap_base(const struct vm_area_struct *vma)
return rmap_base(vma->vm_start, vma->vm_pgoff);
}
+#ifndef CONFIG_ANON_VMA_FRACTAL
static inline void get_anon_vma(struct anon_vma *anon_vma)
{
atomic_inc(&anon_vma->refcount);
@@ -255,7 +256,20 @@ static inline void anon_vma_unlock_read(struct anon_vma *anon_vma)
up_read(&anon_vma->root->rwsem);
}
-#ifdef CONFIG_ANON_VMA_FRACTAL
+/* Provide unified anon_vma VMA iteration interfaces. */
+#define ANON_RMAP_FOREACH_VMA(anon_vma, addr, start, last, rmap_proc_vma) \
+do { \
+ struct vm_area_struct *vma; /* must be named vma */ \
+ struct rb_root_cached *_rb_root = &(anon_vma)->rb_root; \
+ struct anon_vma_chain *_avc; \
+ \
+ anon_vma_interval_tree_foreach(_avc, _rb_root, start, last) { \
+ vma = _avc->vma; \
+ (rmap_proc_vma); \
+ } \
+} while (0)
+
+#else
static inline struct anon_node *anon_node_next_rbc_child(
struct anon_node *anon_nod, struct anon_node *node)
@@ -361,6 +375,36 @@ do { \
} \
} while (0)
+static inline void get_anon_vma(struct anon_vma *anon_vma)
+{
+ get_anon_node((void *)anon_vma);
+}
+
+static inline void put_anon_vma(struct anon_vma *anon_vma)
+{
+ put_anon_node((void *)anon_vma);
+}
+
+/* Huge pages are protected by folio_lock(), these APIs no longer needed. */
+static inline void anon_vma_lock_write(struct anon_vma *anon_vma) {}
+static inline int anon_vma_trylock_write(struct anon_vma *anon_vma) { return 1; }
+static inline void anon_vma_unlock_write(struct anon_vma *anon_vma) {}
+
+/* These are still used as rmap lock APIs. */
+static inline void anon_vma_lock_read(struct anon_vma *anon_vma)
+{
+ anon_node_lock_rmap((void *)anon_vma);
+}
+
+static inline int anon_vma_trylock_read(struct anon_vma *anon_vma)
+{
+ return anon_node_trylock_rmap((void *)anon_vma);
+}
+
+static inline void anon_vma_unlock_read(struct anon_vma *anon_vma)
+{
+ anon_node_unlock_rmap((void *)anon_vma);
+}
#endif
struct anon_vma *folio_get_anon_vma(const struct folio *folio);
@@ -380,18 +424,20 @@ int __anon_vma_prepare(struct vm_area_struct *vma);
void unlink_anon_vmas(struct vm_area_struct *vma);
#ifndef CONFIG_ANON_VMA_FRACTAL
+static inline int anon_vma_fork_with_prev(struct vm_area_struct *vma,
+ struct vm_area_struct *pvma, struct vm_area_struct *vma_prev,
+ struct vm_area_struct *pvma_prev)
+{
+ return anon_vma_fork(vma, pvma);
+}
static inline int vma_pre_update_rmap_base(struct vm_area_struct *vma,
unsigned long diff) { return 0; }
static inline void vma_post_update_rmap_base(struct vm_area_struct *vma,
unsigned long diff) {}
#else
-int anon_node_clone(struct vm_area_struct *vma, struct vm_area_struct *src,
- enum vma_operation operation);
-int anon_node_fork_with_prev(struct vm_area_struct *vma,
+int anon_vma_fork_with_prev(struct vm_area_struct *vma,
struct vm_area_struct *pvma, struct vm_area_struct *vma_prev,
struct vm_area_struct *pvma_prev);
-int __anon_node_prepare(struct vm_area_struct *vma);
-void unlink_anon_nodes(struct vm_area_struct *vma);
/* relocate_vma_down() changes vma_rmap_base. */
int vma_pre_update_rmap_base(struct vm_area_struct *vma, unsigned long diff);
void vma_post_update_rmap_base(struct vm_area_struct *vma, unsigned long diff);
diff --git a/mm/ksm.c b/mm/ksm.c
index 7d5b76478f0b..176b4da494ea 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -3174,8 +3174,6 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
/* Ignore the stable/unstable/sqnr flags */
const unsigned long addr = rmap_item->address & PAGE_MASK;
struct anon_vma *anon_vma = rmap_item->anon_vma;
- struct anon_vma_chain *vmac;
- struct vm_area_struct *vma;
cond_resched();
if (!anon_vma_trylock_read(anon_vma)) {
@@ -3186,11 +3184,8 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
anon_vma_lock_read(anon_vma);
}
- anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
- 0, ULONG_MAX) {
-
+ ANON_RMAP_FOREACH_VMA(anon_vma, addr, 0, ULONG_MAX, ({
cond_resched();
- vma = vmac->vma;
if (addr < vma->vm_start || addr >= vma->vm_end)
continue;
@@ -3214,7 +3209,7 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
anon_vma_unlock_read(anon_vma);
return;
}
- }
+ }));
anon_vma_unlock_read(anon_vma);
}
if (!search_new_forks++)
@@ -3230,7 +3225,6 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page,
{
struct ksm_stable_node *stable_node;
struct ksm_rmap_item *rmap_item;
- struct vm_area_struct *vma;
struct task_struct *tsk;
stable_node = folio_stable_node(folio);
@@ -3242,22 +3236,19 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page,
anon_vma_lock_read(av);
rcu_read_lock();
for_each_process(tsk) {
- struct anon_vma_chain *vmac;
unsigned long addr;
struct task_struct *t =
task_early_kill(tsk, force_early);
if (!t)
continue;
- anon_vma_interval_tree_foreach(vmac, &av->rb_root, 0,
- ULONG_MAX)
- {
- vma = vmac->vma;
+ ANON_RMAP_FOREACH_VMA(av, rmap_item->address,
+ 0, ULONG_MAX, ({
if (vma->vm_mm == t->mm) {
addr = rmap_item->address & PAGE_MASK;
add_to_kill_ksm(t, page, vma, to_kill,
addr);
}
- }
+ }));
}
rcu_read_unlock();
anon_vma_unlock_read(av);
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index 51508a55c405..1f44e5f76a42 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -545,21 +545,17 @@ static void collect_procs_anon(const struct folio *folio,
pgoff = page_pgoff(folio, page);
rcu_read_lock();
for_each_process(tsk) {
- struct vm_area_struct *vma;
- struct anon_vma_chain *vmac;
struct task_struct *t = task_early_kill(tsk, force_early);
unsigned long addr;
if (!t)
continue;
- anon_vma_interval_tree_foreach(vmac, &av->rb_root,
- pgoff, pgoff) {
- vma = vmac->vma;
+ ANON_RMAP_FOREACH_VMA(av, 0, pgoff, pgoff, ({
if (vma->vm_mm != t->mm)
continue;
addr = page_mapped_in_vma(page, vma);
add_to_kill_anon_file(t, page, vma, to_kill, addr);
- }
+ }));
}
rcu_read_unlock();
anon_vma_unlock_read(av);
diff --git a/mm/mmap.c b/mm/mmap.c
index 2311ae7c2ff4..c283e0de819a 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1730,7 +1730,8 @@ bool mmap_read_lock_maybe_expand(struct mm_struct *mm,
__latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
{
- struct vm_area_struct *mpnt, *tmp;
+ struct vm_area_struct *mpnt = NULL, *tmp = NULL;
+ struct vm_area_struct *prev = NULL, *tmp_prev = NULL;
int retval;
unsigned long charge = 0;
LIST_HEAD(uf);
@@ -1759,7 +1760,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
goto out;
mt_clear_in_rcu(vmi.mas.tree);
- for_each_vma(vmi, mpnt) {
+ for ( ; (mpnt = vma_next(&vmi)) != NULL; prev = mpnt, tmp_prev = tmp) {
struct file *file;
retval = vma_start_write_killable(mpnt);
@@ -1800,7 +1801,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
* copy page for current vma.
*/
tmp->anon_vma = NULL;
- } else if (anon_vma_fork(tmp, mpnt))
+ } else if (anon_vma_fork_with_prev(tmp, mpnt, tmp_prev, prev))
goto fail_nomem_anon_vma_fork;
vm_flags_clear(tmp, VM_LOCKED_MASK);
/*
diff --git a/mm/rmap.c b/mm/rmap.c
index 12c6a1bdff3f..11239838b722 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -84,6 +84,7 @@
#include "internal.h"
#include "swap.h"
+#ifndef CONFIG_ANON_VMA_FRACTAL
static struct kmem_cache *anon_vma_cachep;
static struct kmem_cache *anon_vma_chain_cachep;
@@ -558,7 +559,16 @@ void __init anon_vma_init(void)
SLAB_PANIC|SLAB_ACCOUNT);
}
-#ifdef CONFIG_ANON_VMA_FRACTAL
+void __put_anon_vma(struct anon_vma *anon_vma)
+{
+ struct anon_vma *root = anon_vma->root;
+
+ anon_vma_free(anon_vma);
+ if (root != anon_vma && atomic_dec_and_test(&root->refcount))
+ anon_vma_free(root);
+}
+
+#else
static struct kmem_cache *anon_node_cachep;
@@ -581,7 +591,7 @@ static void anon_node_ctor(void *data)
atomic_set(&anon_nod->refcount, 0);
}
-static inline void anon_vma_fractal_init(void)
+void __init anon_vma_init(void)
{
struct kmem_cache_args args = {
.use_freeptr_offset = true,
@@ -621,6 +631,11 @@ void __put_anon_node(struct anon_node *anon_nod)
anon_node_free(root);
}
+static inline void __put_anon_vma(struct anon_vma *anon_vma)
+{
+ __put_anon_node((struct anon_node *)(anon_vma));
+}
+
static inline struct anon_node *anon_node_alloc(struct vm_area_struct *vma,
struct anon_node *parent, unsigned long rmap_base, bool is_fork)
{
@@ -679,7 +694,7 @@ static bool try_reuse_anon_node(struct anon_node *anon_nod,
}
/* attach an anon_node to the VMA. */
-int __anon_node_prepare(struct vm_area_struct *vma)
+int __anon_vma_prepare(struct vm_area_struct *vma)
{
struct mm_struct *mm = vma->vm_mm;
unsigned long rmap_base = vma_rmap_base(vma);
@@ -785,7 +800,7 @@ static int anon_node_track_rmap(struct anon_node *anon_nod,
}
/* vma clones src anon_node and increments the count. */
-int anon_node_clone(struct vm_area_struct *vma, struct vm_area_struct *src,
+int anon_vma_clone(struct vm_area_struct *vma, struct vm_area_struct *src,
enum vma_operation operation)
{
struct anon_node *anon_nod = vma_anon_node(src);
@@ -827,7 +842,7 @@ static struct anon_node *fork_reuse_ancestor(struct anon_node *parent,
}
/* Clone an anon_node from prev or fork one from the parent pvma. */
-int anon_node_fork_with_prev(struct vm_area_struct *vma,
+int anon_vma_fork_with_prev(struct vm_area_struct *vma,
struct vm_area_struct *pvma, struct vm_area_struct *prev,
struct vm_area_struct *pvma_prev)
{
@@ -921,7 +936,7 @@ static void release_anon_nodes(struct anon_node *anon_nod)
}
/* Remove link between this VMA and its anon_node. */
-void unlink_anon_nodes(struct vm_area_struct *vma)
+void unlink_anon_vmas(struct vm_area_struct *vma)
{
struct anon_node *anon_nod = vma_anon_node(vma);
struct rw_semaphore *rmap_sem;
@@ -3377,15 +3392,6 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr,
EXPORT_SYMBOL_GPL(make_device_exclusive);
#endif
-void __put_anon_vma(struct anon_vma *anon_vma)
-{
- struct anon_vma *root = anon_vma->root;
-
- anon_vma_free(anon_vma);
- if (root != anon_vma && atomic_dec_and_test(&root->refcount))
- anon_vma_free(root);
-}
-
static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio,
struct rmap_walk_control *rwc)
{
@@ -3433,7 +3439,6 @@ static void rmap_walk_anon(struct folio *folio,
{
struct anon_vma *anon_vma;
pgoff_t pgoff_start, pgoff_end;
- struct anon_vma_chain *avc;
/*
* The folio lock ensures that folio->mapping can't be changed under us
@@ -3453,9 +3458,7 @@ static void rmap_walk_anon(struct folio *folio,
pgoff_start = folio_pgoff(folio);
pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
- anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
- pgoff_start, pgoff_end) {
- struct vm_area_struct *vma = avc->vma;
+ ANON_RMAP_FOREACH_VMA(anon_vma, 0, pgoff_start, pgoff_end, ({
unsigned long address = vma_address(vma, pgoff_start,
folio_nr_pages(folio));
@@ -3469,7 +3472,7 @@ static void rmap_walk_anon(struct folio *folio,
break;
if (rwc->done && rwc->done(folio))
break;
- }
+ }));
if (!locked)
anon_vma_unlock_read(anon_vma);
diff --git a/mm/vma.c b/mm/vma.c
index 02178112daa0..76294a7201ca 100644
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -78,7 +78,11 @@ static bool vma_is_fork_child(struct vm_area_struct *vma)
* parents. This can improve scalability caused by the anon_vma root
* lock.
*/
+#ifndef CONFIG_ANON_VMA_FRACTAL
return vma && vma->anon_vma && !list_is_singular(&vma->anon_vma_chain);
+#else
+ return vma && vma->anon_vma && vma->anon_vma->depth > 1;
+#endif
}
static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next)
diff --git a/mm/vma_init.c b/mm/vma_init.c
index 3c0b65950510..ffb7359ea47f 100644
--- a/mm/vma_init.c
+++ b/mm/vma_init.c
@@ -134,7 +134,9 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
return NULL;
}
vma_lock_init(new, true);
+#ifndef CONFIG_ANON_VMA_FRACTAL
INIT_LIST_HEAD(&new->anon_vma_chain);
+#endif
vma_numab_state_init(new);
dup_anon_vma_name(orig, new);
--
2.17.1