[PATCH v5 04/17] mm/memcg: add folio-based lruvec live helper

From: Kairui Song via B4 Relay

Date: Wed Nov 18 2026 - 16:05:52 EST


From: Kairui Song <kasong@xxxxxxxxxxx>

Add a helper that resolves a live lruvec for a folio under RCU
without taking the lruvec lock. It takes a folio directly so the
lruvec lookup happens inside the RCU read-side critical section,
which a lruvec-based interface cannot guarantee. The first user is
the lockless folio_inc_lru_refs() path later in the series.

Factor the ancestor walk out of lruvec_live_lock_irq() into
__lruvec_get_live(), so that both entry points share it, and spell
out why the walk is enough while only the RCU read lock is held:
reparenting of a memcg that starts dying waits for a grace period,
so it cannot race with the caller until the lock is released.

Move memcg_is_dying() up in memcontrol.h, next to the other memcg state
predicates, and use it for the walk rather than repeating the open coded
css_is_dying() test a few definitions away from where it is explained.

No functional change for existing callers.

Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@xxxxxxxxxxxxx>
Signed-off-by: Kairui Song <kasong@xxxxxxxxxxx>
---
include/linux/memcontrol.h | 71 ++++++++++++++++++++++++++++++++++++----------
1 file changed, 56 insertions(+), 15 deletions(-)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 980358bc6f49..441a5d57e723 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -571,6 +571,11 @@ static inline bool mem_cgroup_disabled(void)
return !cgroup_subsys_enabled(memory_cgrp_subsys);
}

+static inline bool memcg_is_dying(struct mem_cgroup *memcg)
+{
+ return memcg ? css_is_dying(&memcg->css) : false;
+}
+
static inline void mem_cgroup_protection(const struct mem_cgroup *root,
const struct mem_cgroup *memcg,
unsigned long *min,
@@ -1349,6 +1354,11 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
return true;
}

+static inline bool memcg_is_dying(struct mem_cgroup *memcg)
+{
+ return false;
+}
+
static inline
unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec,
enum lru_list lru, int zone_idx)
@@ -1510,28 +1520,41 @@ static inline void lruvec_lock_irq(struct lruvec *lruvec)
spin_lock_irq(&lruvec->lru_lock);
}

-static inline struct lruvec *lruvec_live_lock_irq(struct lruvec *lruvec)
+/*
+ * Reparenting moves a dying memcg's LRU state to its parent at any time,
+ * so anything added to that LRU afterwards would be lost. This helper
+ * walks up to the nearest live ancestor while the caller must hold the
+ * RCU read lock: reparenting of a memcg that starts dying waits for a
+ * grace period, so it cannot race with the caller until the lock is
+ * released.
+ */
+static inline struct lruvec *__lruvec_get_live(struct lruvec *lruvec)
{
#ifdef CONFIG_MEMCG
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);

- rcu_read_lock();
-
/*
* The memcg can be NULL when the memory controller is disabled.
* Otherwise, the caller keeps the memcg owning @lruvec alive.
*/
- while (unlikely(memcg && css_is_dying(&memcg->css))) {
+ while (unlikely(memcg_is_dying(memcg))) {
memcg = parent_mem_cgroup(memcg);
lruvec = mem_cgroup_lruvec(memcg, pgdat);
}
+#endif
+ return lruvec;
+}

+static inline struct lruvec *lruvec_live_lock_irq(struct lruvec *lruvec)
+{
+#ifdef CONFIG_MEMCG
+ rcu_read_lock();
+ lruvec = __lruvec_get_live(lruvec);
spin_lock_irq(&lruvec->lru_lock);
#else
lruvec_lock_irq(lruvec);
#endif
-
return lruvec;
}

@@ -1561,6 +1584,34 @@ static inline bool folio_matches_lruvec(const struct folio *folio,
lruvec_memcg(lruvec) == folio_memcg(folio);
}

+/**
+ * folio_lruvec_live_get - get a live lruvec for a folio under RCU
+ * @folio: the folio
+ *
+ * Same as folio_lruvec(), except that a folio whose memcg is dying is
+ * accounted to the nearest live ancestor, which is where reparenting
+ * would put it anyway.
+ *
+ * Return: the live lruvec, with rcu_read_lock() held. Pair it with
+ * folio_lruvec_live_put().
+ */
+static inline struct lruvec *folio_lruvec_live_get(struct folio *folio)
+{
+#ifdef CONFIG_MEMCG
+ rcu_read_lock();
+ return __lruvec_get_live(folio_lruvec(folio));
+#else
+ return folio_lruvec(folio);
+#endif
+}
+
+static inline void folio_lruvec_live_put(struct lruvec *lruvec)
+{
+#ifdef CONFIG_MEMCG
+ rcu_read_unlock();
+#endif
+}
+
/* Don't lock again iff page's lruvec locked */
static inline struct lruvec *folio_lruvec_relock_irq(const struct folio *folio,
struct lruvec *locked_lruvec)
@@ -1826,11 +1877,6 @@ void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask);

void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg);

-static inline bool memcg_is_dying(struct mem_cgroup *memcg)
-{
- return memcg ? css_is_dying(&memcg->css) : false;
-}
-
#else
static inline bool mem_cgroup_kmem_disabled(void)
{
@@ -1901,11 +1947,6 @@ static inline void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg,
static inline void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg)
{
}
-
-static inline bool memcg_is_dying(struct mem_cgroup *memcg)
-{
- return false;
-}
#endif /* CONFIG_MEMCG */

#if defined(CONFIG_MEMCG) && defined(CONFIG_LRU_GEN)

--
2.56.1