[PATCH RFC 04/32] mm/mglru: introduce and use helpers for updating lru_gen refs and gen
From: Kairui Song via B4 Relay
Date: Fri May 01 2026 - 17:04:24 EST
From: Kairui Song <kasong@xxxxxxxxxxx>
Instead of keep touching the raw page flags, use helpers for adjusting
folio's refs and gen info, make the code easier to debug and understand.
Signed-off-by: Kairui Song <kasong@xxxxxxxxxxx>
---
include/linux/mm_inline.h | 79 +++++++++++++++++++++++++++++++++++++++++------
include/linux/mmzone.h | 2 ++
mm/migrate.c | 2 --
mm/swap.c | 17 +++++-----
mm/vmscan.c | 52 ++++++++++++++++++-------------
5 files changed, 111 insertions(+), 41 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 9c8ad8af37de..eade9f2d6afc 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -141,10 +141,42 @@ static inline int lru_tier_from_refs(int refs, bool workingset)
return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs);
}
-static inline int folio_lru_refs(const struct folio *folio)
+/**
+ * lru_gen_from_flags - Return the LRU generation number from folio flags.
+ * @flags: folio flags
+ *
+ * Returns: A number between 0 and LRU_GEN_MAX, inclusive. Returns -1 if the
+ * flags indicate the folio is off the list (e.g., isolated).
+ */
+static inline int lru_gen_from_flags(unsigned long flags)
{
- unsigned long flags = READ_ONCE(*const_folio_flags(folio, 0));
+ int gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF);
+ gen -= 1;
+ VM_WARN_ON_ONCE(gen != -1 && gen > LRU_GEN_MAX);
+ return gen;
+}
+
+/**
+ * lru_gen_set_flags - Set the LRU generation number to specified folio flags.
+ * @flags: pointer to the folio flags
+ * @gen: generation number, between 0 and LRU_GEN_MAX, inclusive.
+ */
+static inline void lru_gen_set_flags(unsigned long *flags, int gen)
+{
+ VM_WARN_ON_ONCE(gen > LRU_GEN_MAX || gen < 0);
+ BUILD_BUG_ON((LRU_GEN_MAX + 1) != MAX_NR_GENS);
+
+ *flags &= ~LRU_GEN_MASK;
+ *flags |= (gen + 1UL) << LRU_GEN_PGOFF;
+}
+
+/**
+ * lru_refs_from_flags - Return LRU referenced / access count from folio flags.
+ * @flags: folio flags
+ */
+static inline int lru_refs_from_flags(unsigned long flags)
+{
if (!(flags & BIT(PG_referenced)))
return 0;
/*
@@ -154,18 +186,46 @@ static inline int folio_lru_refs(const struct folio *folio)
return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1;
}
-static inline int folio_lru_gen(const struct folio *folio)
+/**
+ * lru_refs_set_flags - Set the LRU referenced / access count to specified folio flags.
+ * @flags: pointer to the folio flags
+ * @refs: referenced / access count number, between 0 and LRU_REFS_MAX, inclusive.
+ */
+static inline void lru_refs_set_flags(unsigned long *flags, unsigned int refs)
+{
+ VM_WARN_ON_ONCE(refs > LRU_REFS_MAX);
+
+ *flags &= ~LRU_REFS_FLAGS;
+ if (!refs)
+ return;
+ *flags |= (BIT(PG_referenced) | ((refs - 1UL) << LRU_REFS_PGOFF));
+}
+
+static inline int folio_lru_refs(const struct folio *folio)
+{
+ return lru_refs_from_flags(READ_ONCE(*const_folio_flags(folio, 0)));
+}
+
+static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs)
{
- unsigned long flags = READ_ONCE(*const_folio_flags(folio, 0));
+ unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+
+ do {
+ new_flags = old_flags;
+ lru_refs_set_flags(&new_flags, refs);
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
+}
- return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+static inline int folio_lru_gen(const struct folio *folio)
+{
+ return lru_gen_from_flags(READ_ONCE(*const_folio_flags(folio, 0)));
}
static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen)
{
unsigned long max_seq = lruvec->lrugen.max_seq;
- VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
+ VM_WARN_ON_ONCE(gen > LRU_GEN_MAX);
/* see the comment on MIN_NR_GENS */
return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1);
@@ -305,9 +365,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
static inline void folio_migrate_refs(struct folio *new, const struct folio *old)
{
- unsigned long refs = READ_ONCE(*const_folio_flags(old, 0)) & LRU_REFS_MASK;
-
- set_mask_bits(folio_flags(new, 0), LRU_REFS_MASK, refs);
+ folio_set_lru_refs(new, folio_lru_refs(old));
}
#else /* !CONFIG_LRU_GEN */
@@ -338,7 +396,8 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
static inline void folio_migrate_refs(struct folio *new, const struct folio *old)
{
-
+ if (folio_test_referenced(old))
+ folio_set_referenced(new);
}
#endif /* CONFIG_LRU_GEN */
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 9adb2ad21da5..e4c51961ec27 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -493,7 +493,9 @@ enum lruvec_flags {
#ifndef __GENERATING_BOUNDS_H
#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
+#define LRU_GEN_MAX (BIT(LRU_GEN_WIDTH - 1) - 1)
#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
+#define LRU_REFS_MAX (BIT(LRU_REFS_WIDTH) - 1)
/*
* For folios accessed multiple times through file descriptors,
diff --git a/mm/migrate.c b/mm/migrate.c
index 8a64291ab5b4..23248484a165 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -763,8 +763,6 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
{
int cpupid;
- if (folio_test_referenced(folio))
- folio_set_referenced(newfolio);
if (folio_test_uptodate(folio))
folio_mark_uptodate(newfolio);
if (folio_test_clear_active(folio)) {
diff --git a/mm/swap.c b/mm/swap.c
index e7037ea2c10f..6204496d48f5 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -393,24 +393,26 @@ static void __lru_cache_activate_folio(struct folio *folio)
static void lru_gen_inc_refs(struct folio *folio)
{
unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+ int refs;
if (folio_test_unevictable(folio))
return;
- /* see the comment on LRU_REFS_FLAGS */
- if (!folio_test_referenced(folio)) {
- set_mask_bits(folio_flags(folio, 0), LRU_REFS_MASK, BIT(PG_referenced));
+ if (!folio_lru_refs(folio)) {
+ folio_set_lru_refs(folio, 1);
return;
}
+ /* see the comment on LRU_REFS_FLAGS */
do {
- if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) {
+ new_flags = old_flags;
+ refs = lru_refs_from_flags(old_flags);
+ if (refs == LRU_REFS_MAX) {
if (!folio_test_workingset(folio))
folio_set_workingset(folio);
return;
}
-
- new_flags = old_flags + BIT(LRU_REFS_PGOFF);
+ lru_refs_set_flags(&new_flags, refs + 1);
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
}
@@ -423,7 +425,8 @@ static bool lru_gen_clear_refs(struct folio *folio)
if (gen < 0)
return true;
- set_mask_bits(folio_flags(folio, 0), LRU_REFS_FLAGS | BIT(PG_workingset), 0);
+ folio_set_lru_refs(folio, 0);
+ folio_clear_workingset(folio);
rcu_read_lock();
seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 7a1f08147dee..2ca1d6d80259 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -843,12 +843,14 @@ enum folio_references {
static bool lru_gen_set_refs(struct folio *folio)
{
/* see the comment on LRU_REFS_FLAGS */
- if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
- set_mask_bits(folio_flags(folio, 0), LRU_REFS_MASK, BIT(PG_referenced));
+ if (!folio_lru_refs(folio) && !folio_test_workingset(folio)) {
+ folio_set_lru_refs(folio, 1);
return false;
}
- set_mask_bits(folio_flags(folio, 0), LRU_REFS_FLAGS, BIT(PG_workingset));
+ folio_set_lru_refs(folio, 0);
+ folio_set_workingset(folio);
+
return true;
}
#else
@@ -3191,28 +3193,31 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
******************************************************************************/
/* promote pages accessed through page tables */
-static int folio_update_gen(struct folio *folio, int gen)
+static int folio_update_gen(struct folio *folio, int new_gen)
{
unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
-
- VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
+ int old_gen;
/* see the comment on LRU_REFS_FLAGS */
- if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
- set_mask_bits(folio_flags(folio, 0), LRU_REFS_MASK, BIT(PG_referenced));
+ if (!lru_refs_from_flags(old_flags) && !folio_test_workingset(folio)) {
+ folio_set_lru_refs(folio, 1);
return -1;
}
do {
+ old_gen = lru_gen_from_flags(old_flags);
+ new_flags = old_flags;
+
/* lru_gen_del_folio() has isolated this page? */
- if (!(old_flags & LRU_GEN_MASK))
- return -1;
+ if (old_gen < 0)
+ break;
- new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
- new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset);
+ lru_gen_set_flags(&new_flags, new_gen);
+ lru_refs_set_flags(&new_flags, 0);
+ new_flags |= BIT(PG_workingset);
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
- return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+ return old_gen;
}
/* protect pages accessed multiple times through file descriptors */
@@ -3220,17 +3225,18 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
{
int type = folio_is_file_lru(folio);
struct lru_gen_folio *lrugen = &lruvec->lrugen;
- int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
+ int old_gen, new_gen, min_gen = lru_gen_from_seq(lrugen->min_seq[type]);
unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
- VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
-
do {
- new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+ old_gen = lru_gen_from_flags(old_flags);
+ VM_WARN_ON_ONCE_FOLIO(old_gen < 0, folio);
+
/* folio_update_gen() has promoted this page? */
- if (new_gen >= 0 && new_gen != old_gen)
- return new_gen;
+ if (old_gen >= 0 && old_gen != min_gen)
+ return old_gen;
+ new_flags = old_flags;
new_gen = (old_gen + 1) % MAX_NR_GENS;
new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
@@ -4639,7 +4645,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
/* see the comment on LRU_REFS_FLAGS */
if (!folio_test_referenced(folio))
- set_mask_bits(folio_flags(folio, 0), LRU_REFS_MASK, 0);
+ folio_set_lru_refs(folio, 0);
success = lru_gen_del_folio(lruvec, folio, true);
VM_WARN_ON_ONCE_FOLIO(!success, folio);
@@ -4854,8 +4860,10 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
}
/* don't add rejected folios to the oldest generation */
- if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])
- set_mask_bits(folio_flags(folio, 0), LRU_REFS_FLAGS, BIT(PG_active));
+ if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
+ folio_set_lru_refs(folio, 0);
+ folio_set_active(folio);
+ }
}
move_folios_to_lru(&list);
--
2.54.0