Re: [PATCH v5 16/17] mm/mglru: make folio_inc_lru_refs lruvec lockless
From: KunWu Chan
Date: Fri Nov 20 2026 - 04:42:18 EST
On Thu, Nov 19, 2026 at 5:04 AM Kairui Song via B4 Relay
<devnull+kasong.tencent.com@xxxxxxxxxx> wrote:
>
> From: Kairui Song <kasong@xxxxxxxxxxx>
>
> The lruvec spinlock in folio_inc_lru_refs was only taken to keep
> concurrent aging from corrupting the size counters. This is no
> longer needed: the per-generation counters are atomic, and the
> active/inactive ABI counters are updated per folio and linearized by
> the CAS on the folio flags, so concurrent aging can no longer corrupt
> them. The same argument is what lets folio_reset_lru_refs() update
> the counters without the lruvec lock.
>
> Use folio_lruvec_live_get()/folio_lruvec_live_put() to hold the RCU
> read lock around the lruvec lookup, and drop the spinlock entirely.
>
> Without the lock, aging can advance max_seq while a promotion is in
> flight. Generations are indexed by seq % MAX_NR_GENS, so the folio
> flags can return to the value read earlier and the cmpxchg succeeds
> with a target gen computed from a stale max_seq. Recheck max_seq after
> a gen-changing cmpxchg: if it moved, account the committed update and
> retry with LRU_REF_FORCE, which promotes the folio to the newest gen.
> The retry may count the access twice, which is acceptable and avoids
> folio_activate(), whose LRU lock causes latency jitter. smp_rmb()
> orders the flags read, including the one returned by a failed
> cmpxchg(), before the max_seq read.
>
> Also simplify the post-CAS accounting guards: gen is only assigned
> non-negative values after the old_gen < 0 early exit, so gen != old_gen
> implies gen >= 0, and lru_gen_update_size() needs no explicit guard.
> The active/inactive ABI update keeps its gen >= 0 check because an
> off-LRU folio has no lruvec to update.
>
> Signed-off-by: Kairui Song <kasong@xxxxxxxxxxx>
> ---
> include/linux/mm_inline.h | 10 +++++-----
> include/linux/mmzone.h | 3 ++-
> mm/vmscan.c | 51 +++++++++++++++++++++++++++++------------------
> 3 files changed, 39 insertions(+), 25 deletions(-)
>
> diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
> index 77f514143266..2c6cbd0df686 100644
> --- a/include/linux/mm_inline.h
> +++ b/include/linux/mm_inline.h
> @@ -317,10 +317,9 @@ static inline bool lru_gen_is_old(struct lruvec *lruvec, int type, int gen)
> gen == lru_gen_from_seq(min_seq + 1);
> }
>
> -static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,
> - int old_gen, int new_gen)
> +static inline void lru_gen_update_size(struct lruvec *lruvec, int type,
> + struct folio *folio, int old_gen, int new_gen)
> {
> - int type = folio_is_file_lru(folio);
> int zone = folio_zonenum(folio);
> int delta = folio_nr_pages(folio);
> struct lru_gen_folio *lrugen = &lruvec->lrugen;
> @@ -399,7 +398,7 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli
> /* use the refs from the atomic snapshot to avoid raced update */
> refs = lru_get_refs_flags(flags);
>
> - lru_gen_update_size(lruvec, folio, -1, gen);
> + lru_gen_update_size(lruvec, type, folio, -1, gen);
> if (lru_refs_is_active(refs))
> lru += LRU_ACTIVE;
> __update_lru_size(lruvec, lru, zone, delta);
> @@ -429,6 +428,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
> unsigned long flags;
> unsigned int refs;
> int gen;
> + int type = folio_is_file_lru(folio);
> int zone = folio_zonenum(folio);
> int delta = folio_nr_pages(folio);
> enum lru_list lru = folio_is_file_lru(folio) * LRU_INACTIVE_FILE;
> @@ -452,7 +452,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
> if (!reclaiming && ((max_seq - gen) % MAX_NR_GENS) < MIN_NR_GENS)
> folio_set_active(folio);
>
> - lru_gen_update_size(lruvec, folio, gen, -1);
> + lru_gen_update_size(lruvec, type, folio, gen, -1);
> if (lru_refs_is_active(refs))
> lru += LRU_ACTIVE;
> __update_lru_size(lruvec, lru, zone, -delta);
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index 2680e15b6de9..235fb04f8d78 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -540,9 +540,10 @@ enum lruvec_flags {
> #define LRU_TIER_MIN 0U
> #define LRU_TIER_MAX (MAX_NR_TIERS - 1)
>
> -/* Access source flags for folio_inc_lru_refs() */
> +/* Flags for folio_inc_lru_refs() */
> #define LRU_REF_MAPPED 0x1U
> #define LRU_REF_EXEC 0x2U
> +#define LRU_REF_FORCE 0x4U
>
> #define LRU_REFS_REFERENCED 0x1
> #define LRU_REFS_WORKINGSET 0x2
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index 78a1b4de7076..4576a7a369e3 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -876,37 +876,35 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
> long nr_pages = folio_nr_pages(folio);
> struct lru_gen_folio *lrugen;
> struct lruvec *lruvec = NULL;
> + bool aged = false;
>
> +retry:
> old_flags = READ_ONCE(*folio_flags(folio, 0));
> do {
> new_flags = old_flags;
> old_gen = lru_get_gen_flags(old_flags);
> old_refs = lru_get_refs_flags(old_flags);
> - file = folio_flags_is_file_lru(&old_flags);
> refs = old_refs + 1;
> gen = old_gen;
> + file = folio_flags_is_file_lru(&old_flags);
> if (old_gen < 0)
> goto out;
> - /*
> - * Lock the lruvec if the folio is on-list. We are already
> - * doing lazy promotion so in theory we don't need this,
> - * but for now, concurrent aging would still corrupt the
> - * size counters. This is a temporary limitation and
> - * will be lifted very soon, so the lock here is not a
> - * performance concern.
> - */
> if (!lruvec) {
> - lruvec = lruvec_live_lock_irq(folio_lruvec(folio));
> + lruvec = folio_lruvec_live_get(folio);
> lrugen = &lruvec->lrugen;
> }
> + /*
> + * Read the flags before max_seq, paired with the store of a
> + * gen, which samples max_seq first.
> + */
> + smp_rmb();
> max_seq = READ_ONCE(lrugen->max_seq);
> max_gen = lru_gen_from_seq(max_seq);
> if (old_gen == max_gen)
> goto out;
> -
> - if (flags & (LRU_REF_MAPPED | LRU_REF_EXEC)) {
> - /* Promote second page table access or executable */
> - if (refs > LRU_REFS_REFERENCED || flags & LRU_REF_EXEC)
> + if (flags & (LRU_REF_MAPPED | LRU_REF_EXEC | LRU_REF_FORCE)) {
> + if (lru_refs_is_workingset(refs) ||
> + flags & (LRU_REF_EXEC | LRU_REF_FORCE))
> gen = max_gen;
> /* First access only defers eviction of an old generation */
> else if (lru_gen_is_old(lruvec, file, old_gen))
> @@ -931,9 +929,19 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
> break;
> } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
>
> - if (gen != old_gen)
> - lru_gen_update_size(lruvec, folio, old_gen, gen);
> - if (lru_refs_is_active(old_refs) != lru_refs_is_active(refs) && old_gen >= 0) {
> + if (gen != old_gen) {
> + /*
> + * Gen-index reuse can fool cmpxchg(). Retry locklessly, accepting
> + * an extra reference to avoid folio_activate() latency.
> + */
> + if (unlikely(READ_ONCE(lrugen->max_seq) != max_seq)) {
> + flags = LRU_REF_FORCE;
> + aged = true;
> + }
> + lru_gen_update_size(lruvec, file, folio, old_gen, gen);
> + }
> +
> + if (lru_refs_is_active(old_refs) != lru_refs_is_active(refs) && gen >= 0) {
> enum lru_list lru = file * LRU_INACTIVE_FILE;
>
> __update_lru_size(lruvec, lru + lru_refs_is_active(old_refs),
> @@ -941,8 +949,12 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
> __update_lru_size(lruvec, lru + lru_refs_is_active(refs),
> folio_zonenum(folio), nr_pages);
Could this leave the parent's local active count too high?
Isolation can debit the child's active count after the refs
2 -> 3 CAS. If CSS_DYING is set before the stats update, the credit
can go to the parent. On SMP, reparenting can clamp the child's
negative local count to zero.
Thanks,
Kunwu
> }
> + if (aged) {
> + aged = false;
> + goto retry;
> + }
> if (lruvec)
> - lruvec_unlock_irq(lruvec);
> + folio_lruvec_live_put(lruvec);
> }
>
> /*
> @@ -3517,12 +3529,13 @@ static int __folio_inc_gen(struct lruvec *lruvec, struct folio *folio,
> /* protect pages selected by the tier PID controller */
> static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, int old_gen)
> {
> + int type = folio_is_file_lru(folio);
> int new_gen;
> bool gen_increased;
>
> new_gen = __folio_inc_gen(lruvec, folio, old_gen, &gen_increased);
> if (gen_increased)
> - lru_gen_update_size(lruvec, folio, old_gen, new_gen);
> + lru_gen_update_size(lruvec, type, folio, old_gen, new_gen);
>
> return new_gen;
> }
>
> --
> 2.56.1
>
>