Re: [PATCH v5 03/17] mm/mglru: make generation page counters atomic

From: KunWu Chan

Date: Fri Nov 20 2026 - 01:49:44 EST


On Thu, Nov 19, 2026 at 5:03 AM Kairui Song via B4 Relay
<devnull+kasong.tencent.com@xxxxxxxxxx> wrote:
>
> From: Kairui Song <kasong@xxxxxxxxxxx>
>
> No feature change, convert them to atomic so we can update them without
> holding the LRU lock. A temporary underflow is fine: readers clamp a
> negative value to zero, and the counters only need to be eventually
> consistent.
>
> Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@xxxxxxxxxxxxx>
> Signed-off-by: Kairui Song <kasong@xxxxxxxxxxx>
> ---
> include/linux/mm_inline.h | 6 ++----
> include/linux/mmzone.h | 2 +-
> mm/vmscan.c | 26 +++++++++++++-------------
> mm/vmscan.h | 2 +-
> 4 files changed, 17 insertions(+), 19 deletions(-)
>
> diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
> index 2edb9beef6d3..46beded51c7d 100644
> --- a/include/linux/mm_inline.h
> +++ b/include/linux/mm_inline.h
> @@ -265,11 +265,9 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli
> VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1);
>
> if (old_gen >= 0)
> - WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
> - lrugen->nr_pages[old_gen][type][zone] - delta);
> + atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]);
> if (new_gen >= 0)
> - WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone],
> - lrugen->nr_pages[new_gen][type][zone] + delta);
> + atomic_long_add(delta, &lrugen->nr_pages[new_gen][type][zone]);
>
> /* addition */
> if (old_gen < 0) {
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index c161d563ce8f..4c4aad95b0b6 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -554,7 +554,7 @@ struct lru_gen_folio {
> /* the multi-gen LRU lists, lazily sorted on eviction */
> struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
> /* the multi-gen LRU sizes, eventually consistent */
> - long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
> + atomic_long_t nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
> /* the exponential moving average of refaulted */
> unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS];
> /* the exponential moving average of evicted+protected */
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index fe1fe1439c7c..8d14831cdf83 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -3350,8 +3350,7 @@ static void reset_batch_size(struct lru_gen_mm_walk *walk)
> continue;
>
> walk->nr_pages[gen][type][zone] = 0;
> - WRITE_ONCE(lrugen->nr_pages[gen][type][zone],
> - lrugen->nr_pages[gen][type][zone] + delta);
> + atomic_long_add(delta, &lrugen->nr_pages[gen][type][zone]);
>
> if (lru_gen_is_active(lruvec, gen))
> lru += LRU_ACTIVE;
> @@ -3971,11 +3970,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
> break;
> }
> flush_lru_batch(head, &batch_end, target_list);
> -
> - WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
> - lrugen->nr_pages[old_gen][type][zone] - delta);
> - WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone],
> - lrugen->nr_pages[target_gen][type][zone] + delta);
> + atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]);
> + atomic_long_add(delta, &lrugen->nr_pages[target_gen][type][zone]);
> if (!remaining)
> return false;
> }
> @@ -4093,8 +4089,8 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness
> for (type = 0; type < ANON_AND_FILE; type++) {
> for (zone = 0; zone < MAX_NR_ZONES; zone++) {
> enum lru_list lru = type * LRU_INACTIVE_FILE;
> - long delta = lrugen->nr_pages[prev][type][zone] -
> - lrugen->nr_pages[next][type][zone];
> + long delta = atomic_long_read(&lrugen->nr_pages[prev][type][zone]) -
> + atomic_long_read(&lrugen->nr_pages[next][type][zone]);
>
> if (!delta)
> continue;
> @@ -4625,8 +4621,13 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
> child_lrugen = &child_lruvec->lrugen;
> parent_lrugen = &parent_lruvec->lrugen;
>
> + /*
> + * The child is offline and the lockless updaters, which hold
> + * rcu_read_lock() over the update, avoid racing with reparenting,
> + * so the read/set/add operations below are safe.
> + */
> for (gen = 0; gen < MAX_NR_GENS; gen++) {
> - long nr_pages = child_lrugen->nr_pages[gen][type][zone];
> + long nr_pages = atomic_long_read(&child_lrugen->nr_pages[gen][type][zone]);
> int child_lru_active = lru_gen_is_active(child_lruvec, gen) ? LRU_ACTIVE : 0;
> int parent_lru_active = lru_gen_is_active(parent_lruvec, gen) ? LRU_ACTIVE : 0;
>
> @@ -4634,9 +4635,8 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
> list_splice_tail_init(&child_lrugen->folios[gen][type][zone],
> &parent_lrugen->folios[gen][type][zone]);
>
> - WRITE_ONCE(child_lrugen->nr_pages[gen][type][zone], 0);
> - WRITE_ONCE(parent_lrugen->nr_pages[gen][type][zone],
> - parent_lrugen->nr_pages[gen][type][zone] + nr_pages);
> + atomic_long_set(&child_lrugen->nr_pages[gen][type][zone], 0);
> + atomic_long_add(nr_pages, &parent_lrugen->nr_pages[gen][type][zone]);
>
> if (lru_gen_is_active(child_lruvec, gen) != lru_gen_is_active(parent_lruvec, gen)) {
> __update_lru_size(child_lruvec, lru + child_lru_active, zone, -nr_pages);
> diff --git a/mm/vmscan.h b/mm/vmscan.h
> index c2cea36289f3..6551f0a6691a 100644
> --- a/mm/vmscan.h
> +++ b/mm/vmscan.h
> @@ -126,7 +126,7 @@ static inline unsigned long lruvec_gen_size(struct lru_gen_folio *lrugen,
> unsigned long size = 0;
>
> for (int zone = 0; zone < max_zone; zone++)
> - size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
> + size += max(atomic_long_read(&lrugen->nr_pages[gen][type][zone]), 0L);
>
> return size;
> }
>
> --
> 2.56.1
>
>

Reviewed-by: Kunwu Chan <kunwu.chan@xxxxxxxxx>

Thanks,
Kunwu