Re: [PATCH for-next v3 6/7] mm/slab: refactor free_to_pcs()

From: Vlastimil Babka (SUSE)

Date: Wed Nov 18 2026 - 11:34:01 EST


On 11/15/26 16:09, Harry Yoo (Meta) wrote:
> Every caller of free_to_pcs() has to call can_free_to_pcs() before it.
> Refactor this by adding a slab parameter and calling can_free_to_pcs()
> from free_to_pcs(). To do this, move can_free_to_pcs() above
> free_to_pcs().
>
> Note that can_free_to_pcs() cannot be folded into free_to_pcs() as
> free_to_pcs_bulk() still needs it.
>
> While at it, let free_to_pcs() take free_flags instead of allow_spin.
>
> No functional change intended.
>
> Signed-off-by: Harry Yoo (Meta) <harry@xxxxxxxxxx>

Nice.

Reviewed-by: Vlastimil Babka (SUSE) <vbabka@xxxxxxxxxx>

I see that makes the nit for 4/7 immaterial.
But I have a new one :)

> ---
> mm/slub.c | 119 ++++++++++++++++++++++++++++++++------------------------------
> 1 file changed, 61 insertions(+), 58 deletions(-)
>
> diff --git a/mm/slub.c b/mm/slub.c
> index aec9fb6e2235..fd3de3bee1de 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -6148,15 +6148,71 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
> return pcs;
> }
>
> +static __always_inline bool can_free_to_pcs(struct slab *slab)
> +{
> + int slab_node;
> + int numa_node;
> +
> + if (!IS_ENABLED(CONFIG_NUMA))
> + goto check_pfmemalloc;
> +
> + slab_node = slab_nid(slab);
> +
> +#ifdef CONFIG_HAVE_MEMORYLESS_NODES
> + /*
> + * numa_mem_id() points to the closest node with memory so only allow
> + * objects from that node to the percpu sheaves
> + */
> + numa_node = numa_mem_id();
> +
> + if (likely(slab_node == numa_node))
> + goto check_pfmemalloc;
> +#else
> +
> + /*
> + * numa_mem_id() is only a wrapper to numa_node_id() which is where this
> + * cpu belongs to, but it might be a memoryless node anyway. We don't
> + * know what the closest node is.
> + */
> + numa_node = numa_node_id();
> +
> + /* freed object is from this cpu's node, proceed */
> + if (likely(slab_node == numa_node))
> + goto check_pfmemalloc;
> +
> + /*
> + * Freed object isn't from this cpu's node, but that node is memoryless
> + * or only has ZONE_MOVABLE memory, which slab cannot allocate from.
> + * Proceed as it's better to cache remote objects than falling back to
> + * the slowpath for everything. The allocation side can never obtain
> + * a local object anyway, if none exist. We don't have numa_mem_id() to
> + * point to the closest node as we would on a proper memoryless node
> + * setup.
> + */
> + if (unlikely(!node_state(numa_node, N_NORMAL_MEMORY)))
> + goto check_pfmemalloc;
> +#endif
> +
> + return false;
> +
> +check_pfmemalloc:
> + return likely(!slab_test_pfmemalloc(slab));
> +}
> +
> /*
> * Free an object to the percpu sheaves.
> * The object is expected to have passed slab_free_hook() already.
> */
> static __fastpath_inline
> -bool free_to_pcs(struct kmem_cache *s, void *object, bool allow_spin)
> +bool free_to_pcs(struct kmem_cache *s, struct slab *slab, void *object,
> + unsigned int free_flags)
> {
> + bool allow_spin = free_flags_allow_spinning(free_flags);

This could be moved to the if () block below that actually uses allow_spin.

> struct slub_percpu_sheaves *pcs;
>
> + if (unlikely(!can_free_to_pcs(slab)))
> + return false;
> +
> if (!local_trylock(&s->cpu_sheaves->lock))
> return false;
>
> @@ -6355,57 +6411,6 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
> return false;
> }
>
> -static __always_inline bool can_free_to_pcs(struct slab *slab)
> -{
> - int slab_node;
> - int numa_node;
> -
> - if (!IS_ENABLED(CONFIG_NUMA))
> - goto check_pfmemalloc;
> -
> - slab_node = slab_nid(slab);
> -
> -#ifdef CONFIG_HAVE_MEMORYLESS_NODES
> - /*
> - * numa_mem_id() points to the closest node with memory so only allow
> - * objects from that node to the percpu sheaves
> - */
> - numa_node = numa_mem_id();
> -
> - if (likely(slab_node == numa_node))
> - goto check_pfmemalloc;
> -#else
> -
> - /*
> - * numa_mem_id() is only a wrapper to numa_node_id() which is where this
> - * cpu belongs to, but it might be a memoryless node anyway. We don't
> - * know what the closest node is.
> - */
> - numa_node = numa_node_id();
> -
> - /* freed object is from this cpu's node, proceed */
> - if (likely(slab_node == numa_node))
> - goto check_pfmemalloc;
> -
> - /*
> - * Freed object isn't from this cpu's node, but that node is memoryless
> - * or only has ZONE_MOVABLE memory, which slab cannot allocate from.
> - * Proceed as it's better to cache remote objects than falling back to
> - * the slowpath for everything. The allocation side can never obtain
> - * a local object anyway, if none exist. We don't have numa_mem_id() to
> - * point to the closest node as we would on a proper memoryless node
> - * setup.
> - */
> - if (unlikely(!node_state(numa_node, N_NORMAL_MEMORY)))
> - goto check_pfmemalloc;
> -#endif
> -
> - return false;
> -
> -check_pfmemalloc:
> - return likely(!slab_test_pfmemalloc(slab));
> -}
> -
> /*
> * Try to free as many objects (already processed by free hooks) as possible to
> * a single per-cpu sheaf.
> @@ -6642,7 +6647,7 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
> if (unlikely(!slab_free_hook(s, object, free_flags)))
> return;
>
> - if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, object, true)))
> + if (likely(free_to_pcs(s, slab, object, free_flags)))
> return;
>
> __slab_free(s, slab, object, object, 1, addr);
> @@ -6656,7 +6661,6 @@ void memcg_alloc_abort_single(struct kmem_cache *s, void *object,
> unsigned int alloc_flags)
> {
> struct slab *slab = virt_to_slab(object);
> - bool allow_spin = alloc_flags_allow_spinning(alloc_flags);
> unsigned int free_flags = to_free_flags(alloc_flags);
>
> if (slab_want_init_on_free(s))
> @@ -6667,11 +6671,10 @@ void memcg_alloc_abort_single(struct kmem_cache *s, void *object,
> if (unlikely(!slab_free_hook(s, object, free_flags)))
> return;
>
> - if (likely(can_free_to_pcs(slab)) &&
> - likely(free_to_pcs(s, object, allow_spin)))
> + if (likely(free_to_pcs(s, slab, object, free_flags)))
> return;
>
> - if (likely(allow_spin))
> + if (likely(free_flags_allow_spinning(free_flags)))
> __slab_free(s, slab, object, object, 1, _RET_IP_);
> else
> defer_free(s, object);
> @@ -6991,7 +6994,7 @@ void kfree_nolock(const void *object)
> if (unlikely(!slab_free_hook(s, x, free_flags)))
> return;
>
> - if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, x, false)))
> + if (likely(free_to_pcs(s, slab, x, free_flags)))
> return;
>
> /*
>