[PATCH for-next v3 6/7] mm/slab: refactor free_to_pcs()

From: Harry Yoo (Meta)

Date: Sun Nov 15 2026 - 10:10:41 EST


Every caller of free_to_pcs() has to call can_free_to_pcs() before it.
Refactor this by adding a slab parameter and calling can_free_to_pcs()
from free_to_pcs(). To do this, move can_free_to_pcs() above
free_to_pcs().

Note that can_free_to_pcs() cannot be folded into free_to_pcs() as
free_to_pcs_bulk() still needs it.

While at it, let free_to_pcs() take free_flags instead of allow_spin.

No functional change intended.

Signed-off-by: Harry Yoo (Meta) <harry@xxxxxxxxxx>
---
mm/slub.c | 119 ++++++++++++++++++++++++++++++++------------------------------
1 file changed, 61 insertions(+), 58 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index aec9fb6e2235..fd3de3bee1de 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -6148,15 +6148,71 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
return pcs;
}

+static __always_inline bool can_free_to_pcs(struct slab *slab)
+{
+ int slab_node;
+ int numa_node;
+
+ if (!IS_ENABLED(CONFIG_NUMA))
+ goto check_pfmemalloc;
+
+ slab_node = slab_nid(slab);
+
+#ifdef CONFIG_HAVE_MEMORYLESS_NODES
+ /*
+ * numa_mem_id() points to the closest node with memory so only allow
+ * objects from that node to the percpu sheaves
+ */
+ numa_node = numa_mem_id();
+
+ if (likely(slab_node == numa_node))
+ goto check_pfmemalloc;
+#else
+
+ /*
+ * numa_mem_id() is only a wrapper to numa_node_id() which is where this
+ * cpu belongs to, but it might be a memoryless node anyway. We don't
+ * know what the closest node is.
+ */
+ numa_node = numa_node_id();
+
+ /* freed object is from this cpu's node, proceed */
+ if (likely(slab_node == numa_node))
+ goto check_pfmemalloc;
+
+ /*
+ * Freed object isn't from this cpu's node, but that node is memoryless
+ * or only has ZONE_MOVABLE memory, which slab cannot allocate from.
+ * Proceed as it's better to cache remote objects than falling back to
+ * the slowpath for everything. The allocation side can never obtain
+ * a local object anyway, if none exist. We don't have numa_mem_id() to
+ * point to the closest node as we would on a proper memoryless node
+ * setup.
+ */
+ if (unlikely(!node_state(numa_node, N_NORMAL_MEMORY)))
+ goto check_pfmemalloc;
+#endif
+
+ return false;
+
+check_pfmemalloc:
+ return likely(!slab_test_pfmemalloc(slab));
+}
+
/*
* Free an object to the percpu sheaves.
* The object is expected to have passed slab_free_hook() already.
*/
static __fastpath_inline
-bool free_to_pcs(struct kmem_cache *s, void *object, bool allow_spin)
+bool free_to_pcs(struct kmem_cache *s, struct slab *slab, void *object,
+ unsigned int free_flags)
{
+ bool allow_spin = free_flags_allow_spinning(free_flags);
struct slub_percpu_sheaves *pcs;

+ if (unlikely(!can_free_to_pcs(slab)))
+ return false;
+
if (!local_trylock(&s->cpu_sheaves->lock))
return false;

@@ -6355,57 +6411,6 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
return false;
}

-static __always_inline bool can_free_to_pcs(struct slab *slab)
-{
- int slab_node;
- int numa_node;
-
- if (!IS_ENABLED(CONFIG_NUMA))
- goto check_pfmemalloc;
-
- slab_node = slab_nid(slab);
-
-#ifdef CONFIG_HAVE_MEMORYLESS_NODES
- /*
- * numa_mem_id() points to the closest node with memory so only allow
- * objects from that node to the percpu sheaves
- */
- numa_node = numa_mem_id();
-
- if (likely(slab_node == numa_node))
- goto check_pfmemalloc;
-#else
-
- /*
- * numa_mem_id() is only a wrapper to numa_node_id() which is where this
- * cpu belongs to, but it might be a memoryless node anyway. We don't
- * know what the closest node is.
- */
- numa_node = numa_node_id();
-
- /* freed object is from this cpu's node, proceed */
- if (likely(slab_node == numa_node))
- goto check_pfmemalloc;
-
- /*
- * Freed object isn't from this cpu's node, but that node is memoryless
- * or only has ZONE_MOVABLE memory, which slab cannot allocate from.
- * Proceed as it's better to cache remote objects than falling back to
- * the slowpath for everything. The allocation side can never obtain
- * a local object anyway, if none exist. We don't have numa_mem_id() to
- * point to the closest node as we would on a proper memoryless node
- * setup.
- */
- if (unlikely(!node_state(numa_node, N_NORMAL_MEMORY)))
- goto check_pfmemalloc;
-#endif
-
- return false;
-
-check_pfmemalloc:
- return likely(!slab_test_pfmemalloc(slab));
-}
-
/*
* Try to free as many objects (already processed by free hooks) as possible to
* a single per-cpu sheaf.
@@ -6642,7 +6647,7 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
if (unlikely(!slab_free_hook(s, object, free_flags)))
return;

- if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, object, true)))
+ if (likely(free_to_pcs(s, slab, object, free_flags)))
return;

__slab_free(s, slab, object, object, 1, addr);
@@ -6656,7 +6661,6 @@ void memcg_alloc_abort_single(struct kmem_cache *s, void *object,
unsigned int alloc_flags)
{
struct slab *slab = virt_to_slab(object);
- bool allow_spin = alloc_flags_allow_spinning(alloc_flags);
unsigned int free_flags = to_free_flags(alloc_flags);

if (slab_want_init_on_free(s))
@@ -6667,11 +6671,10 @@ void memcg_alloc_abort_single(struct kmem_cache *s, void *object,
if (unlikely(!slab_free_hook(s, object, free_flags)))
return;

- if (likely(can_free_to_pcs(slab)) &&
- likely(free_to_pcs(s, object, allow_spin)))
+ if (likely(free_to_pcs(s, slab, object, free_flags)))
return;

- if (likely(allow_spin))
+ if (likely(free_flags_allow_spinning(free_flags)))
__slab_free(s, slab, object, object, 1, _RET_IP_);
else
defer_free(s, object);
@@ -6991,7 +6994,7 @@ void kfree_nolock(const void *object)
if (unlikely(!slab_free_hook(s, x, free_flags)))
return;

- if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, x, false)))
+ if (likely(free_to_pcs(s, slab, x, free_flags)))
return;

/*

--
2.54.0