Re: [PATCH v1] slab: support for compiler-assisted type-based slab cache partitioning
From: Harry Yoo (Oracle)
Date: Mon Apr 06 2026 - 00:28:32 EST
On Fri, Apr 03, 2026 at 08:29:22PM +0200, Vlastimil Babka (SUSE) wrote:
> On 4/3/26 08:27, Harry Yoo (Oracle) wrote:
> >> diff --git a/include/linux/slab.h b/include/linux/slab.h
> >> index 15a60b501b95..c0bf00ee6025 100644
> >> --- a/include/linux/slab.h
> >> +++ b/include/linux/slab.h
> >> @@ -864,10 +877,10 @@ unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf);
> >> * with the exception of kunit tests
> >> */
> >>
> >> -void *__kmalloc_noprof(size_t size, gfp_t flags)
> >> +void *__kmalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token)
> >> __assume_kmalloc_alignment __alloc_size(1);
> >>
> >> -void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
> >> +void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node, kmalloc_token_t token)
> >> __assume_kmalloc_alignment __alloc_size(1);
> >
> > So the @token parameter is unused when CONFIG_PARTITION_KMALLOC_CACHES is
> > disabled but still increases the kernel size by a few kilobytes...
> > but yeah I'm not sure if we can get avoid it without hurting readability.
> >
> > Just saying. (does anybody care?)
>
> Well we did care enough with CONFIG_SLAB_BUCKETS to hide the unused param
> using DECL_BUCKET_PARAMS(),
Hmm yeah.
I wasn't sure if we could do this without hurting readability,
but perhaps we could...
> so maybe extend that idea?
> I think it's not just kernel size, but increased register pressure etc.
Something like this should work? (diff on top of this patch)
--
Cheers,
Harry / Hyeonggon
diff --git a/include/linux/slab.h b/include/linux/slab.h
index c0bf00ee6025..0496d2e63f5e 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -871,16 +871,32 @@ unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf);
#define PASS_BUCKET_PARAM(_b) NULL
#endif
+#ifdef CONFIG_PARTITION_KMALLOC_CACHES
+#define DECL_TOKEN_PARAM(_token) , kmalloc_token_t (_token)
+#define _PASS_TOKEN_PARAM(_token) , (_token)
+#define PASS_TOKEN_PARAM(_token) (_token)
+#else
+#define DECL_TOKEN_PARAM(_token)
+#define _PASS_TOKEN_PARAM(_token)
+#define PASS_TOKEN_PARAM(_token) ((kmalloc_token_t){})
+#endif
+
+#define DECL_KMALLOC_PARAMS(_size, _b, _token) DECL_BUCKET_PARAMS(_size, _b) \
+ DECL_TOKEN_PARAM(_token)
+
+#define PASS_KMALLOC_PARAMS(_size, _b, _token) PASS_BUCKET_PARAMS(_size, _b) \
+ _PASS_TOKEN_PARAM(_token)
+
/*
* The following functions are not to be used directly and are intended only
* for internal use from kmalloc() and kmalloc_node()
* with the exception of kunit tests
*/
-void *__kmalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token)
+void *__kmalloc_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags)
__assume_kmalloc_alignment __alloc_size(1);
-void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node, kmalloc_token_t token)
+void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node)
__assume_kmalloc_alignment __alloc_size(1);
void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size)
@@ -964,7 +980,7 @@ static __always_inline __alloc_size(1) void *_kmalloc_noprof(size_t size, gfp_t
kmalloc_caches[kmalloc_type(flags, token)][index],
flags, size);
}
- return __kmalloc_noprof(size, flags, token);
+ return __kmalloc_noprof(PASS_KMALLOC_PARAMS(size, NULL, token), flags);
}
#define kmalloc_noprof(...) _kmalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
#define kmalloc(...) alloc_hooks(kmalloc_noprof(__VA_ARGS__))
@@ -1075,10 +1091,10 @@ void *_kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node, kmalloc_tok
__alloc_flex(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT)
#define kmem_buckets_alloc(_b, _size, _flags) \
- alloc_hooks(__kmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE, __kmalloc_token(_size)))
+ alloc_hooks(__kmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE))
#define kmem_buckets_alloc_track_caller(_b, _size, _flags) \
- alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE, _RET_IP_, __kmalloc_token(_size)))
+ alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE, _RET_IP_))
static __always_inline __alloc_size(1) void *_kmalloc_node_noprof(size_t size, gfp_t flags, int node, kmalloc_token_t token)
{
@@ -1093,7 +1109,7 @@ static __always_inline __alloc_size(1) void *_kmalloc_node_noprof(size_t size, g
kmalloc_caches[kmalloc_type(flags, token)][index],
flags, node, size);
}
- return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node, token);
+ return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(size, NULL, token), flags, node);
}
#define kmalloc_node_noprof(...) _kmalloc_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
#define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__))
@@ -1154,10 +1170,10 @@ static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(voi
*/
#define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
-void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node,
- unsigned long caller, kmalloc_token_t token) __alloc_size(1);
+void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node,
+ unsigned long caller) __alloc_size(1);
#define kmalloc_node_track_caller_noprof(size, flags, node, caller) \
- __kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node, caller, __kmalloc_token(size))
+ __kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(size, NULL, __kmalloc_token(size)), flags, node, caller)
#define kmalloc_node_track_caller(...) \
alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_))
@@ -1183,7 +1199,7 @@ static inline __alloc_size(1, 2) void *_kmalloc_array_node_noprof(size_t n, size
return NULL;
if (__builtin_constant_p(n) && __builtin_constant_p(size))
return _kmalloc_node_noprof(bytes, flags, node, token);
- return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(bytes, NULL), flags, node, token);
+ return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), flags, node);
}
#define kmalloc_array_node_noprof(...) _kmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
#define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__))
@@ -1209,10 +1225,10 @@ static inline __alloc_size(1) void *_kzalloc_noprof(size_t size, gfp_t flags, km
#define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__))
#define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
-void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align,
- gfp_t flags, int node, kmalloc_token_t token) __alloc_size(1);
+void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), unsigned long align,
+ gfp_t flags, int node) __alloc_size(1);
#define kvmalloc_node_align_noprof(_size, _align, _flags, _node) \
- __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, NULL), _align, _flags, _node, __kmalloc_token(_size))
+ __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, NULL, __kmalloc_token(_size)), _align, _flags, _node)
#define kvmalloc_node_align(...) \
alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__))
#define kvmalloc_node(_s, _f, _n) kvmalloc_node_align(_s, 1, _f, _n)
@@ -1225,7 +1241,7 @@ void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align,
#define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
#define kmem_buckets_valloc(_b, _size, _flags) \
- alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), 1, _flags, NUMA_NO_NODE, __kmalloc_token(_size)))
+ alloc_hooks(__kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), 1, _flags, NUMA_NO_NODE))
static inline __alloc_size(1, 2) void *
_kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node, kmalloc_token_t token)
@@ -1235,7 +1251,7 @@ _kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node, kmallo
if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL;
- return __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(bytes, NULL), 1, flags, node, token);
+ return __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), 1, flags, node);
}
#define kvmalloc_array_node_noprof(...) _kvmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
#define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
diff --git a/mm/slub.c b/mm/slub.c
index 5630dde94df0..84f129d79c99 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -5293,15 +5293,17 @@ void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node,
trace_kmalloc(caller, ret, size, s->size, flags, node);
return ret;
}
-void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node, kmalloc_token_t token)
+void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node)
{
- return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, _RET_IP_, token);
+ return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node,
+ _RET_IP_, PASS_TOKEN_PARAM(token));
}
EXPORT_SYMBOL(__kmalloc_node_noprof);
-void *__kmalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token)
+void *__kmalloc_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags)
{
- return __do_kmalloc_node(size, NULL, flags, NUMA_NO_NODE, _RET_IP_, token);
+ return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags,
+ NUMA_NO_NODE, _RET_IP_, PASS_TOKEN_PARAM(token));
}
EXPORT_SYMBOL(__kmalloc_noprof);
@@ -5394,10 +5396,11 @@ void *_kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node, kmalloc_tok
}
EXPORT_SYMBOL_GPL(_kmalloc_nolock_noprof);
-void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags,
- int node, unsigned long caller, kmalloc_token_t token)
+void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags,
+ int node, unsigned long caller)
{
- return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, caller, token);
+ return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node,
+ caller, PASS_TOKEN_PARAM(token));
}
EXPORT_SYMBOL(__kmalloc_node_track_caller_noprof);
@@ -6812,8 +6815,8 @@ static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size)
*
* Return: pointer to the allocated memory of %NULL in case of failure
*/
-void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align,
- gfp_t flags, int node, kmalloc_token_t token)
+void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token),
+ unsigned long align, gfp_t flags, int node)
{
bool allow_block;
void *ret;
@@ -6824,7 +6827,7 @@ void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align,
*/
ret = __do_kmalloc_node(size, PASS_BUCKET_PARAM(b),
kmalloc_gfp_adjust(flags, size),
- node, _RET_IP_, token);
+ node, _RET_IP_, PASS_TOKEN_PARAM(token));
if (ret || size <= PAGE_SIZE)
return ret;
--
2.43.0