[PATCH RFC 07/12] mm/slab: replace slab.stride with obj_exts_in_object
From: Vlastimil Babka (SUSE)
Date: Wed Jul 15 2026 - 06:13:35 EST
The stride field is used to convert object index to an slabobj_ext so
both compact arrays (kmalloc() or in-slab-leftover) and spread
in-object-padding obj_ext layouts are supported.
In practice thus the stride is always sizeof(slabobj_ext) or s->size.
This simplifies the calculations, but with the upcoming slabobj_ext
handling changes, it will be easier to stop storing the stride and
instead just have a flag whether obj_ext is in the object padding.
obj_exts_in_object() can then rely on this flag and slab_obj_ext()
can use that to determine the stride.
No functional change intended. Performance impact TBD, hopefully
in the noise.
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@xxxxxxxxxx>
---
mm/slab.h | 43 ++++++++++++++++++++++++-------------------
mm/slub.c | 42 ++++++++++++++++--------------------------
2 files changed, 40 insertions(+), 45 deletions(-)
diff --git a/mm/slab.h b/mm/slab.h
index e3f8e42070f1..3ad9777ad600 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -81,10 +81,10 @@ struct freelist_counters {
#ifdef CONFIG_64BIT
/*
* Some optimizations use free bits in 'counters' field
- * to save memory. In case ->stride field is not available,
+ * to save memory. If these free bits are not available,
* such optimizations are disabled.
*/
- unsigned int stride;
+ unsigned obj_exts_in_object:1;
#endif
};
};
@@ -617,22 +617,20 @@ static inline void put_slab_obj_exts(unsigned long obj_exts)
}
#ifdef CONFIG_64BIT
-static inline void slab_set_stride(struct slab *slab, unsigned int stride)
+static inline bool obj_exts_in_object(struct slab *slab)
{
- slab->stride = stride;
-}
-static inline unsigned int slab_get_stride(struct slab *slab)
-{
- return slab->stride;
+ /*
+ * Note we cannot rely on the SLAB_OBJ_EXT_IN_OBJ flag here and need to
+ * check the per-slab bit. A cache can have SLAB_OBJ_EXT_IN_OBJ set, but
+ * allocations within_slab_leftover are preferred. And those may be
+ * possible or not depending on the particular slab's size.
+ */
+ return slab->obj_exts_in_object;
}
#else
-static inline void slab_set_stride(struct slab *slab, unsigned int stride)
+static inline bool obj_exts_in_object(struct slab *slab)
{
- VM_WARN_ON_ONCE(stride != sizeof(struct slabobj_ext));
-}
-static inline unsigned int slab_get_stride(struct slab *slab)
-{
- return sizeof(struct slabobj_ext);
+ return false;
}
#endif
@@ -656,8 +654,14 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts,
VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab));
index = obj_to_index(s, slab, obj);
- obj_ext = (struct slabobj_ext *)(obj_exts +
- slab_get_stride(slab) * index);
+
+ if (!obj_exts_in_object(slab)) {
+ obj_ext = ((struct slabobj_ext *)obj_exts) + index;
+ } else {
+ unsigned int stride = s->size;
+ obj_ext = (struct slabobj_ext *)(obj_exts + index * stride);
+ }
+
return kasan_reset_tag(obj_ext);
}
@@ -693,9 +697,10 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts,
return NULL;
}
-static inline void slab_set_stride(struct slab *slab, unsigned int stride) { }
-static inline unsigned int slab_get_stride(struct slab *slab) { return 0; }
-
+static inline bool obj_exts_in_object(struct slab *slab)
+{
+ return false;
+}
#endif /* CONFIG_SLAB_OBJ_EXT */
diff --git a/mm/slub.c b/mm/slub.c
index 2bfcabc4c51a..98a14e5842a2 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -870,18 +870,6 @@ static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab)
#endif
#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
-static bool obj_exts_in_object(struct kmem_cache *s, struct slab *slab)
-{
- /*
- * Note we cannot rely on the SLAB_OBJ_EXT_IN_OBJ flag here and need to
- * check the stride. A cache can have SLAB_OBJ_EXT_IN_OBJ set, but
- * allocations within_slab_leftover are preferred. And those may be
- * possible or not depending on the particular slab's size.
- */
- return obj_exts_in_slab(s, slab) &&
- (slab_get_stride(slab) == s->size);
-}
-
static unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
{
unsigned int offset = get_info_end(s);
@@ -896,16 +884,20 @@ static unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
return offset;
}
-#else
-static inline bool obj_exts_in_object(struct kmem_cache *s, struct slab *slab)
+
+static inline void slab_set_obj_exts_in_object(struct slab *slab)
{
- return false;
+ slab->obj_exts_in_object = 1;
}
-
+#else
static inline unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
{
return 0;
}
+
+static inline void slab_set_obj_exts_in_object(struct slab *slab)
+{
+}
#endif
#ifdef CONFIG_SLUB_DEBUG
@@ -1206,7 +1198,7 @@ static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
off += kasan_metadata_size(s, false);
- if (obj_exts_in_object(s, slab))
+ if (obj_exts_in_object(slab))
off += sizeof(struct slabobj_ext);
if (off != size_from_object(s))
@@ -1411,7 +1403,7 @@ static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
off += kasan_metadata_size(s, false);
- if (obj_exts_in_object(s, slab))
+ if (obj_exts_in_object(slab))
off += sizeof(struct slabobj_ext);
if (size_from_object(s) == off)
@@ -1439,7 +1431,7 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab)
length = slab_size(slab);
end = start + length;
- if (obj_exts_in_slab(s, slab) && !obj_exts_in_object(s, slab)) {
+ if (obj_exts_in_slab(s, slab) && !obj_exts_in_object(slab)) {
remainder = length;
remainder -= obj_exts_offset_in_slab(s, slab);
remainder -= obj_exts_size_in_slab(slab);
@@ -2253,9 +2245,6 @@ static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab)
void *addr;
unsigned long obj_exts;
- /* Initialize stride early to avoid memory ordering issues */
- slab_set_stride(slab, sizeof(struct slabobj_ext));
-
if (!need_slab_obj_exts(s))
return;
@@ -2289,7 +2278,7 @@ static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab)
obj_exts |= MEMCG_DATA_OBJEXTS;
#endif
slab->obj_exts = obj_exts;
- slab_set_stride(slab, s->size);
+ slab_set_obj_exts_in_object(slab);
}
}
@@ -3402,9 +3391,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags,
stat(s, ORDER_FALLBACK);
}
+ /* Initializes frozen, inuse, and any extra 64bit-only flags */
+ slab->counters = 0;
+
slab->objects = oo_objects(oo);
- slab->inuse = 0;
- slab->frozen = 0;
slab->slab_cache = s;
@@ -6537,7 +6527,7 @@ static inline size_t slab_ksize(struct slab *slab)
*/
if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))
return s->inuse;
- else if (obj_exts_in_object(s, slab))
+ else if (obj_exts_in_object(slab))
return s->inuse;
/*
* Else we can use all the padding etc for the allocation
--
2.55.0