[PATCH RFC 2/5] mm/slab, kfence: support kfence objects in kfree_nolock()
From: Vlastimil Babka (SUSE)
Date: Fri Aug 07 2026 - 09:59:16 EST
KFENCE objects are one of the reasons why kfree_nolock() cannot
currently handle kmalloc() objects. They are however rare so we can
simply defer their freeing to irq_work.
The only complication is where to put the llist node. We cannot use the
freepointer location like in defer_free() because for some caches it may
be outside the object area and KFENCE would detect writes there.
Since KFENCE already solves a similar situation when freeing objects
from SLAB_TYPESAFE_BY_RCU caches with an rcu_head in its internal
metadata, reuse that rcu_head also for the llist node. Introduce
kfence_obj_to_llnode() and kfence_llnode_to_obj() so SLAB can work with
this llist node without being exposed to KFENCE internals.
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@xxxxxxxxxx>
---
include/linux/kfence.h | 5 +++++
mm/kfence/core.c | 14 ++++++++++++++
mm/kfence/kfence.h | 5 ++++-
mm/slub.c | 39 ++++++++++++++++++++++++++++++++++++---
4 files changed, 59 insertions(+), 4 deletions(-)
diff --git a/include/linux/kfence.h b/include/linux/kfence.h
index e5822f6e7f27..00721c85258d 100644
--- a/include/linux/kfence.h
+++ b/include/linux/kfence.h
@@ -188,6 +188,9 @@ static __always_inline __must_check bool kfence_free(void *addr)
return true;
}
+struct llist_node *kfence_obj_to_llnode(void *addr);
+void *kfence_llnode_to_obj(struct llist_node *llnode);
+
/**
* kfence_handle_page_fault() - perform page fault handling for KFENCE pages
* @addr: faulting address
@@ -235,6 +238,8 @@ static inline size_t kfence_ksize(const void *addr) { return 0; }
static inline void *kfence_object_start(const void *addr) { return NULL; }
static inline void __kfence_free(void *addr) { }
static inline bool __must_check kfence_free(void *addr) { return false; }
+static inline struct llist_node *kfence_obj_to_llnode(void *addr) { return NULL; }
+static inline void *kfence_llnode_to_obj(struct llist_node *llnode) { return NULL; }
static inline bool __must_check kfence_handle_page_fault(unsigned long addr, bool is_write,
struct pt_regs *regs)
{
diff --git a/mm/kfence/core.c b/mm/kfence/core.c
index 6577bd76954e..42519d24687f 100644
--- a/mm/kfence/core.c
+++ b/mm/kfence/core.c
@@ -1271,6 +1271,20 @@ void __kfence_free(void *addr)
}
}
+struct llist_node *kfence_obj_to_llnode(void *addr)
+{
+ struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);
+
+ return &meta->llnode;
+}
+
+void *kfence_llnode_to_obj(struct llist_node *llnode)
+{
+ struct kfence_metadata *meta = container_of(llnode, struct kfence_metadata, llnode);
+
+ return (void *)meta->addr;
+}
+
bool kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs *regs)
{
const int page_index = (addr - (unsigned long)__kfence_pool) / PAGE_SIZE;
diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h
index 1f618f9b0d12..0fca1dc2c794 100644
--- a/mm/kfence/kfence.h
+++ b/mm/kfence/kfence.h
@@ -58,7 +58,10 @@ struct kfence_track {
/* KFENCE metadata per guarded allocation. */
struct kfence_metadata {
struct list_head list __guarded_by(&kfence_freelist_lock); /* Freelist node. */
- struct rcu_head rcu_head; /* For delayed freeing. */
+ union {
+ struct rcu_head rcu_head; /* For delayed freeing. */
+ struct llist_node llnode; /* For kfree_nolock(). */
+ };
/*
* Lock protecting below data; to ensure consistency of the below data,
diff --git a/mm/slub.c b/mm/slub.c
index 044db93d64a0..2d7648b96bfa 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4050,6 +4050,7 @@ static void flush_all(struct kmem_cache *s)
struct deferred_percpu_work {
struct llist_head objects;
+ struct llist_head objects_kfence;
struct llist_head objects_by_rcu;
struct llist_head rcu_sheaves;
struct irq_work work;
@@ -4059,6 +4060,7 @@ static void deferred_percpu_work_fn(struct irq_work *work);
static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = {
.objects = LLIST_HEAD_INIT(objects),
+ .objects_kfence = LLIST_HEAD_INIT(objects_kfence),
.objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu),
.rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves),
.work = IRQ_WORK_INIT(deferred_percpu_work_fn),
@@ -6399,6 +6401,13 @@ static void deferred_percpu_work_fn(struct irq_work *work)
stat(s, FREE_SLOWPATH);
}
+ llnode = llist_del_all(&dpw->objects_kfence);
+ llist_for_each_safe(pos, t, llnode) {
+ void *obj = kfence_llnode_to_obj(pos);
+
+ __kfence_free(obj);
+ }
+
llnode = llist_del_all(&dpw->objects_by_rcu);
llist_for_each_safe(pos, t, llnode) {
void *head = pos;
@@ -6431,6 +6440,21 @@ static void defer_free(struct kmem_cache *s, void *obj)
irq_work_queue(&dpw->work);
}
+static void defer_free_kfence(void *obj)
+{
+ struct deferred_percpu_work *dpw;
+ struct llist_node *llnode;
+
+ /* kasan_reset_tag() is not necessary, kfence objects are not tagged */
+ llnode = kfence_obj_to_llnode(obj);
+
+ guard(preempt)();
+
+ dpw = this_cpu_ptr(&deferred_percpu_work);
+ if (llist_add(llnode, &dpw->objects_kfence))
+ irq_work_queue(&dpw->work);
+}
+
void defer_kfree_rcu(struct kvfree_rcu_head *head)
{
struct deferred_percpu_work *dpw;
@@ -6758,10 +6782,13 @@ EXPORT_SYMBOL(kfree);
/*
* Can be called while holding raw_spinlock_t or from IRQ and NMI,
* but ONLY for objects allocated by kmalloc_nolock().
- * Debug checks (like kmemleak and kfence) were skipped on allocation,
- * hence
+ *
+ * In case kmemleak is enabled,
+ *
* obj = kmalloc(); kfree_nolock(obj);
- * will miss kmemleak/kfence book keeping and will cause false positives.
+ *
+ * will miss kmemleak book keeping and will cause false positives.
+ *
* large_kmalloc is not supported either.
*/
void kfree_nolock(const void *object)
@@ -6793,6 +6820,12 @@ void kfree_nolock(const void *object)
* since they take spinlocks or not safe from any context.
*/
kmsan_slab_free(s, x);
+
+ if (is_kfence_address(x)) {
+ defer_free_kfence(x);
+ return;
+ }
+
/*
* If KASAN finds a kernel bug it will do kasan_report_invalid_free()
* which will call raw_spin_lock_irqsave() which is technically
--
2.55.0