[PATCH RFC 1/5] mm/slab: cleanup deferred free handling

[Vlastimil Babka (SUSE)]

In deferred_percpu_work_fn() we have a bunch of single-use local
variables for the various llists. Remove them and access the lists
directly.

In defer_free() make it more obvious and documented what we are doing.
Also restrict guard(preempt) to only the necessary part.

Signed-off-by: Vlastimil Babka (SUSE) <vbabka@xxxxxxxxxx>
---
 mm/slub.c | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index b9aeb02a880f..044db93d64a0 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -6371,16 +6371,12 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
 static void deferred_percpu_work_fn(struct irq_work *work)
 {
 	struct deferred_percpu_work *dpw;
-	struct llist_head *objs, *objs_by_rcu, *rcu_sheaves;
 	struct llist_node *llnode, *pos, *t;
 	struct slab_sheaf *sheaf, *next;
 
 	dpw = container_of(work, struct deferred_percpu_work, work);
-	rcu_sheaves = &dpw->rcu_sheaves;
-	objs = &dpw->objects;
-	objs_by_rcu = &dpw->objects_by_rcu;
 
-	llnode = llist_del_all(objs);
+	llnode = llist_del_all(&dpw->objects);
 	llist_for_each_safe(pos, t, llnode) {
 		struct kmem_cache *s;
 		struct slab *slab;
@@ -6403,7 +6399,7 @@ static void deferred_percpu_work_fn(struct irq_work *work)
 		stat(s, FREE_SLOWPATH);
 	}
 
-	llnode = llist_del_all(objs_by_rcu);
+	llnode = llist_del_all(&dpw->objects_by_rcu);
 	llist_for_each_safe(pos, t, llnode) {
 		void *head = pos;
 		void *objp = kvmalloc_obj_start_addr(head);
@@ -6411,21 +6407,27 @@ static void deferred_percpu_work_fn(struct irq_work *work)
 		kvfree_call_rcu(head, objp);
 	}
 
-	llnode = llist_del_all(rcu_sheaves);
+	llnode = llist_del_all(&dpw->rcu_sheaves);
 	llist_for_each_entry_safe(sheaf, next, llnode, llnode)
 		call_rcu(&sheaf->rcu_head, rcu_free_sheaf);
 }
 
-static void defer_free(struct kmem_cache *s, void *head)
+static void defer_free(struct kmem_cache *s, void *obj)
 {
 	struct deferred_percpu_work *dpw;
+	struct llist_node *llnode;
 
-	guard(preempt)();
+	/*
+	 * Place the llist node where the freepointer would be if we freed the
+	 * object immediately. That means we can write there safely, only need
+	 * to remove kasan tag first.
+	 */
+	llnode = kasan_reset_tag(obj) + s->offset;
 
-	head = kasan_reset_tag(head);
+	guard(preempt)();
 
 	dpw = this_cpu_ptr(&deferred_percpu_work);
-	if (llist_add(head + s->offset, &dpw->objects))
+	if (llist_add(llnode, &dpw->objects))
 		irq_work_queue(&dpw->work);
 }
 

-- 
2.55.0