[PATCH 3/9] bpf/verifier: Route verifier stack state node allocations to folio_pool

[Jim Cromie via B4 Relay]

From: Jim Cromie <jim.cromie@xxxxxxxxx>

Embed a struct folio_pool inside struct bpf_verifier_env to allocate
transient bpf_verifier_stack_elem frames using direct-map large folios,
releasing all frames in bulk at the end of bpf_check().

Use folio_pool_alloc_obj() to preserve strict type specificity at callsites.

Signed-off-by: Jim Cromie <jim.cromie@xxxxxxxxx>
---
 include/linux/bpf_verifier.h |  3 +++
 kernel/bpf/verifier.c        | 16 +++++++++++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 39a851e690ec..43f5f0eaffac 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -898,6 +898,8 @@ struct bpf_scc_info {
 
 struct bpf_liveness;
 
+#include <linux/folio_pool.h>
+
 /* single container for all structs
  * one verifier_env per bpf_check() call
  */
@@ -907,6 +909,7 @@ struct bpf_verifier_env {
 	struct bpf_prog *prog;		/* eBPF program being verified */
 	const struct bpf_verifier_ops *ops;
 	struct module *attach_btf_mod;	/* The owner module of prog->aux->attach_btf */
+	struct folio_pool state_pool;	/* pool for transient state nodes */
 	struct bpf_verifier_stack_elem *head; /* stack of verifier states to be processed */
 	int stack_size;			/* number of states to be processed */
 	bool strict_alignment;		/* perform strict pointer alignment checks */
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index fdc5fbb1f78c..8a9e66ce4dc8 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1725,7 +1725,7 @@ static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
 		*prev_insn_idx = head->prev_insn_idx;
 	elem = head->next;
 	bpf_free_verifier_state(&head->st, false);
-	kfree(head);
+	folio_pool_free_elem(head);
 	env->head = elem;
 	env->stack_size--;
 	return 0;
@@ -1743,6 +1743,8 @@ static bool error_recoverable_with_nospec(int err)
 	return err == -EPERM || err == -EACCES || err == -EINVAL;
 }
 
+DEFINE_STATIC_KEY_TRUE(bpf_state_pool_key);
+
 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
 					     int insn_idx, int prev_insn_idx,
 					     bool speculative)
@@ -1751,7 +1753,9 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
 	struct bpf_verifier_stack_elem *elem;
 	int err;
 
-	elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT);
+	elem = folio_pool_alloc_obj(env, state_pool,
+				    struct bpf_verifier_stack_elem,
+				    GFP_KERNEL_ACCOUNT);
 	if (!elem)
 		return ERR_PTR(-ENOMEM);
 
@@ -2275,7 +2279,9 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
 	struct bpf_verifier_stack_elem *elem;
 	struct bpf_func_state *frame;
 
-	elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT);
+	elem = folio_pool_alloc_obj(env, state_pool,
+				    struct bpf_verifier_stack_elem,
+				    GFP_KERNEL_ACCOUNT);
 	if (!elem)
 		return ERR_PTR(-ENOMEM);
 
@@ -19789,6 +19795,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (!env)
 		return -ENOMEM;
 
+	folio_pool_init_key(&env->state_pool, sizeof(struct bpf_verifier_stack_elem),
+			    get_order(SZ_64K), &bpf_state_pool_key);
+
 	env->bt.env = env;
 
 	len = (*prog)->len;
@@ -20055,6 +20064,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	bpf_clear_insn_aux_data(env, 0, env->prog->len);
 err_free_env:
 	bpf_stack_liveness_free(env);
+	folio_pool_free(&env->state_pool);
 	kvfree(env->cfg.insn_postorder);
 	kvfree(env->scc_info);
 	kvfree(env->succ);

-- 
2.55.0