[RFC PATCH v3 14/25] net: Use per-node PISA allocators for skb_page_frag_refill()
From: Luigi Rizzo
Date: Thu Nov 19 2026 - 20:09:56 EST
Transmit socket buffers use memory coming from skb_page_frag_refill().
Sockets whose leaf device has enabled pisa_tx_bufs should use PISA
memory.
Add a dedicated per-task page_frag, current->task_frag_pisa, populated
with read-only PISA blocks via __GFP_PISA, and select it on TX when
the socket's (or cork's) route goes out in the clear through a PISA-enabled
device.
Special paths that hold plaintext before encryption (IPsec, kTLS) or
do not use a DMA device keep using regular pages; ESP output uses
pisa_tx_bufs_gfp() to opt in when the egress device has pisa_tx_bufs
enabled.
Signed-off-by: Luigi Rizzo <lrizzo@xxxxxxxxxx>
---
include/linux/pisa.h | 15 +++++++++++++++
include/linux/sched.h | 4 ++++
include/net/pisa.h | 33 +++++++++++++++++++++++++++++++++
kernel/exit.c | 2 ++
kernel/fork.c | 2 ++
net/core/sock.c | 10 ++++++++++
net/ipv4/esp4.c | 7 +++++--
net/ipv4/ip_output.c | 12 ++++++++----
net/ipv4/tcp.c | 3 ++-
net/ipv6/esp6.c | 7 +++++--
net/ipv6/ip6_output.c | 12 ++++++++----
11 files changed, 94 insertions(+), 13 deletions(-)
diff --git a/include/linux/pisa.h b/include/linux/pisa.h
index 6d184e393e297..84692852299f1 100644
--- a/include/linux/pisa.h
+++ b/include/linux/pisa.h
@@ -202,6 +202,18 @@ static inline bool pisa_page_can_recycle(const struct page *page)
__pisa_page_can_recycle(page);
}
+/* Hooks for dup_task_struct() and do_exit() (guarded by @pisa_key on release). */
+static inline void pisa_task_frag_init(struct task_struct *tsk)
+{
+ tsk->task_frag_pisa.page = NULL;
+}
+
+static inline void pisa_task_frag_release(struct task_struct *tsk)
+{
+ if (pisa_enabled() && tsk->task_frag_pisa.page)
+ put_page(tsk->task_frag_pisa.page);
+}
+
#else /* !CONFIG_PISA */
static inline int pisa_meta_init(void) { return -EOPNOTSUPP; }
@@ -240,6 +252,9 @@ static inline bool pisa_page_can_recycle(const struct page *page)
return true;
}
+static inline void pisa_task_frag_init(struct task_struct *tsk) {}
+static inline void pisa_task_frag_release(struct task_struct *tsk) {}
+
static inline bool pisa_upgrade_user_pages(struct iov_iter *iter,
struct page **pages, size_t size,
size_t offset,
diff --git a/include/linux/sched.h b/include/linux/sched.h
index e4f6323d78555..a8edeeb2661f4 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1455,6 +1455,10 @@ struct task_struct {
struct pipe_inode_info *splice_pipe;
struct page_frag task_frag;
+#ifdef CONFIG_PISA
+ /* TX frag that may hold PISA pages, see sk_page_frag_dst(). */
+ struct page_frag task_frag_pisa;
+#endif
#ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE
struct lazy_mmu_state lazy_mmu_state;
diff --git a/include/net/pisa.h b/include/net/pisa.h
index 1f95e67e99bf8..8e23d8b01e73d 100644
--- a/include/net/pisa.h
+++ b/include/net/pisa.h
@@ -4,6 +4,7 @@
#define _NET_PISA_H
#include <linux/pisa.h>
+#include <linux/tcp.h>
#include <net/sock.h>
/*
@@ -72,10 +73,42 @@ static inline gfp_t pisa_tx_bufs_gfp(const struct sk_buff *skb)
__pisa_tx_gfp(skb_dst(skb), PISA_TX_FLAG_BUFS | PISA_TX_XFRM_OK) : 0;
}
+/* Hook for skb_page_frag_refill() and __ip*_append_data() (@net_pisa_tx_bufs_key). */
+static inline bool is_pisa_pfrag(const struct page_frag *pfrag)
+{
+ return static_branch_unlikely(&net_pisa_tx_bufs_key) &&
+ pfrag == ¤t->task_frag_pisa;
+}
+
+/* Hook for task/socket TX page_frag selection (@net_pisa_tx_bufs_key). */
+static inline struct page_frag *task_page_frag(const struct dst_entry *dst)
+{
+ return use_pisa_tx_bufs(dst) ? ¤t->task_frag_pisa : ¤t->task_frag;
+}
+
+static inline struct page_frag *sk_page_frag_dst(struct sock *sk,
+ const struct dst_entry *dst)
+{
+ return static_branch_unlikely(&net_pisa_tx_bufs_key) &&
+ sk->sk_use_task_frag ? task_page_frag(dst) : sk_page_frag(sk);
+}
+
#else /* !CONFIG_PISA */
static inline bool use_pisa_tx_bufs(const struct dst_entry *dst) { return false; }
static inline gfp_t pisa_tx_bufs_gfp(const struct sk_buff *skb) { return 0; }
+static inline bool is_pisa_pfrag(const struct page_frag *pfrag) { return false; }
+
+static inline struct page_frag *task_page_frag(const struct dst_entry *dst)
+{
+ return ¤t->task_frag;
+}
+
+static inline struct page_frag *sk_page_frag_dst(struct sock *sk,
+ const struct dst_entry *dst)
+{
+ return sk_page_frag(sk);
+}
#endif /* CONFIG_PISA */
#endif /* _NET_PISA_H */
diff --git a/kernel/exit.c b/kernel/exit.c
index d5e7bc2af804a..ac7ca1fb180c3 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -6,6 +6,7 @@
*/
#include <linux/mm.h>
+#include <linux/pisa.h>
#include <linux/slab.h>
#include <linux/sched/autogroup.h>
#include <linux/sched/mm.h>
@@ -1009,6 +1010,7 @@ void __noreturn do_exit(long code)
if (tsk->task_frag.page)
put_page(tsk->task_frag.page);
+ pisa_task_frag_release(tsk);
exit_task_stack_account(tsk);
diff --git a/kernel/fork.c b/kernel/fork.c
index 5efceb8d58a78..d235d7ab04687 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -44,6 +44,7 @@
#include <linux/mmu_notifier.h>
#include <linux/fs.h>
#include <linux/mm.h>
+#include <linux/pisa.h>
#include <linux/mm_inline.h>
#include <linux/memblock.h>
#include <linux/nsproxy.h>
@@ -982,6 +983,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#endif
tsk->splice_pipe = NULL;
tsk->task_frag.page = NULL;
+ pisa_task_frag_init(tsk);
tsk->wake_q.next = NULL;
tsk->worker_private = NULL;
diff --git a/net/core/sock.c b/net/core/sock.c
index ad414878dc5ba..56dea49baaec2 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -115,6 +115,7 @@
#include <linux/memcontrol.h>
#include <linux/prefetch.h>
#include <linux/compat.h>
+#include <linux/pisa.h>
#include <linux/mroute.h>
#include <linux/mroute6.h>
#include <linux/icmpv6.h>
@@ -147,6 +148,7 @@
#include <net/tcp.h>
#include <net/busy_poll.h>
#include <net/phonet/phonet.h>
+#include <net/pisa.h>
#include <linux/ethtool.h>
@@ -3257,13 +3259,21 @@ DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
*/
bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
{
+ /* PISA pages only go into current->task_frag_pisa, see sk_page_frag_dst(). */
+ if (is_pisa_pfrag(pfrag))
+ gfp |= __GFP_PISA;
+
if (pfrag->page) {
if (page_ref_count(pfrag->page) == 1) {
+ if (!!(gfp & __GFP_PISA) != is_pisa_page(pfrag->page) ||
+ ((gfp & __GFP_PISA) && !pisa_page_can_recycle(pfrag->page)))
+ goto drop_page;
pfrag->offset = 0;
return true;
}
if (pfrag->offset + sz <= pfrag->size)
return true;
+drop_page:
put_page(pfrag->page);
}
diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c
index ae701561d9f12..8ed713772e4ee 100644
--- a/net/ipv4/esp4.c
+++ b/net/ipv4/esp4.c
@@ -20,6 +20,7 @@
#include <net/udp.h>
#include <net/tcp.h>
#include <net/espintcp.h>
+#include <net/pisa.h>
#include <linux/skbuff_ref.h>
#include <linux/highmem.h>
@@ -451,7 +452,8 @@ int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *
spin_lock_bh(&x->lock);
- if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
+ if (unlikely(!skb_page_frag_refill(allocsize, pfrag,
+ GFP_ATOMIC | pisa_tx_bufs_gfp(skb)))) {
spin_unlock_bh(&x->lock);
goto cow;
}
@@ -560,7 +562,8 @@ int esp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *
allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES);
spin_lock_bh(&x->lock);
- if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
+ if (unlikely(!skb_page_frag_refill(allocsize, pfrag,
+ GFP_ATOMIC | pisa_tx_bufs_gfp(skb)))) {
spin_unlock_bh(&x->lock);
goto error_free;
}
diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index d42d776af273e..50b8c92c3cd5e 100644
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -71,6 +71,7 @@
#include <net/xfrm.h>
#include <linux/skbuff.h>
#include <net/sock.h>
+#include <net/pisa.h>
#include <net/arp.h>
#include <net/icmp.h>
#include <net/checksum.h>
@@ -986,7 +987,10 @@ static int __ip_append_data(struct sock *sk,
exthdrlen = !skb ? rt->dst.header_len : 0;
mtu = cork->gso_size ? IP_MAX_MTU : cork->fragsize;
- paged = !!cork->gso_size;
+ paged = !!cork->gso_size ||
+ (getfrag == ip_generic_getfrag &&
+ (rt->dst.dev->features & NETIF_F_SG) &&
+ is_pisa_pfrag(pfrag));
hh_len = LL_RESERVED_SPACE(rt->dst.dev);
@@ -1383,8 +1387,8 @@ int ip_append_data(struct sock *sk, struct flowi4 *fl4,
}
return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base,
- sk_page_frag(sk), getfrag,
- from, length, transhdrlen, flags);
+ sk_page_frag_dst(sk, inet->cork.base.dst),
+ getfrag, from, length, transhdrlen, flags);
}
static void ip_cork_release(struct inet_cork *cork)
@@ -1592,7 +1596,7 @@ struct sk_buff *ip_make_skb(struct sock *sk,
return ERR_PTR(err);
err = __ip_append_data(sk, fl4, &queue, cork,
- ¤t->task_frag, getfrag,
+ sk_page_frag_dst(sk, cork->dst), getfrag,
from, length, transhdrlen, flags);
if (err) {
__ip_flush_pending_frames(sk, &queue, cork);
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 832b25ddd3c2e..3c3edc48705d7 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -273,6 +273,7 @@
#include <net/inet_common.h>
#include <net/inet_ecn.h>
#include <net/tcp.h>
+#include <net/pisa.h>
#include <net/tcp_ecn.h>
#include <net/mptcp.h>
#include <net/proto_memory.h>
@@ -1295,7 +1296,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
if (zc == 0) {
bool merge = true;
int i = skb_shinfo(skb)->nr_frags;
- struct page_frag *pfrag = sk_page_frag(sk);
+ struct page_frag *pfrag = sk_page_frag_dst(sk, __sk_dst_get(sk));
if (!sk_page_frag_refill(sk, pfrag))
goto wait_for_space;
diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c
index d654f0f442388..98ed9120ae7b4 100644
--- a/net/ipv6/esp6.c
+++ b/net/ipv6/esp6.c
@@ -36,6 +36,7 @@
#include <net/tcp.h>
#include <net/espintcp.h>
#include <net/inet6_hashtables.h>
+#include <net/pisa.h>
#include <linux/skbuff_ref.h>
#include <linux/highmem.h>
@@ -481,7 +482,8 @@ int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info
spin_lock_bh(&x->lock);
- if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
+ if (unlikely(!skb_page_frag_refill(allocsize, pfrag,
+ GFP_ATOMIC | pisa_tx_bufs_gfp(skb)))) {
spin_unlock_bh(&x->lock);
goto cow;
}
@@ -592,7 +594,8 @@ int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info
allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES);
spin_lock_bh(&x->lock);
- if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
+ if (unlikely(!skb_page_frag_refill(allocsize, pfrag,
+ GFP_ATOMIC | pisa_tx_bufs_gfp(skb)))) {
spin_unlock_bh(&x->lock);
goto error_free;
}
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index cb648c8394310..8f6f0bed15499 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -40,6 +40,7 @@
#include <linux/netfilter_ipv6.h>
#include <net/sock.h>
+#include <net/pisa.h>
#include <net/snmp.h>
#include <net/gso.h>
@@ -1489,7 +1490,10 @@ static int __ip6_append_data(struct sock *sk,
dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
}
- paged = !!cork->gso_size;
+ paged = !!cork->gso_size ||
+ (getfrag == ip_generic_getfrag &&
+ (rt->dst.dev->features & NETIF_F_SG) &&
+ is_pisa_pfrag(pfrag));
mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize;
orig_mtu = mtu;
@@ -1892,8 +1896,8 @@ int ip6_append_data(struct sock *sk,
}
return __ip6_append_data(sk, &sk->sk_write_queue, &inet->cork,
- sk_page_frag(sk), getfrag,
- from, length, transhdrlen, flags);
+ sk_page_frag_dst(sk, inet->cork.base.dst),
+ getfrag, from, length, transhdrlen, flags);
}
EXPORT_SYMBOL_GPL(ip6_append_data);
@@ -2095,7 +2099,7 @@ struct sk_buff *ip6_make_skb(struct sock *sk,
}
err = __ip6_append_data(sk, &queue, cork,
- ¤t->task_frag, getfrag, from,
+ sk_page_frag_dst(sk, cork->base.dst), getfrag, from,
length + exthdrlen, transhdrlen + exthdrlen,
flags);
if (err) {
--
2.56.1.809.ga0b61035b5-goog