Re: [PATCH net-next v1 05/11] xsk: Install UMEM page pool providers in the core
From: Magnus Karlsson
Date: Wed Nov 11 2026 - 03:59:38 EST
On Tue, 10 Nov 2026 at 11:36, Björn Töpel <bjorn@xxxxxxxxxx> wrote:
>
> A page_pool driver should not need its own AF_XDP setup. When it sets
> netdev_queue_mgmt_ops.xsk_page_pool, a zero-copy bind installs the
> UMEM provider through the queue API, as devmem and io_uring do. The
> driver only judges the chunk size, headroom and multi-buffer mode
> against its MTU and XDP program, in XDP_VALIDATE_XSK_POOL; MTU and
> program changes of a bound device get the same check. The UMEM
> headroom belongs to the socket, so the queue gets it as rx_headroom,
> next to rx_page_size.
>
> After a queue restart, the core checks that the new page pools took
> the provider. A readable provider must back the header pool, where an
> unsplit packet lands, so pools declare the buffers they supply in
> rx_roles. The old queue's pools carry the same provider during the
> check, so a per-queue generation, mp_gen, tells the new ones apart.
>
> Generic XDP runs outside the queue NAPI, so it cannot feed the socket.
>
> The provider needs no xdp_buff_xsk per chunk. The buffer pool
> allocates them only for copy mode and driver zero copy, which saves
> 128 bytes per chunk.
>
> Signed-off-by: Björn Töpel <bjorn@xxxxxxxxxx>
> ---
> include/linux/netdevice.h | 8 ++
> include/net/netdev_queues.h | 12 ++
> include/net/netdev_rx_queue.h | 2 +
> include/net/page_pool/types.h | 12 ++
> include/net/xdp_sock.h | 4 +
> include/net/xdp_sock_drv.h | 45 +++++++
> include/net/xsk_buff_pool.h | 3 +
> net/core/dev.c | 16 +++
> net/core/netdev_config.c | 2 +
> net/core/netdev_rx_queue.c | 9 +-
> net/core/page_pool.c | 6 +-
> net/core/page_pool_user.c | 25 ++--
> net/xdp/xsk.h | 7 +-
> net/xdp/xsk_buff_pool.c | 69 ++++++++---
> net/xdp/xsk_pp.c | 221 ++++++++++++++++++++++++++++++++++
> 15 files changed, 410 insertions(+), 31 deletions(-)
>
> diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
> index 435be7880d7e..11e51e684003 100644
> --- a/include/linux/netdevice.h
> +++ b/include/linux/netdevice.h
> @@ -77,6 +77,7 @@ struct udp_tunnel_info;
> struct udp_tunnel_nic_info;
> struct udp_tunnel_nic;
> struct bpf_prog;
> +struct xsk_rx_check;
> struct xdp_buff;
> struct xdp_frame;
> struct xdp_metadata_ops;
> @@ -1010,6 +1011,8 @@ enum bpf_netdev_command {
> BPF_OFFLOAD_MAP_ALLOC,
> BPF_OFFLOAD_MAP_FREE,
> XDP_SETUP_XSK_POOL,
> + /* Check an AF_XDP page pool setup, see netdev_queue_mgmt_ops */
> + XDP_VALIDATE_XSK_POOL,
> };
>
> struct bpf_prog_offload_ops;
> @@ -1048,6 +1051,11 @@ struct netdev_bpf {
> struct xsk_buff_pool *pool;
> u16 queue_id;
> } xsk;
> + /* XDP_VALIDATE_XSK_POOL */
> + struct {
> + const struct xsk_rx_check *check;
> + struct netlink_ext_ack *extack;
> + } xsk_check;
> };
> };
>
> diff --git a/include/net/netdev_queues.h b/include/net/netdev_queues.h
> index c5335e935b7d..ad5f0420c21d 100644
> --- a/include/net/netdev_queues.h
> +++ b/include/net/netdev_queues.h
> @@ -45,12 +45,15 @@ struct netdev_config {
> /**
> * struct netdev_queue_config - rendered configuration for an RX queue
> * @rx_page_size: Size of one RX page-pool allocation.
> + * @rx_headroom: Space before the packet in a head buffer, including
> + * XDP_PACKET_HEADROOM. Zero selects the driver default.
> * @rx_ring_size: Configured size of the regular RX ring.
> * @rx_mini_ring_size: Configured size of the RX mini ring.
> * @rx_jumbo_ring_size: Configured size of the RX jumbo ring.
> */
> struct netdev_queue_config {
> u32 rx_page_size;
> + u32 rx_headroom;
> u32 rx_ring_size;
> u32 rx_mini_ring_size;
> u32 rx_jumbo_ring_size;
> @@ -156,6 +159,8 @@ void netdev_stat_queue_sum(struct net_device *netdev,
> enum {
> /* The queue checks and honours the page size qcfg parameter */
> QCFG_RX_PAGE_SIZE = 0x1,
> + /* The queue checks and honours the headroom qcfg parameter */
> + QCFG_RX_HEADROOM = 0x2,
> };
>
> /**
> @@ -199,6 +204,12 @@ enum {
> *
> * @supported_params: Bitmask of supported parameters, see QCFG_*.
> *
> + * @xsk_page_pool: AF_XDP zero copy installs the UMEM as the page pool
> + * memory provider of the RX queue. The driver selects it
> + * with PP_FLAG_MEM_PROVIDER and PP_RX_ROLE_HEAD, honours
> + * whole buffers and QCFG_RX_HEADROOM, and accepts or
> + * rejects the setup in XDP_VALIDATE_XSK_POOL.
> + *
> * Note that @ndo_queue_mem_alloc and @ndo_queue_mem_free may be called while
> * the interface is closed. @ndo_queue_start and @ndo_queue_stop will only
> * be called for an interface which is open.
> @@ -229,6 +240,7 @@ struct netdev_queue_mgmt_ops {
> struct netlink_ext_ack *extack);
>
> unsigned int supported_params;
> + bool xsk_page_pool;
> };
>
> void netdev_queue_config(struct net_device *dev, int rxq,
> diff --git a/include/net/netdev_rx_queue.h b/include/net/netdev_rx_queue.h
> index 9415a94d333d..989120cbf452 100644
> --- a/include/net/netdev_rx_queue.h
> +++ b/include/net/netdev_rx_queue.h
> @@ -31,6 +31,8 @@ struct netdev_rx_queue {
> struct napi_struct *napi;
> struct netdev_queue_config qcfg;
> struct pp_memory_provider_params mp_params;
> + /* stamps the page pools of each queue allocation */
> + u32 mp_gen;
>
> /* If a queue is leased, then the lease pointer is always
> * valid. From the physical device it points to the virtual
> diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h
> index 0d48c05f1462..e9f3af91e305 100644
> --- a/include/net/page_pool/types.h
> +++ b/include/net/page_pool/types.h
> @@ -40,6 +40,12 @@
> */
> #define PP_FLAG_MEM_PROVIDER BIT(4)
>
> +/* RX buffers a page pool supplies, see page_pool_params.rx_roles */
> +enum pp_rx_role {
> + PP_RX_ROLE_HEAD = BIT(0), /* packet headers */
> + PP_RX_ROLE_PAYLOAD = BIT(1), /* payload after header split */
> +};
> +
> #define PP_FLAG_ALL (PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | \
> PP_FLAG_SYSTEM_POOL | \
> PP_FLAG_ALLOW_UNREADABLE_NETMEM | \
> @@ -94,6 +100,8 @@ struct pp_alloc_cache {
> * @queue_idx: queue idx this page_pool is being created for.
> * @flags: PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV, PP_FLAG_SYSTEM_POOL,
> * PP_FLAG_ALLOW_UNREADABLE_NETMEM, PP_FLAG_MEM_PROVIDER.
> + * @rx_roles: &enum pp_rx_role mask, for pools that select a memory
> + * provider. Zero means payload buffers.
> */
> struct page_pool_params {
> struct_group_tagged(page_pool_params_fast, fast,
> @@ -110,6 +118,7 @@ struct page_pool_params {
> struct net_device *netdev;
> unsigned int queue_idx;
> unsigned int flags;
> + unsigned int rx_roles;
> /* private: used by test code only */
> void (*init_callback)(netmem_ref netmem, void *arg);
> void *init_arg;
> @@ -182,6 +191,7 @@ struct pp_memory_provider_params {
> void *mp_priv;
> const struct memory_provider_ops *mp_ops;
> u32 rx_page_size;
> + u32 rx_headroom;
> };
>
> struct page_pool {
> @@ -245,6 +255,8 @@ struct page_pool {
>
> void *mp_priv;
> const struct memory_provider_ops *mp_ops;
> + /* queue allocation that selected the provider */
> + u32 mp_gen;
>
> struct xarray dma_mapped;
>
> diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
> index 6e70b320b399..e0e22ef839e1 100644
> --- a/include/net/xdp_sock.h
> +++ b/include/net/xdp_sock.h
> @@ -19,6 +19,10 @@
> struct net_device;
> struct xsk_queue;
> struct xdp_buff;
> +struct memory_provider_ops;
> +
> +/* The memory provider of AF_XDP page pool zero copy */
> +extern const struct memory_provider_ops xsk_pp_ops;
>
> struct xdp_umem {
> void *addrs;
> diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
> index d94aeb506379..5bb1949a0aa7 100644
> --- a/include/net/xdp_sock_drv.h
> +++ b/include/net/xdp_sock_drv.h
> @@ -22,8 +22,29 @@ struct xsk_cb_desc {
> u8 bytes;
> };
>
> +/**
> + * struct xsk_rx_check - AF_XDP page pool RX setup a driver must accept
> + * @buf_size: size of each RX buffer
> + * @headroom: space before the packet in a head buffer, including
> + * XDP_PACKET_HEADROOM. The UMEM headroom is the rest.
> + * @mtu: MTU of the device
> + * @sg: the socket takes packets in more than one buffer
> + * @prog: XDP program to check, or NULL for the current one
> + */
> +struct xsk_rx_check {
> + u32 buf_size;
> + u32 headroom;
> + unsigned int mtu;
> + bool sg;
> + struct bpf_prog *prog;
> +};
> +
> #ifdef CONFIG_XDP_SOCKETS
>
> +int xsk_validate_dev(struct net_device *dev, unsigned int mtu,
> + struct bpf_prog *prog, enum bpf_xdp_mode mode,
> + struct netlink_ext_ack *extack);
> +struct xsk_buff_pool *xsk_rxq_pool(struct net_device *dev, u16 queue_id);
> void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
> bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
> u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max);
> @@ -36,6 +57,11 @@ void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool);
> void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool);
> bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool);
>
> +static inline bool xsk_pool_uses_sg(struct xsk_buff_pool *pool)
> +{
> + return pool->umem->flags & XDP_UMEM_SG_FLAG;
> +}
> +
> static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
> {
> return XDP_PACKET_HEADROOM + pool->headroom;
> @@ -351,6 +377,20 @@ static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
>
> #else
>
> +static inline int xsk_validate_dev(struct net_device *dev, unsigned int mtu,
> + struct bpf_prog *prog,
> + enum bpf_xdp_mode mode,
> + struct netlink_ext_ack *extack)
> +{
> + return 0;
> +}
> +
> +static inline struct xsk_buff_pool *xsk_rxq_pool(struct net_device *dev,
> + u16 queue_id)
> +{
> + return NULL;
> +}
> +
> static inline void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
> {
> }
> @@ -397,6 +437,11 @@ static inline bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
> return false;
> }
>
> +static inline bool xsk_pool_uses_sg(struct xsk_buff_pool *pool)
> +{
> + return false;
> +}
> +
> static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
> {
> return 0;
> diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
> index 02e89350faf9..6d7cfdf99a8b 100644
> --- a/include/net/xsk_buff_pool.h
> +++ b/include/net/xsk_buff_pool.h
> @@ -11,6 +11,7 @@
> #include <net/xdp.h>
>
> struct xsk_buff_pool;
> +struct xsk_pp;
> struct xdp_rxq_info;
> struct xsk_cb_desc;
> struct xsk_queue;
> @@ -54,6 +55,8 @@ struct xsk_buff_pool {
> spinlock_t xsk_tx_list_lock;
> refcount_t users;
> struct xdp_umem *umem;
> + /* page pool memory provider, see net/xdp/xsk_pp.c */
> + struct xsk_pp *pp;
> struct work_struct work;
> /* Protects generic receive in shared and non-shared umem mode. */
> spinlock_t rx_lock;
> diff --git a/net/core/dev.c b/net/core/dev.c
> index 663fe3e32ea0..7e985fb82488 100644
> --- a/net/core/dev.c
> +++ b/net/core/dev.c
> @@ -163,6 +163,7 @@
> #include <net/page_pool/types.h>
> #include <net/page_pool/helpers.h>
> #include <net/page_pool/memory_provider.h>
> +#include <net/xdp_sock_drv.h>
> #include <net/rps.h>
> #include <linux/phy_link_topology.h>
>
> @@ -10535,6 +10536,8 @@ u8 dev_xdp_sb_prog_count(struct net_device *dev)
>
> int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
> {
> + int err;
> +
> if (!dev->netdev_ops->ndo_bpf)
> return -EOPNOTSUPP;
>
> @@ -10552,6 +10555,13 @@ int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
> return -EBUSY;
> }
>
> + if (bpf->command == XDP_SETUP_PROG && bpf->prog) {
> + err = xsk_validate_dev(dev, dev->mtu, bpf->prog, XDP_MODE_DRV,
> + bpf->extack);
> + if (err)
> + return err;
> + }
> +
> return dev->netdev_ops->ndo_bpf(dev, bpf);
> }
> EXPORT_SYMBOL_GPL(netif_xdp_propagate);
> @@ -10629,6 +10639,12 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
> return -EBUSY;
> }
>
> + if (mode != XDP_MODE_HW && prog) {
> + err = xsk_validate_dev(dev, dev->mtu, prog, mode, extack);
> + if (err)
> + return err;
> + }
> +
> memset(&xdp, 0, sizeof(xdp));
> xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
> xdp.extack = extack;
> diff --git a/net/core/netdev_config.c b/net/core/netdev_config.c
> index 1975de42a60d..97e33764737b 100644
> --- a/net/core/netdev_config.c
> +++ b/net/core/netdev_config.c
> @@ -88,6 +88,8 @@ static int __netdev_queue_config(struct net_device *dev, int rxq_idx,
> mpp = &__netif_get_rx_queue(dev, rxq_idx)->mp_params;
> if (mpp->rx_page_size)
> qcfg->rx_page_size = mpp->rx_page_size;
> + if (mpp->rx_headroom)
> + qcfg->rx_headroom = mpp->rx_headroom;
> err = validate_cb(dev, qcfg, extack);
> if (err)
> return err;
> diff --git a/net/core/netdev_rx_queue.c b/net/core/netdev_rx_queue.c
> index 9e14a747f9b5..4e2f906fbc73 100644
> --- a/net/core/netdev_rx_queue.c
> +++ b/net/core/netdev_rx_queue.c
> @@ -6,6 +6,7 @@
> #include <net/netdev_queues.h>
> #include <net/netdev_rx_queue.h>
> #include <net/page_pool/memory_provider.h>
> +#include <net/xdp_sock.h>
>
> #include "dev.h"
> #include "page_pool_priv.h"
> @@ -141,6 +142,7 @@ static int netdev_rx_queue_reconfig(struct net_device *dev,
> goto err_free_new_mem;
> }
>
> + rxq->mp_gen++;
> err = qops->ndo_queue_mem_alloc(dev, qcfg_new, new_mem, rxq_idx);
> if (err)
> goto err_free_old_mem;
> @@ -235,6 +237,10 @@ static int __netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
> NL_SET_ERR_MSG(extack, "device does not support: rx_page_size");
> return -EOPNOTSUPP;
> }
> + if (p->rx_headroom && !(qops->supported_params & QCFG_RX_HEADROOM)) {
> + NL_SET_ERR_MSG(extack, "device does not support: rx_headroom");
> + return -EOPNOTSUPP;
> + }
>
> rxq = __netif_get_rx_queue(dev, rxq_idx);
> if (rxq->mp_params.mp_ops) {
> @@ -242,7 +248,8 @@ static int __netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
> return -EEXIST;
> }
> #ifdef CONFIG_XDP_SOCKETS
> - if (rxq->pool) {
> + /* An AF_XDP pool owns the queue, unless it installs its provider. */
> + if (rxq->pool && p->mp_ops != &xsk_pp_ops) {
> NL_SET_ERR_MSG(extack, "designated queue already in use by AF_XDP");
> return -EBUSY;
> }
> diff --git a/net/core/page_pool.c b/net/core/page_pool.c
> index 8237fda598ab..e897e79d31f1 100644
> --- a/net/core/page_pool.c
> +++ b/net/core/page_pool.c
> @@ -215,7 +215,8 @@ bool netmem_can_retain(netmem_ref netmem)
> EXPORT_SYMBOL(netmem_can_retain);
>
> static int page_pool_select_mp(struct page_pool *pool,
> - const struct pp_memory_provider_params *mpp)
> + const struct pp_memory_provider_params *mpp,
> + u32 rxq_mp_gen)
> {
> const struct memory_provider_ops *ops = mpp->mp_ops;
> unsigned int flags = pool->slow.flags;
> @@ -234,6 +235,7 @@ static int page_pool_select_mp(struct page_pool *pool,
>
> pool->mp_priv = mpp->mp_priv;
> pool->mp_ops = ops;
> + pool->mp_gen = rxq_mp_gen;
> return 0;
> }
>
> @@ -326,7 +328,7 @@ static int page_pool_init(struct page_pool *pool,
> netdev_assert_locked(pool->slow.netdev);
> rxq = __netif_get_rx_queue(pool->slow.netdev,
> pool->slow.queue_idx);
> - err = page_pool_select_mp(pool, &rxq->mp_params);
> + err = page_pool_select_mp(pool, &rxq->mp_params, rxq->mp_gen);
> if (err)
> goto free_ptr_ring;
> }
> diff --git a/net/core/page_pool_user.c b/net/core/page_pool_user.c
> index f30f3779624c..b69da0f74756 100644
> --- a/net/core/page_pool_user.c
> +++ b/net/core/page_pool_user.c
> @@ -398,28 +398,37 @@ void page_pool_unlist(struct page_pool *pool)
> mutex_unlock(&page_pools_lock);
> }
>
> +/* Check that the queue allocation that just ran put the provider in every
> + * pool role it needs: headers for a readable provider, payload otherwise.
> + * Pools of earlier allocations do not count.
> + */
> int page_pool_check_memory_provider(struct net_device *dev,
> struct netdev_rx_queue *rxq)
> {
> - void *binding = rxq->mp_params.mp_priv;
> + const struct memory_provider_ops *ops = rxq->mp_params.mp_ops;
> + void *mp_priv = rxq->mp_params.mp_priv;
> + unsigned int need, have = 0;
> struct page_pool *pool;
> struct hlist_node *n;
>
> - if (!binding)
> + if (!mp_priv)
> return 0;
>
> + need = PP_RX_ROLE_PAYLOAD;
> + if (ops->caps & MP_CAP_READABLE)
> + need = PP_RX_ROLE_HEAD;
> +
> mutex_lock(&page_pools_lock);
> hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) {
> - if (pool->mp_priv != binding)
> + if (pool->mp_priv != mp_priv || pool->mp_gen != rxq->mp_gen ||
> + pool->slow.queue_idx != get_netdev_rx_queue_index(rxq))
> continue;
>
> - if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) {
> - mutex_unlock(&page_pools_lock);
> - return 0;
> - }
> + have |= pool->slow.rx_roles ? : PP_RX_ROLE_PAYLOAD;
> }
> mutex_unlock(&page_pools_lock);
> - return -ENODATA;
> +
> + return (have & need) == need ? 0 : -ENODATA;
> }
>
> static void page_pool_unreg_netdev_wipe(struct net_device *netdev)
> diff --git a/net/xdp/xsk.h b/net/xdp/xsk.h
> index 4dab7b9a1487..9699e9a499a3 100644
> --- a/net/xdp/xsk.h
> +++ b/net/xdp/xsk.h
> @@ -36,10 +36,13 @@ void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs,
> struct xdp_sock __rcu **map_entry);
> struct xsk_pp;
>
> -extern const struct memory_provider_ops xsk_pp_ops;
> -
> struct xsk_pp *xsk_pp_create(struct xsk_buff_pool *pool);
> void xsk_pp_put(struct xsk_pp *pp);
> +bool xsk_pp_supported(struct net_device *dev);
> +int xsk_pp_bind(struct xsk_buff_pool *pool, struct net_device *dev,
> + u16 queue_id);
> +void xsk_pp_unbind(struct xsk_buff_pool *pool);
> +int xp_alloc_heads(struct xsk_buff_pool *pool);
> void xsk_clear_pool_at_qid(struct net_device *dev, u16 queue_id);
> int xsk_reg_pool_at_qid(struct net_device *dev, struct xsk_buff_pool *pool,
> u16 queue_id);
> diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
> index 18966bbdb5b7..c620a0c32c17 100644
> --- a/net/xdp/xsk_buff_pool.c
> +++ b/net/xdp/xsk_buff_pool.c
> @@ -38,6 +38,8 @@ void xp_destroy(struct xsk_buff_pool *pool)
> if (!pool)
> return;
>
> + if (pool->pp)
> + xsk_pp_put(pool->pp);
> kvfree(pool->tx_descs);
> kvfree(pool->heads);
> kvfree(pool);
> @@ -62,18 +64,13 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
> {
> bool unaligned = umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG;
> struct xsk_buff_pool *pool;
> - struct xdp_buff_xsk *xskb;
> - u32 i, entries;
> + u32 entries;
>
> entries = unaligned ? umem->chunks : 0;
> pool = kvzalloc_flex(*pool, free_heads, entries);
> if (!pool)
> goto out;
>
> - pool->heads = kvzalloc_objs(*pool->heads, umem->chunks);
> - if (!pool->heads)
> - goto out;
> -
> if (xs->tx)
> if (xp_alloc_tx_descs(pool, xs, max_segs))
> goto out;
> @@ -104,10 +101,30 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
> pool->fq = xs->fq_tmp;
> pool->cq = xs->cq_tmp;
>
> - for (i = 0; i < pool->free_heads_cnt; i++) {
> + return pool;
> +
> +out:
> + xp_destroy(pool);
> + return NULL;
> +}
> +
> +/* Only copy mode and driver zero copy use heads */
> +int xp_alloc_heads(struct xsk_buff_pool *pool)
> +{
> + struct xdp_buff_xsk *xskb;
> + u32 i;
> +
> + if (pool->heads)
> + return 0;
> +
> + pool->heads = kvzalloc_objs(*pool->heads, pool->heads_cnt);
> + if (!pool->heads)
> + return -ENOMEM;
> +
> + for (i = 0; i < pool->heads_cnt; i++) {
Now I get why you are not reusing the existing xdp_buff_xsk infra, you
are removing them here for the PP case. Please mention this in the
cover letter of the previous patch. There might be a good argument for
it at least in aligned mode. It is wasting memory versus performance.
Today when memory is expensive and also for umems that are shared
between many sockets, the current scheme in the xsk_buffer pool for
aligned mode might not be worth it.
I have two questions for you. What would the performance be if you
moved the aligned mode over to use the same free_list as unaligned
mode and then use that in PP? Would reuse more code, but would it be
fast enough?
If that is not performant, what if you instead decreased the cost of
the current pre-allocated xskbs in aligned mode by just creating a
single copy per umem and using those in PP? Might be a good thing for
regular xsk drivers too these days, but do not know if it will work
for you.
I do not know if any of these suggestions would work, but I think you
should examine them as less code paths is generally a good thing. You
might also improve the non-PP case too by optimizing it for your case!
> xskb = &pool->heads[i];
> xskb->pool = pool;
> - xskb->xdp.frame_sz = umem->chunk_size - umem->headroom;
> + xskb->xdp.frame_sz = pool->chunk_size - pool->headroom;
> INIT_LIST_HEAD(&xskb->list_node);
> if (pool->unaligned)
> pool->free_heads[i] = xskb;
> @@ -115,11 +132,7 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
> xp_init_xskb_addr(xskb, pool, (u64)i * pool->chunk_size);
> }
>
> - return pool;
> -
> -out:
> - xp_destroy(pool);
> - return NULL;
> + return 0;
Agree with Stan that these changes should be in a separate commit.
Think also about if it would make sense to put your page_pool changes
in yet another commit.
> }
>
> void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq)
> @@ -212,9 +225,24 @@ int xp_assign_dev(struct xsk_buff_pool *pool,
> */
> hlist_add_head(&pool->dev_node, &netdev->xsk_pools);
>
> - if (force_copy)
> + if (force_copy) {
> /* For copy-mode, we are done. */
> + err = xp_alloc_heads(pool);
> + goto out;
> + }
> +
> + if (xsk_pp_supported(netdev)) {
> + err = xsk_pp_bind(pool, netdev, queue_id);
> + if (err)
> + goto err_unreg_pool;
> +
> + pool->umem->zc = true;
> return 0;
> + }
> +
> + err = xp_alloc_heads(pool);
> + if (err)
> + goto out;
>
> if ((netdev->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK) {
> err = -EOPNOTSUPP;
> @@ -267,8 +295,10 @@ int xp_assign_dev(struct xsk_buff_pool *pool,
> err_unreg_xsk:
> xp_disable_drv_zc(pool);
> err_unreg_pool:
> - if (!force_zc)
> - err = 0; /* fallback to copy mode */
> + /* An old page pool may still own chunks of a failed provider. */
> + if (!force_zc && !pool->pp)
> + err = xp_alloc_heads(pool); /* fallback to copy mode */
> +out:
> if (err) {
> pool->umem->flags = umem_flags;
> xsk_clear_pool_at_qid(netdev, queue_id);
> @@ -301,7 +331,10 @@ static void __xp_clear_dev(struct xsk_buff_pool *pool)
> struct net_device *netdev = pool->netdev;
>
> netdev_assert_locked_ops_compat(netdev);
> - xp_disable_drv_zc(pool);
> + if (pool->pp)
> + xsk_pp_unbind(pool);
> + else
> + xp_disable_drv_zc(pool);
> xsk_clear_pool_at_qid(netdev, pool->queue_id);
> hlist_del(&pool->dev_node);
> pool->netdev = NULL;
> @@ -474,7 +507,7 @@ static void xp_check_dma_contiguity(struct xsk_dma_map *dma_map)
>
> static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_map)
> {
> - if (!pool->unaligned) {
> + if (!pool->unaligned && pool->heads) {
> u32 i;
>
> for (i = 0; i < pool->heads_cnt; i++) {
> diff --git a/net/xdp/xsk_pp.c b/net/xdp/xsk_pp.c
> index fdd55265a6cd..c2214e718cdb 100644
> --- a/net/xdp/xsk_pp.c
> +++ b/net/xdp/xsk_pp.c
> @@ -8,6 +8,9 @@
> * +-----RX ring-------+
> */
>
> +#include <net/netdev_lock.h>
> +#include <net/netdev_queues.h>
> +#include <net/netdev_rx_queue.h>
> #include <net/page_pool/helpers.h>
> #include <net/page_pool/memory_provider.h>
> #include <net/xdp_sock_drv.h>
> @@ -15,6 +18,8 @@
> #include "xsk.h"
> #include "xsk_queue.h"
>
> +#define XSK_PP_DMA_ATTR (DMA_ATTR_WEAK_ORDERING | DMA_ATTR_SKIP_CPU_SYNC)
> +
> enum xsk_pp_chunk {
> XSK_PP_USER, /* owned by userspace */
> XSK_PP_POOL, /* owned by a page pool */
> @@ -30,6 +35,8 @@ struct xsk_pp {
> refcount_t refs;
> u32 nr_pools;
> bool dma_need_sync;
> + /* RX queue of the provider, NULL once it is removed */
> + struct netdev_rx_queue *rxq;
> u8 *state;
> u32 reuse_cnt;
> u32 reuse[];
> @@ -236,11 +243,225 @@ static int xsk_pp_nl_fill(void *mp_priv, struct sk_buff *rsp,
> return 0;
> }
>
> +/* The queue no longer uses the provider, and its DMA stopped */
> +static void xsk_pp_detach(struct xsk_pp *pp)
> +{
> + pp->rxq = NULL;
> + spin_lock_bh(&pp->lock);
Just curious, but why this protection of the fq here?
> + pp->fq = NULL;
> + spin_unlock_bh(&pp->lock);
> + xp_dma_unmap(pp->pool, XSK_PP_DMA_ATTR);
> +}
> +
> +/* The device unregisters, and is closed */
> +static void xsk_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq)
> +{
> + xsk_pp_detach(mp_priv);
> +}
> +
> const struct memory_provider_ops xsk_pp_ops = {
> .init = xsk_pp_init,
> .destroy = xsk_pp_destroy,
> .alloc_netmems = xsk_pp_alloc_netmems,
> .release_netmem = xsk_pp_release_netmem,
> .nl_fill = xsk_pp_nl_fill,
> + .uninstall = xsk_pp_uninstall,
> .caps = MP_CAP_READABLE,
> };
> +
> +static bool xsk_pp_abandon(struct xsk_pp *pp)
> +{
> + struct xsk_buff_pool *pool = pp->pool;
> + bool busy;
> + u32 idx;
> +
> + spin_lock_bh(&pp->lock);
> + busy = pp->nr_pools;
> + while (!busy && pp->reuse_cnt) {
> + idx = pp->reuse[--pp->reuse_cnt];
> + pp->state[idx] = XSK_PP_USER;
> + xp_free(&pool->heads[idx]);
> + }
> + spin_unlock_bh(&pp->lock);
> +
> + return !busy;
> +}
> +
> +static int xsk_pp_validate(struct net_device *dev, struct xsk_buff_pool *pool,
> + unsigned int mtu, struct bpf_prog *prog,
> + struct netlink_ext_ack *extack)
> +{
> + struct xsk_rx_check check = {
> + .buf_size = pool->chunk_size,
> + .headroom = xsk_pool_get_headroom(pool),
> + .mtu = mtu,
> + .sg = xsk_pool_uses_sg(pool),
> + .prog = prog,
> + };
> + struct netdev_bpf bpf = {
> + .command = XDP_VALIDATE_XSK_POOL,
> + .xsk_check.check = &check,
> + .xsk_check.extack = extack,
> + };
> +
> + return dev->netdev_ops->ndo_bpf(dev, &bpf);
> +}
> +
> +/**
> + * xsk_validate_dev - check AF_XDP page pool queues against a new setting
> + * @dev: device of the queues
> + * @mtu: MTU to check
> + * @prog: XDP program to check, or NULL for the current one
> + * @mode: XDP mode of @prog
> + * @extack: netlink extended ack
> + *
> + * Context: holds the instance lock of @dev, or RTNL for a device without it.
> + * Return: 0 if the driver accepts all queues of @dev that have an AF_XDP
> + * memory provider, or its error.
> + */
> +int xsk_validate_dev(struct net_device *dev, unsigned int mtu,
> + struct bpf_prog *prog, enum bpf_xdp_mode mode,
> + struct netlink_ext_ack *extack)
> +{
> + struct netdev_rx_queue *rxq;
> + struct xsk_pp *pp;
> + unsigned int i;
> + int err;
> +
> + netdev_assert_locked_ops_compat(dev);
> +
> + for (i = 0; i < dev->real_num_rx_queues; i++) {
> + rxq = __netif_get_rx_queue(dev, i);
> + if (rxq->mp_params.mp_ops != &xsk_pp_ops)
> + continue;
> +
> + /* Generic XDP runs outside the queue NAPI */
> + if (mode == XDP_MODE_SKB) {
> + NL_SET_ERR_MSG(extack,
> + "generic XDP cannot feed an AF_XDP page pool socket");
> + return -EBUSY;
> + }
> +
> + pp = rxq->mp_params.mp_priv;
> + err = xsk_pp_validate(dev, pp->pool, mtu, prog, extack);
> + if (err)
> + return err;
> + }
> +
> + return 0;
> +}
> +EXPORT_SYMBOL_GPL(xsk_validate_dev);
> +
> +/**
> + * xsk_rxq_pool - get the AF_XDP pool that provides memory to an RX queue
> + * @dev: device of the queue
> + * @queue_id: RX queue
> + *
> + * The pool stays valid while the queue that the caller allocates runs.
> + *
> + * Context: holds the instance lock of @dev.
> + * Return: the buffer pool, or NULL.
> + */
> +struct xsk_buff_pool *xsk_rxq_pool(struct net_device *dev, u16 queue_id)
> +{
> + struct netdev_rx_queue *rxq;
> + struct xsk_pp *pp;
> +
> + netdev_assert_locked(dev);
> +
> + if (queue_id >= dev->real_num_rx_queues)
> + return NULL;
> +
> + rxq = __netif_get_rx_queue(dev, queue_id);
> + if (rxq->mp_params.mp_ops != &xsk_pp_ops)
> + return NULL;
> +
> + pp = rxq->mp_params.mp_priv;
> + return pp->pool;
> +}
> +EXPORT_SYMBOL_GPL(xsk_rxq_pool);
> +
> +bool xsk_pp_supported(struct net_device *dev)
> +{
> + const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops;
> +
> + return qops && qops->xsk_page_pool;
> +}
> +
> +/* Called with RTNL and the instance lock of @dev. On error, @pool->pp stays
> + * set if an old page pool still owns chunks; copy mode must not take the pool.
> + */
> +int xsk_pp_bind(struct xsk_buff_pool *pool, struct net_device *dev,
> + u16 queue_id)
> +{
> + struct pp_memory_provider_params mp = { .mp_ops = &xsk_pp_ops };
> + struct device *dma_dev;
> + struct xsk_pp *pp;
> + int err;
> +
> + if (queue_id >= dev->real_num_rx_queues || pool->unaligned)
> + return -EOPNOTSUPP;
> + if ((dev->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK ||
> + (xsk_pool_uses_sg(pool) && dev->xdp_zc_max_segs == 1))
> + return -EOPNOTSUPP;
> + if (dev_xdp_prog_id(dev, XDP_MODE_SKB))
> + return -EOPNOTSUPP;
> + if (!netif_all_mps_have_cap(dev, MP_CAP_READABLE))
> + return -EBUSY;
> +
> + err = xsk_pp_validate(dev, pool, dev->mtu, NULL, NULL);
> + if (err)
> + return err;
> +
> + dma_dev = netdev_queue_get_dma_dev(dev, queue_id, NETDEV_QUEUE_TYPE_RX);
> + if (!dma_dev)
> + return -EOPNOTSUPP;
> +
> + err = xp_dma_map(pool, dma_dev, XSK_PP_DMA_ATTR, pool->umem->pgs,
> + pool->umem->npgs);
> + if (err)
> + return err;
> +
> + pp = xsk_pp_create(pool);
> + if (IS_ERR(pp)) {
> + xp_dma_unmap(pool, XSK_PP_DMA_ATTR);
> + return PTR_ERR(pp);
> + }
> +
> + mp.mp_priv = pp;
> + mp.rx_page_size = pool->chunk_size;
> + mp.rx_headroom = xsk_pool_get_headroom(pool);
> + err = netif_mp_open_rxq(dev, queue_id, &mp, NULL);
> + if (!err) {
> + pp->rxq = __netif_get_rx_queue(dev, queue_id);
> + pool->pp = pp;
> + pool->xdp_zc_max_segs = dev->xdp_zc_max_segs;
> + return 0;
> + }
> +
> + xp_dma_unmap(pool, XSK_PP_DMA_ATTR);
> + /* Copy mode takes the chunks that the provider holds */
> + if (!xp_alloc_heads(pool) && xsk_pp_abandon(pp))
> + xsk_pp_put(pp);
> + else
> + pool->pp = pp;
> +
> + return err;
> +}
> +
> +/* Called with RTNL and the instance lock of the device */
> +void xsk_pp_unbind(struct xsk_buff_pool *pool)
> +{
> + struct xsk_pp *pp = pool->pp;
> + struct pp_memory_provider_params mp = {
> + .mp_ops = &xsk_pp_ops,
> + .mp_priv = pp,
> + };
> +
> + /* Unregister removed the provider already */
> + if (!pp->rxq)
> + return;
> +
> + netif_mp_close_rxq(pool->netdev, pool->queue_id, &mp);
> + xsk_pp_detach(pp);
> +}
> --
> 2.56.0
>
>