[PATCH net-next 01/16] libeth: convert to netmem

From: Alexander Lobakin
Date: Wed Mar 05 2025 - 11:23:25 EST

Next message: Alexander Lobakin: "[PATCH net-next 03/16] libeth: add a couple of XDP helpers (libeth_xdp)"
Previous message: Alexander Lobakin: "[PATCH net-next 02/16] libeth: support native XDP and register memory model"
In reply to: Alexander Lobakin: "[PATCH net-next 02/16] libeth: support native XDP and register memory model"
Next in thread: Mina Almasry: "Re: [PATCH net-next 01/16] libeth: convert to netmem"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

Back when the libeth Rx core was initially written, devmem was a draft
and netmem_ref didn't exist in the mainline. Now that it's here, make
libeth MP-agnostic before introducing any new code or any new library
users.
When it's known that the created PP/FQ is for header buffers, use faster
"unsafe" underscored netmem <--> virt accessors as netmem_is_net_iov()
is always false in that case, but consumes some cycles (bit test +
true branch).
Misc: replace explicit EXPORT_SYMBOL_NS_GPL("NS") with
DEFAULT_SYMBOL_NAMESPACE.

Signed-off-by: Alexander Lobakin <aleksander.lobakin@xxxxxxxxx>
---
include/net/libeth/rx.h | 22 +++++++------
drivers/net/ethernet/intel/iavf/iavf_txrx.c | 14 ++++----
.../ethernet/intel/idpf/idpf_singleq_txrx.c | 2 +-
drivers/net/ethernet/intel/idpf/idpf_txrx.c | 33 +++++++++++--------
drivers/net/ethernet/intel/libeth/rx.c | 20 ++++++-----
5 files changed, 51 insertions(+), 40 deletions(-)

diff --git a/include/net/libeth/rx.h b/include/net/libeth/rx.h
index ab05024be518..7d5dc58984b1 100644
--- a/include/net/libeth/rx.h
+++ b/include/net/libeth/rx.h
@@ -1,5 +1,5 @@
/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2024 Intel Corporation */
+/* Copyright (C) 2024-2025 Intel Corporation */

#ifndef __LIBETH_RX_H
#define __LIBETH_RX_H
@@ -31,7 +31,7 @@

/**
* struct libeth_fqe - structure representing an Rx buffer (fill queue element)
- * @page: page holding the buffer
+ * @netmem: network memory reference holding the buffer
* @offset: offset from the page start (to the headroom)
* @truesize: total space occupied by the buffer (w/ headroom and tailroom)
*
@@ -40,7 +40,7 @@
* former, @offset is always 0 and @truesize is always ```PAGE_SIZE```.
*/
struct libeth_fqe {
- struct page *page;
+ netmem_ref netmem;
u32 offset;
u32 truesize;
} __aligned_largest;
@@ -102,15 +102,16 @@ static inline dma_addr_t libeth_rx_alloc(const struct libeth_fq_fp *fq, u32 i)
struct libeth_fqe *buf = &fq->fqes[i];

buf->truesize = fq->truesize;
- buf->page = page_pool_dev_alloc(fq->pp, &buf->offset, &buf->truesize);
- if (unlikely(!buf->page))
+ buf->netmem = page_pool_dev_alloc_netmem(fq->pp, &buf->offset,
+ &buf->truesize);
+ if (unlikely(!buf->netmem))
return DMA_MAPPING_ERROR;

- return page_pool_get_dma_addr(buf->page) + buf->offset +
+ return page_pool_get_dma_addr_netmem(buf->netmem) + buf->offset +
fq->pp->p.offset;
}

-void libeth_rx_recycle_slow(struct page *page);
+void libeth_rx_recycle_slow(netmem_ref netmem);

/**
* libeth_rx_sync_for_cpu - synchronize or recycle buffer post DMA
@@ -126,18 +127,19 @@ void libeth_rx_recycle_slow(struct page *page);
static inline bool libeth_rx_sync_for_cpu(const struct libeth_fqe *fqe,
u32 len)
{
- struct page *page = fqe->page;
+ netmem_ref netmem = fqe->netmem;

/* Very rare, but possible case. The most common reason:
* the last fragment contained FCS only, which was then
* stripped by the HW.
*/
if (unlikely(!len)) {
- libeth_rx_recycle_slow(page);
+ libeth_rx_recycle_slow(netmem);
return false;
}

- page_pool_dma_sync_for_cpu(page->pp, page, fqe->offset, len);
+ page_pool_dma_sync_netmem_for_cpu(netmem_get_pp(netmem), netmem,
+ fqe->offset, len);

return true;
}
diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
index 422312b8b54a..35d353d38129 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
@@ -723,7 +723,7 @@ static void iavf_clean_rx_ring(struct iavf_ring *rx_ring)
for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) {
const struct libeth_fqe *rx_fqes = &rx_ring->rx_fqes[i];

- page_pool_put_full_page(rx_ring->pp, rx_fqes->page, false);
+ libeth_rx_recycle_slow(rx_fqes->netmem);

if (unlikely(++i == rx_ring->count))
i = 0;
@@ -1197,10 +1197,11 @@ static void iavf_add_rx_frag(struct sk_buff *skb,
const struct libeth_fqe *rx_buffer,
unsigned int size)
{
- u32 hr = rx_buffer->page->pp->p.offset;
+ u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset;

- skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
- rx_buffer->offset + hr, size, rx_buffer->truesize);
+ skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags,
+ rx_buffer->netmem, rx_buffer->offset + hr,
+ size, rx_buffer->truesize);
}

/**
@@ -1214,12 +1215,13 @@ static void iavf_add_rx_frag(struct sk_buff *skb,
static struct sk_buff *iavf_build_skb(const struct libeth_fqe *rx_buffer,
unsigned int size)
{
- u32 hr = rx_buffer->page->pp->p.offset;
+ struct page *buf_page = __netmem_to_page(rx_buffer->netmem);
+ u32 hr = buf_page->pp->p.offset;
struct sk_buff *skb;
void *va;

/* prefetch first cache line of first page */
- va = page_address(rx_buffer->page) + rx_buffer->offset;
+ va = page_address(buf_page) + rx_buffer->offset;
net_prefetch(va + hr);

/* build an skb around the page buffer */
diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
index eae1b6f474e6..aeb2ca5f5a0a 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
@@ -1009,7 +1009,7 @@ static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget)
break;

skip_data:
- rx_buf->page = NULL;
+ rx_buf->netmem = 0;

IDPF_SINGLEQ_BUMP_RING_IDX(rx_q, ntc);
cleaned_count++;
diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c
index bdf52cef3891..6254806c2072 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c
@@ -382,12 +382,12 @@ static int idpf_tx_desc_alloc_all(struct idpf_vport *vport)
*/
static void idpf_rx_page_rel(struct libeth_fqe *rx_buf)
{
- if (unlikely(!rx_buf->page))
+ if (unlikely(!rx_buf->netmem))
return;

- page_pool_put_full_page(rx_buf->page->pp, rx_buf->page, false);
+ libeth_rx_recycle_slow(rx_buf->netmem);

- rx_buf->page = NULL;
+ rx_buf->netmem = 0;
rx_buf->offset = 0;
}

@@ -3096,10 +3096,10 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb,
void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb,
unsigned int size)
{
- u32 hr = rx_buf->page->pp->p.offset;
+ u32 hr = netmem_get_pp(rx_buf->netmem)->p.offset;

- skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page,
- rx_buf->offset + hr, size, rx_buf->truesize);
+ skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, rx_buf->netmem,
+ rx_buf->offset + hr, size, rx_buf->truesize);
}

/**
@@ -3122,16 +3122,20 @@ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr,
struct libeth_fqe *buf, u32 data_len)
{
u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN;
+ struct page *hdr_page, *buf_page;
const void *src;
void *dst;

- if (!libeth_rx_sync_for_cpu(buf, copy))
+ if (unlikely(netmem_is_net_iov(buf->netmem)) ||
+ !libeth_rx_sync_for_cpu(buf, copy))
return 0;

- dst = page_address(hdr->page) + hdr->offset + hdr->page->pp->p.offset;
- src = page_address(buf->page) + buf->offset + buf->page->pp->p.offset;
- memcpy(dst, src, LARGEST_ALIGN(copy));
+ hdr_page = __netmem_to_page(hdr->netmem);
+ buf_page = __netmem_to_page(buf->netmem);
+ dst = page_address(hdr_page) + hdr->offset + hdr_page->pp->p.offset;
+ src = page_address(buf_page) + buf->offset + buf_page->pp->p.offset;

+ memcpy(dst, src, LARGEST_ALIGN(copy));
buf->offset += copy;

return copy;
@@ -3147,11 +3151,12 @@ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr,
*/
struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size)
{
- u32 hr = buf->page->pp->p.offset;
+ struct page *buf_page = __netmem_to_page(buf->netmem);
+ u32 hr = buf_page->pp->p.offset;
struct sk_buff *skb;
void *va;

- va = page_address(buf->page) + buf->offset;
+ va = page_address(buf_page) + buf->offset;
prefetch(va + hr);

skb = napi_build_skb(va, buf->truesize);
@@ -3302,7 +3307,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
u64_stats_update_end(&rxq->stats_sync);
}

- hdr->page = NULL;
+ hdr->netmem = 0;

payload:
if (!libeth_rx_sync_for_cpu(rx_buf, pkt_len))
@@ -3318,7 +3323,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
break;

skip_data:
- rx_buf->page = NULL;
+ rx_buf->netmem = 0;

idpf_rx_post_buf_refill(refillq, buf_id);
IDPF_RX_BUMP_NTC(rxq, ntc);
diff --git a/drivers/net/ethernet/intel/libeth/rx.c b/drivers/net/ethernet/intel/libeth/rx.c
index 66d1d23b8ad2..aa5d878181f7 100644
--- a/drivers/net/ethernet/intel/libeth/rx.c
+++ b/drivers/net/ethernet/intel/libeth/rx.c
@@ -1,5 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (C) 2024 Intel Corporation */
+/* Copyright (C) 2024-2025 Intel Corporation */
+
+#define DEFAULT_SYMBOL_NAMESPACE "LIBETH"

#include <net/libeth/rx.h>

@@ -186,7 +188,7 @@ int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi)

return -ENOMEM;
}
-EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_create, "LIBETH");
+EXPORT_SYMBOL_GPL(libeth_rx_fq_create);

/**
* libeth_rx_fq_destroy - destroy a &page_pool created by libeth
@@ -197,19 +199,19 @@ void libeth_rx_fq_destroy(struct libeth_fq *fq)
kvfree(fq->fqes);
page_pool_destroy(fq->pp);
}
-EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_destroy, "LIBETH");
+EXPORT_SYMBOL_GPL(libeth_rx_fq_destroy);

/**
- * libeth_rx_recycle_slow - recycle a libeth page from the NAPI context
- * @page: page to recycle
+ * libeth_rx_recycle_slow - recycle libeth netmem
+ * @netmem: network memory to recycle
*
* To be used on exceptions or rare cases not requiring fast inline recycling.
*/
-void libeth_rx_recycle_slow(struct page *page)
+void __cold libeth_rx_recycle_slow(netmem_ref netmem)
{
- page_pool_recycle_direct(page->pp, page);
+ page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false);
}
-EXPORT_SYMBOL_NS_GPL(libeth_rx_recycle_slow, "LIBETH");
+EXPORT_SYMBOL_GPL(libeth_rx_recycle_slow);

/* Converting abstract packet type numbers into a software structure with
* the packet parameters to do O(1) lookup on Rx.
@@ -251,7 +253,7 @@ void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt *pt)
pt->hash_type |= libeth_rx_pt_xdp_iprot[pt->inner_prot];
pt->hash_type |= libeth_rx_pt_xdp_pl[pt->payload_layer];
}
-EXPORT_SYMBOL_NS_GPL(libeth_rx_pt_gen_hash_type, "LIBETH");
+EXPORT_SYMBOL_GPL(libeth_rx_pt_gen_hash_type);

/* Module */

--
2.48.1

Next message: Alexander Lobakin: "[PATCH net-next 03/16] libeth: add a couple of XDP helpers (libeth_xdp)"
Previous message: Alexander Lobakin: "[PATCH net-next 02/16] libeth: support native XDP and register memory model"
In reply to: Alexander Lobakin: "[PATCH net-next 02/16] libeth: support native XDP and register memory model"
Next in thread: Mina Almasry: "Re: [PATCH net-next 01/16] libeth: convert to netmem"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]