[PATCH net-next v5 3/9] virtio-net: copy the XDP rx hash before the program runs
From: Steffen Trumtrar
Date: Thu Nov 12 2026 - 06:48:43 EST
virtnet_xdp_rx_hash() reads the hash from the vnet header at
xdp->data - vi->hdr_len when the XDP program calls
bpf_xdp_metadata_rx_hash(). That is only the header until the program
changes the buffer: bpf_xdp_adjust_meta() writes the metadata right in
front of xdp->data, over the header, and bpf_xdp_adjust_head() moves
xdp->data, so the callback then reads metadata or packet bytes.
Wrap struct xdp_buff in struct virtnet_xdp_buff, copy hash value and
report type into it in virtnet_xdp_handler() before
bpf_prog_run_xdp(), and let the callback read the copy. The small and
mergeable paths put the wrapper on the stack; for AF_XDP the fields are
in the cb of struct xdp_buff_xsk, checked with XSK_CHECK_PRIV_TYPE().
Move virtio_net_hash_value() up so the copy can use it.
Fixes: aa37f8916d20 ("virtio_net: Support RX hash XDP hint")
Signed-off-by: Steffen Trumtrar <s.trumtrar@xxxxxxxxxxxxxx>
---
drivers/net/virtio_net.c | 86 ++++++++++++++++++++++++++++++++----------------
1 file changed, 58 insertions(+), 28 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 4c740b403ad1e..ffd7313663c18 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -491,6 +491,26 @@ struct padded_vnet_hdr {
char padding[12];
};
+/*
+ * The vnet header sits in the headroom in front of xdp->data, which the
+ * XDP program may overwrite with bpf_xdp_adjust_head() or
+ * bpf_xdp_adjust_meta(). What the metadata kfuncs report is copied from
+ * it before the program runs. For AF_XDP the fields behind xdp live in
+ * the cb of struct xdp_buff_xsk.
+ */
+struct virtnet_xdp_buff {
+ struct xdp_buff xdp;
+ u32 hash_value;
+ u16 hash_report;
+};
+
+static inline u32
+virtio_net_hash_value(const struct virtio_net_hdr_v1_hash *hdr_hash)
+{
+ return __le16_to_cpu(hdr_hash->hash_value_lo) |
+ (__le16_to_cpu(hdr_hash->hash_value_hi) << 16);
+}
+
struct virtio_net_common_hdr {
union {
struct virtio_net_hdr hdr;
@@ -1687,6 +1707,24 @@ static void put_xdp_frags(struct receive_queue *rq, struct xdp_buff *xdp)
}
}
+/* xdp must be embedded in struct virtnet_xdp_buff or struct xdp_buff_xsk */
+static void virtnet_xdp_save_hdr(const struct virtnet_info *vi,
+ struct xdp_buff *xdp)
+{
+ struct virtnet_xdp_buff *ctx = container_of(xdp, struct virtnet_xdp_buff, xdp);
+ const struct virtio_net_hdr_v1_hash *hdr_hash = xdp->data - vi->hdr_len;
+
+ XSK_CHECK_PRIV_TYPE(struct virtnet_xdp_buff);
+
+ if (vi->has_rss_hash_report) {
+ ctx->hash_value = virtio_net_hash_value(hdr_hash);
+ ctx->hash_report = __le16_to_cpu(hdr_hash->hash_report);
+ } else {
+ ctx->hash_value = 0;
+ ctx->hash_report = VIRTIO_NET_HASH_REPORT_NONE;
+ }
+}
+
static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
struct net_device *dev,
unsigned int *xdp_xmit,
@@ -1696,6 +1734,8 @@ static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
int err;
u32 act;
+ virtnet_xdp_save_hdr(netdev_priv(dev), xdp);
+
act = bpf_prog_run_xdp(xdp_prog, xdp);
u64_stats_inc(&stats->xdp_packets);
@@ -1867,7 +1907,8 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
struct page *page = virt_to_head_page(buf);
struct page *xdp_page;
- struct xdp_buff xdp;
+ struct virtnet_xdp_buff vxdp;
+ struct xdp_buff *xdp = &vxdp.xdp;
struct sk_buff *skb;
unsigned int metasize = 0;
u32 act;
@@ -1900,17 +1941,17 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
page = xdp_page;
}
- xdp_init_buff(&xdp, buflen, &rq->xdp_rxq);
- xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len,
+ xdp_init_buff(xdp, buflen, &rq->xdp_rxq);
+ xdp_prepare_buff(xdp, buf + VIRTNET_RX_PAD + vi->hdr_len,
xdp_headroom, len, true);
- act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
+ act = virtnet_xdp_handler(xdp_prog, xdp, dev, xdp_xmit, stats);
switch (act) {
case XDP_PASS:
/* Recalculate length in case bpf program changed it */
- len = xdp.data_end - xdp.data;
- metasize = xdp.data - xdp.data_meta;
+ len = xdp->data_end - xdp->data;
+ metasize = xdp->data - xdp->data_meta;
break;
case XDP_TX:
@@ -1921,7 +1962,7 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
goto err_xdp;
}
- skb = virtnet_build_skb(buf, buflen, xdp.data - buf, len);
+ skb = virtnet_build_skb(buf, buflen, xdp->data - buf, len);
if (unlikely(!skb)) {
u64_stats_inc(&stats->alloc_fail);
goto err;
@@ -2288,7 +2329,8 @@ static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
unsigned int xdp_frags_truesz = 0;
struct sk_buff *head_skb;
unsigned int frame_sz;
- struct xdp_buff xdp;
+ struct virtnet_xdp_buff vxdp;
+ struct xdp_buff *xdp = &vxdp.xdp;
void *data;
u32 act;
int err;
@@ -2298,16 +2340,16 @@ static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
if (unlikely(!data))
goto err_xdp;
- err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz,
+ err = virtnet_build_xdp_buff_mrg(dev, vi, rq, xdp, data, len, frame_sz,
&num_buf, &xdp_frags_truesz, stats);
if (unlikely(err))
goto err_xdp;
- act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
+ act = virtnet_xdp_handler(xdp_prog, xdp, dev, xdp_xmit, stats);
switch (act) {
case XDP_PASS:
- head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
+ head_skb = build_skb_from_xdp_buff(dev, vi, xdp, xdp_frags_truesz);
if (unlikely(!head_skb)) {
u64_stats_inc(&stats->alloc_fail);
break;
@@ -2324,7 +2366,7 @@ static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
break;
}
- put_xdp_frags(rq, &xdp);
+ put_xdp_frags(rq, xdp);
err_xdp:
page_pool_put_page(rq->page_pool, page, -1, true);
@@ -2477,13 +2519,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
return NULL;
}
-static inline u32
-virtio_net_hash_value(const struct virtio_net_hdr_v1_hash *hdr_hash)
-{
- return __le16_to_cpu(hdr_hash->hash_value_lo) |
- (__le16_to_cpu(hdr_hash->hash_value_hi) << 16);
-}
-
static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
struct sk_buff *skb)
{
@@ -6760,23 +6795,18 @@ virtnet_xdp_rss_type[VIRTIO_NET_HASH_REPORT_MAX_TABLE] = {
static int virtnet_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash,
enum xdp_rss_hash_type *rss_type)
{
- const struct xdp_buff *xdp = (void *)_ctx;
- struct virtio_net_hdr_v1_hash *hdr_hash;
- struct virtnet_info *vi;
+ const struct virtnet_xdp_buff *ctx = (void *)_ctx;
u16 hash_report;
- if (!(xdp->rxq->dev->features & NETIF_F_RXHASH))
+ if (!(ctx->xdp.rxq->dev->features & NETIF_F_RXHASH))
return -ENODATA;
- vi = netdev_priv(xdp->rxq->dev);
- hdr_hash = (struct virtio_net_hdr_v1_hash *)(xdp->data - vi->hdr_len);
- hash_report = __le16_to_cpu(hdr_hash->hash_report);
-
+ hash_report = ctx->hash_report;
if (hash_report >= VIRTIO_NET_HASH_REPORT_MAX_TABLE)
hash_report = VIRTIO_NET_HASH_REPORT_NONE;
*rss_type = virtnet_xdp_rss_type[hash_report];
- *hash = virtio_net_hash_value(hdr_hash);
+ *hash = ctx->hash_value;
return 0;
}
--
2.56.0