[PATCH RFC bpf-next 28/52] net, xdp: prefetch data a bit when building an skb from an &xdp_frame

[Alexander Lobakin]

Different cpumap tests showed that a couple of little careful
prefetches helps the performance. The only thing is to not go crazy:
only one cacheline to the right from the frame start and one to the
left -- if there is a metadata in front.

Signed-off-by: Alexander Lobakin <alexandr.lobakin@xxxxxxxxx>
---
 net/bpf/core.c | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/net/bpf/core.c b/net/bpf/core.c
index a8685bcc6e00..775f9648e8cf 100644
--- a/net/bpf/core.c
+++ b/net/bpf/core.c
@@ -620,10 +620,26 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
 					   struct net_device *dev)
 {
 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
+	u32 dist, metasize = xdpf->metasize;
 	unsigned int headroom, frame_size;
+	void *data = xdpf->data;
 	void *hard_start;
 	u8 nr_frags;
 
+	/* Bring the headers to the current CPU, as well as the
+	 * metadata if present. This helps eth_type_trans() and
+	 * xdp_populate_skb_meta_generic().
+	 * The idea here is to prefetch no more than 2 cachelines:
+	 * one to the left from the data start and one to the right.
+	 */
+#define to_cl(ptr) PTR_ALIGN_DOWN(ptr, L1_CACHE_BYTES)
+	dist = min_t(typeof(dist), metasize, L1_CACHE_BYTES);
+	if (dist && to_cl(data - dist) != to_cl(data))
+		prefetch(data - dist);
+#undef to_cl
+
+	prefetch(data);
+
 	/* xdp frags frame */
 	if (unlikely(xdp_frame_has_frags(xdpf)))
 		nr_frags = sinfo->nr_frags;
@@ -636,15 +652,15 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
 	 */
 	frame_size = xdpf->frame_sz;
 
-	hard_start = xdpf->data - headroom;
+	hard_start = data - headroom;
 	skb = build_skb_around(skb, hard_start, frame_size);
 	if (unlikely(!skb))
 		return NULL;
 
 	skb_reserve(skb, headroom);
 	__skb_put(skb, xdpf->len);
-	if (xdpf->metasize)
-		skb_metadata_set(skb, xdpf->metasize);
+	if (metasize)
+		skb_metadata_set(skb, metasize);
 
 	if (unlikely(xdp_frame_has_frags(xdpf)))
 		xdp_update_skb_shared_info(skb, nr_frags,
-- 
2.36.1