[RESEND PATCH] net/core: consolidate RPS dispatch into netif_rps() helpers

[Jemmy Wong]

From: "Jemmy Wong" <jemmywong512@xxxxxxxxx>

The RPS steering logic in netif_rx_internal(), netif_receive_skb_internal()
and netif_receive_skb_list_internal() was open-coded three times, each with
its own #ifdef CONFIG_RPS block and manual rcu_read_lock()/unlock() pairs.

Factor it into two helpers, netif_rps() for the single-skb path and
netif_rps_list() for the list path, and switch the callers to
guard(rcu)/scoped_guard(rcu). A new internal NET_RX_UNHANDLED sentinel lets
a helper report "RPS did not take this skb" so the caller falls back to the
local enqueue / __netif_receive_skb() path; it never escapes to callers.

netif_rps_list() keeps the early static_branch_unlikely(&rps_needed) bail
out so the list is not needlessly walked and re-spliced when RPS is
compiled in but disabled.

No functional change intended.

Signed-off-by: Jemmy Wong <jemmywong512@xxxxxxxxx>
---
 include/linux/netdevice.h |  5 ++-
 net/core/dev.c            | 94 +++++++++++++++++++--------------------
 2 files changed, 48 insertions(+), 51 deletions(-)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 9981d637f8b5..c265b78082e3 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -93,8 +93,9 @@ void netdev_set_default_ethtool_ops(struct net_device *dev,
 void netdev_sw_irq_coalesce_default_on(struct net_device *dev);

 /* Backlog congestion levels */
-#define NET_RX_SUCCESS		0	/* keep 'em coming, baby */
-#define NET_RX_DROP		1	/* packet dropped */
+#define NET_RX_UNHANDLED	-1
+#define NET_RX_SUCCESS		0
+#define NET_RX_DROP		1

 #define MAX_NEST_DEV 8

diff --git a/net/core/dev.c b/net/core/dev.c
index 4b3d5cfdf6e0..259f8c8e5657 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -5426,6 +5426,38 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
 	return NET_RX_DROP;
 }

+static inline int netif_rps(struct sk_buff *skb)
+{
+#ifdef CONFIG_RPS
+	if (static_branch_unlikely(&rps_needed)) {
+		struct rps_dev_flow voidflow, *rflow = &voidflow;
+		int cpu = get_rps_cpu(skb->dev, skb, &rflow);
+
+		if (cpu >= 0)
+			return enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
+	}
+#endif
+	return NET_RX_UNHANDLED;
+}
+
+static inline void netif_rps_list(struct list_head *head)
+{
+#ifdef CONFIG_RPS
+	struct sk_buff *skb, *next;
+	LIST_HEAD(undo_list);
+
+	if (!static_branch_unlikely(&rps_needed))
+		return;
+
+	list_for_each_entry_safe(skb, next, head, list) {
+		skb_list_del_init(skb);
+		if (netif_rps(skb) == NET_RX_UNHANDLED)
+			list_add_tail(&skb->list, &undo_list);
+	}
+	list_splice_init(&undo_list, head);
+#endif
+}
+
 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
 {
 	struct net_device *dev = skb->dev;
@@ -5695,33 +5727,20 @@ EXPORT_SYMBOL_GPL(do_xdp_generic);

 static int netif_rx_internal(struct sk_buff *skb)
 {
-	int ret;
+	int ret = NET_RX_UNHANDLED;
+	unsigned int qtail;

 	net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);

 	trace_netif_rx(skb);

-#ifdef CONFIG_RPS
-	if (static_branch_unlikely(&rps_needed)) {
-		struct rps_dev_flow voidflow, *rflow = &voidflow;
-		int cpu;
-
-		rcu_read_lock();
-
-		cpu = get_rps_cpu(skb->dev, skb, &rflow);
-		if (cpu < 0)
-			cpu = smp_processor_id();
+	scoped_guard(rcu)
+		ret = netif_rps(skb);
+	if (ret != NET_RX_UNHANDLED)
+		return ret;

-		ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
+	ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);

-		rcu_read_unlock();
-	} else
-#endif
-	{
-		unsigned int qtail;
-
-		ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
-	}
 	return ret;
 }

@@ -6389,21 +6408,12 @@ static int netif_receive_skb_internal(struct sk_buff *skb)
 	if (skb_defer_rx_timestamp(skb))
 		return NET_RX_SUCCESS;

-	rcu_read_lock();
-#ifdef CONFIG_RPS
-	if (static_branch_unlikely(&rps_needed)) {
-		struct rps_dev_flow voidflow, *rflow = &voidflow;
-		int cpu = get_rps_cpu(skb->dev, skb, &rflow);
+	guard(rcu)();
+	ret = netif_rps(skb);
+	if (ret != NET_RX_UNHANDLED)
+		return ret;

-		if (cpu >= 0) {
-			ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
-			rcu_read_unlock();
-			return ret;
-		}
-	}
-#endif
 	ret = __netif_receive_skb(skb);
-	rcu_read_unlock();
 	return ret;
 }

@@ -6421,23 +6431,9 @@ void netif_receive_skb_list_internal(struct list_head *head)
 	}
 	list_splice_init(&sublist, head);

-	rcu_read_lock();
-#ifdef CONFIG_RPS
-	if (static_branch_unlikely(&rps_needed)) {
-		list_for_each_entry_safe(skb, next, head, list) {
-			struct rps_dev_flow voidflow, *rflow = &voidflow;
-			int cpu = get_rps_cpu(skb->dev, skb, &rflow);
-
-			if (cpu >= 0) {
-				/* Will be handled, remove from list */
-				skb_list_del_init(skb);
-				enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
-			}
-		}
-	}
-#endif
+	guard(rcu)();
+	netif_rps_list(head);
 	__netif_receive_skb_list(head);
-	rcu_read_unlock();
 }

 /**
--
2.25.1