[PATCH nf-next 2/3] netfilter: flowtable: update upper device stats in the fast path

[Julius Bairaktaris]

The software fast path receives and transmits on the lowest devices of
the input and output stacks, so the bridge, VLAN, PPPoE and tunnel
devices above them stop counting after the first packets of a
connection.

Store the ifindexes of these devices from the forward path and update
their counters from the fast path. Each device counts what it counts in
the classic path: the IP packet plus the headers of the devices above it
(PPPoE, outer IP, inner VLAN tag), plus the Ethernet header on transmit
for Ethernet devices.

A GSO packet sent through a PPPoE device is counted once. The classic
path segments it in front of the ppp device and counts every segment.

struct flow_offload grows from 296 to 328 bytes.

Assisted-by: Claude:claude-fable-5-1
Signed-off-by: Julius Bairaktaris <julius@xxxxxxxxxxxxxx>
---
 include/net/netfilter/nf_flow_table.h |  8 +++-
 net/netfilter/nf_flow_table_core.c    |  3 ++
 net/netfilter/nf_flow_table_ip.c      | 53 +++++++++++++++++++++++++++
 net/netfilter/nf_flow_table_path.c    |  9 +++++
 4 files changed, 72 insertions(+), 1 deletion(-)

diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h
index f2e2771f188f..1f451fcf05d0 100644
--- a/include/net/netfilter/nf_flow_table.h
+++ b/include/net/netfilter/nf_flow_table.h
@@ -106,6 +106,8 @@ enum flow_offload_xmit_type {
 };
 
 #define NF_FLOW_TABLE_ENCAP_MAX		2
+/* Devices above the flowtable device in a forward path. */
+#define NF_FLOW_TABLE_UPPER_MAX		(NET_DEVICE_PATH_STACK_MAX - 1)
 
 struct flow_offload_tunnel {
 	union {
@@ -153,7 +155,8 @@ struct flow_offload_tuple {
 					encap_num:2,
 					needs_gso_segment:1,
 					tun_num:2,
-					in_vlan_ingress:2;
+					in_vlan_ingress:2,
+					num_uppers:3;
 	u16				mtu;
 	u32				dst_cookie;
 	struct dst_entry		*dst_cache;
@@ -171,6 +174,7 @@ struct flow_offload_tuple {
 			u32		iifidx;
 		} tc;
 	};
+	u32				upper_ifidx[NF_FLOW_TABLE_UPPER_MAX];
 };
 
 struct flow_offload_tuple_rhash {
@@ -228,6 +232,8 @@ struct nf_flow_route {
 			u8			num_encaps:2,
 						num_tuns:2,
 						ingress_vlans:2;
+			u32			upper_ifidx[NF_FLOW_TABLE_UPPER_MAX];
+			u8			num_uppers;
 		} in;
 		struct {
 			u32			ifindex;
diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c
index 03241d4bfd5e..2c9a0d97c9fb 100644
--- a/net/netfilter/nf_flow_table_core.c
+++ b/net/netfilter/nf_flow_table_core.c
@@ -122,6 +122,9 @@ static int flow_offload_fill_route(struct flow_offload *flow,
 
 	flow_tuple->tun = route->tuple[dir].in.tun;
 	flow_tuple->encap_num = route->tuple[dir].in.num_encaps;
+	memcpy(flow_tuple->upper_ifidx, route->tuple[dir].in.upper_ifidx,
+	       sizeof(flow_tuple->upper_ifidx));
+	flow_tuple->num_uppers = route->tuple[dir].in.num_uppers;
 	flow_tuple->needs_gso_segment = route->tuple[dir].out.needs_gso_segment;
 	flow_tuple->tun_num = route->tuple[dir].in.num_tuns;
 
diff --git a/net/netfilter/nf_flow_table_ip.c b/net/netfilter/nf_flow_table_ip.c
index c8c29a9a1684..e1a3511d5f1f 100644
--- a/net/netfilter/nf_flow_table_ip.c
+++ b/net/netfilter/nf_flow_table_ip.c
@@ -453,6 +453,49 @@ static void nf_flow_encap_pop(struct nf_flowtable_ctx *ctx,
 		nf_flow_ip_tunnel_pop(ctx, skb);
 }
 
+/* The fast path bypasses the devices above the flowtable device. */
+static void nf_flow_upper_stats_add(struct net *net,
+				    const struct flow_offload_tuple *tuple,
+				    bool rx, unsigned int len)
+{
+	unsigned int n, vlan_hlen = 0;
+	struct net_device *dev;
+	int i;
+
+	for (i = 0; i < tuple->num_uppers; i++) {
+		dev = dev_get_by_index_rcu(net, tuple->upper_ifidx[i]);
+		if (!dev)
+			continue;
+
+		n = len;
+		if (!rx && dev->type == ARPHRD_ETHER)
+			n += ETH_HLEN;
+
+		if (is_vlan_dev(dev)) {
+			/* Q-in-Q: the outer VLAN device counts the inner tag. */
+			n += vlan_hlen;
+			if (rx)
+				vlan_dev_sw_netstats_rx_add(dev, n);
+			else
+				vlan_dev_sw_netstats_tx_add(dev, 1, n);
+			vlan_hlen += VLAN_HLEN;
+		} else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) {
+			if (rx)
+				dev_sw_netstats_rx_add(dev, n);
+			else
+				dev_sw_netstats_tx_add(dev, 1, n);
+		}
+
+		/* The devices below also count this device's header. */
+		if (dev->type == ARPHRD_PPP)
+			len += PPPOE_SES_HLEN;
+		else if (dev->type == ARPHRD_TUNNEL)
+			len += sizeof(struct iphdr);
+		else if (dev->type == ARPHRD_TUNNEL6)
+			len += sizeof(struct ipv6hdr);
+	}
+}
+
 static struct flow_offload_tuple_rhash *
 nf_flow_offload_lookup(struct nf_flowtable_ctx *ctx,
 		       struct nf_flowtable *flow_table, struct sk_buff *skb)
@@ -512,6 +555,11 @@ static int nf_flow_offload_forward(struct nf_flowtable_ctx *ctx,
 	if (flow_table->flags & NF_FLOWTABLE_COUNTER)
 		nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len);
 
+	nf_flow_upper_stats_add(dev_net(ctx->in), &tuplehash->tuple, true,
+				skb->len);
+	nf_flow_upper_stats_add(dev_net(ctx->in), &flow->tuplehash[!dir].tuple,
+				false, skb->len);
+
 	return 1;
 }
 
@@ -1107,6 +1155,11 @@ static int nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx *ctx,
 	if (flow_table->flags & NF_FLOWTABLE_COUNTER)
 		nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len);
 
+	nf_flow_upper_stats_add(dev_net(ctx->in), &tuplehash->tuple, true,
+				skb->len);
+	nf_flow_upper_stats_add(dev_net(ctx->in), &flow->tuplehash[!dir].tuple,
+				false, skb->len);
+
 	return 1;
 }
 
diff --git a/net/netfilter/nf_flow_table_path.c b/net/netfilter/nf_flow_table_path.c
index 1e55644f2edb..ba5ac3ebe614 100644
--- a/net/netfilter/nf_flow_table_path.c
+++ b/net/netfilter/nf_flow_table_path.c
@@ -83,6 +83,8 @@ static int nft_dev_fill_forward_path(const struct dst_entry *dst_cache,
 
 struct nft_forward_info {
 	const struct net_device *dev;
+	u32 upper_ifidx[NF_FLOW_TABLE_UPPER_MAX];
+	u8 num_uppers;
 	struct id {
 		__u16	id;
 		__be16	proto;
@@ -187,6 +189,10 @@ static int nft_dev_path_info(struct net_device_path_stack *stack,
 		}
 	}
 
+	for (i = 0; info->dev && stack->path[i].dev != info->dev; i++)
+		info->upper_ifidx[i] = stack->path[i].dev->ifindex;
+	info->num_uppers = i;
+
 	if (nf_flowtable_hw_offload(&ft->data) &&
 	    nft_is_valid_ether_device(info->dev))
 		info->xmit_type = FLOW_OFFLOAD_XMIT_DIRECT;
@@ -253,6 +259,9 @@ static int nft_dev_forward_path(const struct nft_pktinfo *pkt,
 
 	route->tuple[!dir].in.num_encaps = info.num_encaps;
 	route->tuple[!dir].in.ingress_vlans = info.ingress_vlans;
+	memcpy(route->tuple[!dir].in.upper_ifidx, info.upper_ifidx,
+	       sizeof(info.upper_ifidx));
+	route->tuple[!dir].in.num_uppers = info.num_uppers;
 
 	if (info.xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) {
 		memcpy(route->tuple[dir].out.h_source, info.h_source, ETH_ALEN);
-- 
2.53.0