Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes
From: Zhu Yanjun
Date: Fri Apr 10 2026 - 21:46:43 EST
在 2026/4/10 15:37, yanjun.zhu 写道:
On 4/7/26 5:09 PM, zhenwei pi wrote:
There is a lack of sent/received counter in bytes.
Signed-off-by: zhenwei pi <zhenwei.pi@xxxxxxxxx>
---
drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++
drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++
drivers/infiniband/sw/rxe/rxe_net.c | 2 ++
drivers/infiniband/sw/rxe/rxe_recv.c | 2 ++
drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++
5 files changed, 14 insertions(+)
diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
index 437917a7d8f2..17edaa9a9b9b 100644
--- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c
+++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = {
[RXE_CNT_LINK_DOWNED].name = "link_downed",
[RXE_CNT_RDMA_SEND].name = "rdma_sends",
[RXE_CNT_RDMA_RECV].name = "rdma_recvs",
+ [RXE_CNT_SENT_BYTES].name = "sent_bytes",
+ [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes",
};
int rxe_ib_get_hw_stats(struct ib_device *ibdev,
diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/infiniband/sw/rxe/rxe_hw_counters.h
index 051f9e1c3852..01b355103cbc 100644
--- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h
+++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h
@@ -26,6 +26,8 @@ enum rxe_counters {
RXE_CNT_LINK_DOWNED,
RXE_CNT_RDMA_SEND,
RXE_CNT_RDMA_RECV,
+ RXE_CNT_SENT_BYTES,
+ RXE_CNT_RCVD_BYTES,
RXE_NUM_OF_COUNTERS
};
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 6621d01ac32d..86660031ffa2 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
int err;
int is_request = pkt->mask & RXE_REQ_MASK;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
+ unsigned int skblen = skb->len;
unsigned long flags;
spin_lock_irqsave(&qp->state_lock, flags);
@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
}
rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
+ rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen);
goto done;
drop:
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index 5861e4244049..e7bab89e7d8d 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)
int err;
struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
struct rxe_dev *rxe = pkt->rxe;
+ unsigned int skblen = skb->len - skb_network_offset(skb);
if (unlikely(skb->len < RXE_BTH_BYTES))
goto drop;
@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
if (unlikely(err))
goto drop;
+ rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))
int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
struct sk_buff *skb)
{
int err;
int is_request = pkt->mask & RXE_REQ_MASK;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
unsigned long flags;
skb->len is printed here, that is len1
...
if (pkt->mask & RXE_LOOPBACK_MASK)
err = rxe_loopback(skb, pkt);
else
err = rxe_send(skb, pkt);
...
}
In the following function
static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt)
{
...
if (skb->protocol == htons(ETH_P_IP))
skb_pull(skb, sizeof(struct iphdr));
else
skb_pull(skb, sizeof(struct ipv6hdr));
...
/* remove udp header */
skb_pull(skb, sizeof(struct udphdr));
print skb->len here, that is len2
rxe_rcv(skb);
...
}
Does len1 equal to len2?
I have made tests. The difference between len1 and len2 is 28.
It should be the total of ipv4 header + udp header because I use ipv4 address to make tests.
I am not sure if the bytes of recv data should equal to the bytes of xmit data.
Zhu Yanjun
If not, the transmitted length appears to differ from the received length when using loopback.
I am not sure whether this is expected behavior.
The same observation also applies to the non-loopback case.
Zhu Yanjun
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index e800545d1046..0f5ffd94643f 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
atomic64_inc(&rxe->stats_counters[index]);
}
+static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
+ s64 val)
+{
+ atomic64_add(val, &rxe->stats_counters[index]);
+}
+
static inline struct rxe_dev *to_rdev(struct ib_device *dev)
{
return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
--
Best Regards,
Yanjun.Zhu