[RFC PATCH net-next] net/smc: Introduce SMC-related proc files
From: Wen Gu
Date: Mon Sep 11 2023 - 03:20:43 EST
This patch introduces /proc/net/smc4 and /proc/net/smc6 files to report
statistic information of SMC connections.
Compared with 'smcss' command in smc-tools, getting SMC connections via
proc files is not efficient. However, in some container scenarios, some
dependencies are lacked for compiling and using smc-tools. In this case,
using proc files to check SMC connections becomes a simple and fast way.
Signed-off-by: Wen Gu <guwen@xxxxxxxxxxxxxxxxx>
---
include/net/smc.h | 5 +-
net/smc/Makefile | 2 +-
net/smc/af_smc.c | 22 ++++-
net/smc/smc_diag.c | 29 +++---
net/smc/smc_proc.c | 283 +++++++++++++++++++++++++++++++++++++++++++++++++++++
net/smc/smc_proc.h | 35 +++++++
6 files changed, 355 insertions(+), 21 deletions(-)
create mode 100644 net/smc/smc_proc.c
create mode 100644 net/smc/smc_proc.h
diff --git a/include/net/smc.h b/include/net/smc.h
index a002552..d29b332 100644
--- a/include/net/smc.h
+++ b/include/net/smc.h
@@ -20,10 +20,13 @@
struct sock;
#define SMC_MAX_PNETID_LEN 16 /* Max. length of PNET id */
+#define SMC_HTABLE_SHIFT 9
+#define SMC_HTABLE_SIZE (1 << SMC_HTABLE_SHIFT) /* Size of SMC hashtable buckets */
struct smc_hashinfo {
+ unsigned int bkt_idx;
rwlock_t lock;
- struct hlist_head ht;
+ struct hlist_head ht[SMC_HTABLE_SIZE];
};
int smc_hash_sk(struct sock *sk);
diff --git a/net/smc/Makefile b/net/smc/Makefile
index 875efcd..956810a 100644
--- a/net/smc/Makefile
+++ b/net/smc/Makefile
@@ -4,5 +4,5 @@ obj-$(CONFIG_SMC) += smc.o
obj-$(CONFIG_SMC_DIAG) += smc_diag.o
smc-y := af_smc.o smc_pnet.o smc_ib.o smc_clc.o smc_core.o smc_wr.o smc_llc.o
smc-y += smc_cdc.o smc_tx.o smc_rx.o smc_close.o smc_ism.o smc_netlink.o smc_stats.o
-smc-y += smc_tracepoint.o
+smc-y += smc_tracepoint.o smc_proc.o
smc-$(CONFIG_SYSCTL) += smc_sysctl.o
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index bacdd97..616f9a9 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -53,6 +53,7 @@
#include "smc_stats.h"
#include "smc_tracepoint.h"
#include "smc_sysctl.h"
+#include "smc_proc.h"
static DEFINE_MUTEX(smc_server_lgr_pending); /* serialize link group
* creation on server
@@ -182,9 +183,8 @@ int smc_hash_sk(struct sock *sk)
struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
struct hlist_head *head;
- head = &h->ht;
-
write_lock_bh(&h->lock);
+ head = &h->ht[h->bkt_idx++ & (SMC_HTABLE_SIZE - 1)];
sk_add_node(sk, head);
write_unlock_bh(&h->lock);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
@@ -3448,7 +3448,7 @@ static void __net_exit smc_net_stat_exit(struct net *net)
static int __init smc_init(void)
{
- int rc;
+ int rc, i;
rc = register_pernet_subsys(&smc_net_ops);
if (rc)
@@ -3520,8 +3520,11 @@ static int __init smc_init(void)
pr_err("%s: sock_register fails with %d\n", __func__, rc);
goto out_proto6;
}
- INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
- INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
+
+ for (i = 0; i < SMC_HTABLE_SIZE; i++) {
+ INIT_HLIST_HEAD(&smc_v4_hashinfo.ht[i]);
+ INIT_HLIST_HEAD(&smc_v6_hashinfo.ht[i]);
+ }
rc = smc_ib_register_client();
if (rc) {
@@ -3535,9 +3538,17 @@ static int __init smc_init(void)
goto out_ib;
}
+ rc = smc_proc_init();
+ if (rc) {
+ pr_err("%s: smc_proc_init fails with %d\n", __func__, rc);
+ goto out_ulp;
+ }
+
static_branch_enable(&tcp_have_smc);
return 0;
+out_ulp:
+ tcp_unregister_ulp(&smc_ulp_ops);
out_ib:
smc_ib_unregister_client();
out_sock:
@@ -3572,6 +3583,7 @@ static int __init smc_init(void)
static void __exit smc_exit(void)
{
static_branch_disable(&tcp_have_smc);
+ smc_proc_exit();
tcp_unregister_ulp(&smc_ulp_ops);
sock_unregister(PF_SMC);
smc_core_exit();
diff --git a/net/smc/smc_diag.c b/net/smc/smc_diag.c
index 7ff2152..8f2f8b8 100644
--- a/net/smc/smc_diag.c
+++ b/net/smc/smc_diag.c
@@ -197,24 +197,25 @@ static int smc_diag_dump_proto(struct proto *prot, struct sk_buff *skb,
int snum = cb_ctx->pos[p_type];
struct nlattr *bc = NULL;
struct hlist_head *head;
- int rc = 0, num = 0;
+ int rc = 0, num = 0, slot;
struct sock *sk;
read_lock(&prot->h.smc_hash->lock);
- head = &prot->h.smc_hash->ht;
- if (hlist_empty(head))
- goto out;
-
- sk_for_each(sk, head) {
- if (!net_eq(sock_net(sk), net))
- continue;
- if (num < snum)
- goto next;
- rc = __smc_diag_dump(sk, skb, cb, nlmsg_data(cb->nlh), bc);
- if (rc < 0)
- goto out;
+
+ for (slot = 0; slot < SMC_HTABLE_SIZE; slot++) {
+ head = &prot->h.smc_hash->ht[slot];
+
+ sk_for_each(sk, head) {
+ if (!net_eq(sock_net(sk), net))
+ continue;
+ if (num < snum)
+ goto next;
+ rc = __smc_diag_dump(sk, skb, cb, nlmsg_data(cb->nlh), bc);
+ if (rc < 0)
+ goto out;
next:
- num++;
+ num++;
+ }
}
out:
diff --git a/net/smc/smc_proc.c b/net/smc/smc_proc.c
new file mode 100644
index 0000000..6436b58
--- /dev/null
+++ b/net/smc/smc_proc.c
@@ -0,0 +1,283 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/init.h>
+#include <linux/proc_fs.h>
+#include <net/net_namespace.h>
+#include <net/sock.h>
+#include "smc.h"
+#include "smc_proc.h"
+#include "smc_core.h"
+
+static void *smc_proc_get_next(struct seq_file *seq, void *cur)
+{
+ struct smc_proc_private *sp = seq->private;
+ struct smc_hashinfo *smc_hash =
+ sp->protocol == SMCPROTO_SMC ?
+ smc_proto.h.smc_hash : smc_proto6.h.smc_hash;
+ struct net *net = seq_file_net(seq);
+ struct hlist_head *head;
+ struct sock *sk = cur;
+
+ if (!sk) {
+ read_lock(&smc_hash->lock);
+get_head:
+ head = &smc_hash->ht[sp->bucket];
+ sk = sk_head(head);
+ sp->offset = 0;
+ goto get_sk;
+ }
+ ++sp->num;
+ ++sp->offset;
+
+ sk = sk_next(sk);
+get_sk:
+ sk_for_each_from(sk) {
+ if (!net_eq(sock_net(sk), net))
+ continue;
+ return sk;
+ }
+ sp->offset = 0;
+ if (++sp->bucket < SMC_HTABLE_SIZE)
+ goto get_head;
+
+ read_unlock(&smc_hash->lock);
+ return NULL;
+}
+
+static void *smc_proc_seek_last_pos(struct seq_file *seq)
+{
+ struct smc_proc_private *sp = seq->private;
+ int offset = sp->offset;
+ int orig_num = sp->num;
+ void *rc = NULL;
+
+ if (sp->bucket >= SMC_HTABLE_SIZE)
+ goto out;
+
+ rc = smc_proc_get_next(seq, NULL);
+ while (offset-- && rc)
+ rc = smc_proc_get_next(seq, rc);
+
+ if (rc)
+ goto out;
+
+ sp->bucket = 0;
+out:
+ sp->num = orig_num;
+ return rc;
+}
+
+static void *smc_proc_get_idx(struct seq_file *seq, loff_t pos)
+{
+ struct smc_proc_private *sp = seq->private;
+ void *rc;
+
+ sp->bucket = 0;
+ rc = smc_proc_get_next(seq, NULL);
+
+ while (rc && pos) {
+ rc = smc_proc_get_next(seq, rc);
+ --pos;
+ }
+ return rc;
+}
+
+static void *__smc_proc_conn_start(struct seq_file *seq, loff_t *pos, int protocol)
+{
+ struct smc_proc_private *sp = seq->private;
+ void *rc;
+
+ if (*pos && *pos == sp->last_pos) {
+ rc = smc_proc_seek_last_pos(seq);
+ if (rc)
+ goto out;
+ }
+
+ sp->num = 0;
+ sp->bucket = 0;
+ sp->offset = 0;
+ sp->protocol = protocol;
+ rc = *pos ? smc_proc_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
+
+out:
+ sp->last_pos = *pos;
+ return rc;
+}
+
+static void *smc_proc_conn4_start(struct seq_file *seq, loff_t *pos)
+{
+ return __smc_proc_conn_start(seq, pos, SMCPROTO_SMC);
+}
+
+static void *smc_proc_conn6_start(struct seq_file *seq, loff_t *pos)
+{
+ return __smc_proc_conn_start(seq, pos, SMCPROTO_SMC6);
+}
+
+static void __smc_proc_conn_show(struct seq_file *seq, struct smc_sock *smc, int protocol)
+{
+ struct smc_proc_private *sp = seq->private;
+ const struct in6_addr *dest, *src;
+ struct smc_link_group *lgr;
+ struct socket *clcsock;
+ struct smc_link *lnk;
+ struct sock *sk;
+ bool fb = false;
+ int i;
+
+ fb = smc->use_fallback;
+ clcsock = smc->clcsock;
+ sk = &smc->sk;
+
+ if (protocol == SMCPROTO_SMC)
+ seq_printf(seq, CONN4_ADDR_FORMAT, sp->num,
+ clcsock->sk->sk_rcv_saddr, clcsock->sk->sk_num,
+ clcsock->sk->sk_daddr, ntohs(clcsock->sk->sk_dport));
+ else if (protocol == SMCPROTO_SMC6) {
+ dest = &clcsock->sk->sk_v6_daddr;
+ src = &clcsock->sk->sk_v6_rcv_saddr;
+ seq_printf(seq, CONN6_ADDR_FORMAT, sp->num,
+ src->s6_addr32[0], src->s6_addr32[1],
+ src->s6_addr32[2], src->s6_addr32[3], clcsock->sk->sk_num,
+ dest->s6_addr32[0], dest->s6_addr32[1],
+ dest->s6_addr32[2], dest->s6_addr32[3], ntohs(clcsock->sk->sk_dport));
+ }
+ seq_printf(seq, CONN_SK_FORMAT, fb ? 'Y' : 'N', fb ? smc->fallback_rsn : 0, sk,
+ clcsock->sk, fb ? clcsock->sk->sk_state : sk->sk_state, sock_i_ino(sk));
+
+ lgr = smc->conn.lgr;
+ lnk = smc->conn.lnk;
+
+ if (!fb && sk->sk_state == SMC_ACTIVE && lgr) {
+ for (i = 0; i < SMC_LGR_ID_SIZE; i++)
+ seq_printf(seq, "%02X", lgr->id[i]);
+
+ seq_printf(seq, CONN_LGR_FORMAT, lgr->smc_version > SMC_V1 ? "v2" : "v1",
+ lgr->is_smcd ? 'D' : 'R', lgr->role == SMC_CLNT ? 'C' : 'S');
+
+ if (!lgr->is_smcd && lnk)
+ seq_printf(seq, CONN_SMCR_EXT, lnk->ibname, lnk->ibport,
+ lnk->roce_qp->qp_num, lnk->peer_qpn);
+ }
+ seq_puts(seq, "\n");
+}
+
+static int smc_proc_conn_show(struct seq_file *seq, void *v)
+{
+ struct smc_proc_private *sp = seq->private;
+ struct socket *clcsock;
+ struct smc_sock *smc;
+
+ if (v == SEQ_START_TOKEN) {
+ seq_printf(seq, sp->protocol == SMCPROTO_SMC ? CONN4_HDR : CONN6_HDR,
+ "sl", "laddr", "raddr", "fb", "fbrsn", "sock", "clcsock",
+ "st", "inode", "lgr", "v", "t", "r");
+ goto out;
+ }
+
+ smc = smc_sk(v);
+ clcsock = smc->clcsock;
+ if (!clcsock)
+ goto out;
+
+ __smc_proc_conn_show(seq, smc, sp->protocol);
+out:
+ return 0;
+}
+
+static void *smc_proc_conn_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+ struct smc_proc_private *sp = seq->private;
+ void *rc = NULL;
+
+ if (v == SEQ_START_TOKEN) {
+ rc = smc_proc_get_idx(seq, 0);
+ goto out;
+ }
+ rc = smc_proc_get_next(seq, v);
+out:
+ ++*pos;
+ sp->last_pos = *pos;
+ return rc;
+}
+
+static void smc_proc_conn_stop(struct seq_file *seq, void *v)
+{
+ struct smc_proc_private *sp = seq->private;
+ struct smc_hashinfo *smc_hash =
+ sp->protocol == SMCPROTO_SMC ?
+ smc_proto.h.smc_hash : smc_proto6.h.smc_hash;
+
+ if (v && v != SEQ_START_TOKEN)
+ read_unlock(&smc_hash->lock);
+}
+
+static struct smc_proc_entry smc_proc[] = {
+ {
+ .name = "smc4",
+ .ops = {
+ .show = smc_proc_conn_show,
+ .start = smc_proc_conn4_start,
+ .next = smc_proc_conn_next,
+ .stop = smc_proc_conn_stop,
+ },
+ },
+#if IS_ENABLED(CONFIG_IPV6)
+ {
+ .name = "smc6",
+ .ops = {
+ .show = smc_proc_conn_show,
+ .start = smc_proc_conn6_start,
+ .next = smc_proc_conn_next,
+ .stop = smc_proc_conn_stop,
+ },
+ },
+#endif
+};
+
+static int __net_init smc_proc_dir_init(struct net *net)
+{
+#ifdef CONFIG_PROC_FS
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(smc_proc); i++) {
+ if (!proc_create_net(smc_proc[i].name, 0444,
+ net->proc_net, &smc_proc[i].ops,
+ sizeof(struct smc_proc_private)))
+ goto err;
+ }
+
+ return 0;
+
+err:
+ for (i -= 1; i >= 0; i--)
+ remove_proc_entry(smc_proc[i].name, net->proc_net);
+ return -ENOMEM;
+#else
+ return 0;
+#endif
+}
+
+static void __net_exit smc_proc_dir_exit(struct net *net)
+{
+#ifdef CONFIG_PROC_FS
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(smc_proc); i++)
+ remove_proc_entry(smc_proc[i].name, net->proc_net);
+#endif
+}
+
+static struct pernet_operations smc_proc_ops = {
+ .init = smc_proc_dir_init,
+ .exit = smc_proc_dir_exit,
+};
+
+int __init smc_proc_init(void)
+{
+ return register_pernet_subsys(&smc_proc_ops);
+}
+
+void smc_proc_exit(void)
+{
+ unregister_pernet_subsys(&smc_proc_ops);
+}
diff --git a/net/smc/smc_proc.h b/net/smc/smc_proc.h
new file mode 100644
index 0000000..5f97023
--- /dev/null
+++ b/net/smc/smc_proc.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _SMC_PROC_H_
+#define _SMC_PROC_H_
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/sysctl.h>
+#include <net/sock.h>
+#include <net/net_namespace.h>
+#include "smc.h"
+
+#define CONN4_HDR ("%4s: %-14s%-14s%-3s%-9s%-17s%-17s%-3s%-8s%-9s%-3s%-2s%-2s\n")
+#define CONN6_HDR ("%4s: %-38s%-38s%-3s%-9s%-17s%-17s%-3s%-8s%-9s%-3s%-2s%-2s\n")
+#define CONN4_ADDR_FORMAT ("%4d: %08X:%04X %08X:%04X")
+#define CONN6_ADDR_FORMAT ("%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X")
+#define CONN_SK_FORMAT (" %c %08X %pK %pK %2d %-7lu ")
+#define CONN_LGR_FORMAT (" %-2s %c %c")
+#define CONN_SMCR_EXT (" %-8s %02d %-4X %-4X")
+
+struct smc_proc_private {
+ struct seq_net_private p;
+ int num, bucket, offset;
+ int protocol;
+ loff_t last_pos;
+};
+
+struct smc_proc_entry {
+ const char *name;
+ const struct seq_operations ops;
+};
+
+int __init smc_proc_init(void);
+void smc_proc_exit(void);
+
+#endif
--
1.8.3.1