[PATCH 8/8] smb: server: make use of IPPROTO_SMBDIRECT sockets

[Stefan Metzmacher]

This reduces the function calls for special smbdirect features to
- smbdirect_socket_from_sock()
- smbdirect_socket_set_logging()
- smbdirect_socket_set_initial_parameters()
- smbdirect_socket_set_kernel_settings()
- smbdirect_socket_get_current_parameters()
- smbdirect_connection_is_connected()
- smbdirect_connection_rdma_xmit()

This will also make it easier to implement the QUIC
transport Henrique Carvalho is working on.

In the end the core smb handling should just make
use of a SOCK_STREAM socket, after the initial setup.

There's still a way to go, but almost all changes
will be done within the smbdirect code, e.g.
adding MSG_SPLICE_PAGES support or
splice_read/read_sock/read_skb.

But it's a good start, which will make changes
much easier.

Cc: Steve French <smfrench@xxxxxxxxx>
Cc: Tom Talpey <tom@xxxxxxxxxx>
Cc: Long Li <longli@xxxxxxxxxxxxx>
Cc: Namjae Jeon <linkinjeon@xxxxxxxxxx>
Cc: David Howells <dhowells@xxxxxxxxxx>
Cc: Henrique Carvalho <henrique.carvalho@xxxxxxxx>
Cc: linux-cifs@xxxxxxxxxxxxxxx
Cc: samba-technical@xxxxxxxxxxxxxxx
Cc: David S. Miller <davem@xxxxxxxxxxxxx>
Cc: Eric Dumazet <edumazet@xxxxxxxxxx>
Cc: Jakub Kicinski <kuba@xxxxxxxxxx>
Cc: Paolo Abeni <pabeni@xxxxxxxxxx>
Cc: Simon Horman <horms@xxxxxxxxxx>
Cc: Kuniyuki Iwashima <kuniyu@xxxxxxxxxx>
Cc: Willem de Bruijn <willemb@xxxxxxxxxx>
Cc: netdev@xxxxxxxxxxxxxxx
Cc: Xin Long <lucien.xin@xxxxxxxxx>
Cc: quic@xxxxxxxxxxxxxxx
Cc: linux-rdma@xxxxxxxxxxxxxxx
Cc: linux-kernel@xxxxxxxxxxxxxxx
Signed-off-by: Stefan Metzmacher <metze@xxxxxxxxx>
---
 fs/smb/server/transport_rdma.c | 119 +++++++++++++++++++--------------
 1 file changed, 70 insertions(+), 49 deletions(-)

diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c
index 706a2c897948..64be48165cda 100644
--- a/fs/smb/server/transport_rdma.c
+++ b/fs/smb/server/transport_rdma.c
@@ -82,13 +82,13 @@ static struct smb_direct_listener {
 
 	struct task_struct	*thread;
 
-	struct smbdirect_socket *socket;
+	struct socket		*sock;
 } smb_direct_ib_listener, smb_direct_iw_listener;
 
 struct smb_direct_transport {
 	struct ksmbd_transport	transport;
 
-	struct smbdirect_socket *socket;
+	struct socket		*sock;
 };
 
 static bool smb_direct_logging_needed(struct smbdirect_socket *sc,
@@ -165,18 +165,23 @@ void init_smbd_max_io_size(unsigned int sz)
 unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt)
 {
 	struct smb_direct_transport *t;
+	struct smbdirect_socket *sc;
 	const struct smbdirect_socket_parameters *sp;
 
 	if (kt->ops != &ksmbd_smb_direct_transport_ops)
 		return 0;
 
 	t = SMBD_TRANS(kt);
-	sp = smbdirect_socket_get_current_parameters(t->socket);
+	sc = smbdirect_socket_from_sock(t->sock);
+
+	if (!smbdirect_connection_is_connected(sc))
+		return 0;
+	sp = smbdirect_socket_get_current_parameters(sc);
 
 	return sp->max_read_write_size;
 }
 
-static struct smb_direct_transport *alloc_transport(struct smbdirect_socket *sc)
+static struct smb_direct_transport *alloc_transport(struct socket *client_sock)
 {
 	struct smb_direct_transport *t;
 	struct ksmbd_conn *conn;
@@ -184,7 +189,7 @@ static struct smb_direct_transport *alloc_transport(struct smbdirect_socket *sc)
 	t = kzalloc_obj(*t, KSMBD_DEFAULT_GFP);
 	if (!t)
 		return NULL;
-	t->socket = sc;
+	t->sock = client_sock;
 
 	conn = ksmbd_conn_alloc();
 	if (!conn)
@@ -209,13 +214,13 @@ static void smb_direct_free_transport(struct ksmbd_transport *kt)
 {
 	struct smb_direct_transport *t = SMBD_TRANS(kt);
 
-	smbdirect_socket_release(t->socket);
+	sock_release(t->sock);
 	kfree(t);
 }
 
 static void free_transport(struct smb_direct_transport *t)
 {
-	smbdirect_socket_shutdown(t->socket);
+	kernel_sock_shutdown(t->sock, SHUT_RDWR);
 	ksmbd_conn_free(KSMBD_TRANS(t)->conn);
 }
 
@@ -223,7 +228,6 @@ static int smb_direct_read(struct ksmbd_transport *t, char *buf,
 			   unsigned int size, int unused)
 {
 	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = st->socket;
 	struct msghdr msg = { .msg_flags = 0, };
 	struct kvec iov = {
 		.iov_base = buf,
@@ -231,9 +235,7 @@ static int smb_direct_read(struct ksmbd_transport *t, char *buf,
 	};
 	int ret;
 
-	iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, size);
-
-	ret = smbdirect_connection_recvmsg(sc, &msg, 0);
+	ret = kernel_recvmsg(st->sock, &msg, &iov, 1, size, 0);
 	if (ret == -ERESTARTSYS)
 		ret = -EINTR;
 	return ret;
@@ -244,13 +246,14 @@ static int smb_direct_writev(struct ksmbd_transport *t,
 			     bool need_invalidate, unsigned int remote_key)
 {
 	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = st->socket;
-	struct iov_iter iter;
+	struct msghdr msg = { .msg_flags = MSG_NOSIGNAL, };
+	struct smbdirect_cmsg_buffer cbuffer = { .msg_control = {0, }, };
+	u32 _remote_token = remote_key;
+	const u32 *remote_token = need_invalidate ? &_remote_token : NULL;
 
-	iov_iter_kvec(&iter, ITER_SOURCE, iov, niovs, buflen);
+	smbdirect_buffer_remote_invalidate_cmsg_prepare(&msg, &cbuffer, remote_token);
 
-	return smbdirect_connection_send_iter(sc, &iter, 0,
-					      need_invalidate, remote_key);
+	return kernel_sendmsg(st->sock, &msg, iov, niovs, buflen);
 }
 
 static int smb_direct_rdma_write(struct ksmbd_transport *t,
@@ -259,7 +262,10 @@ static int smb_direct_rdma_write(struct ksmbd_transport *t,
 				 unsigned int desc_len)
 {
 	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = st->socket;
+	struct smbdirect_socket *sc = smbdirect_socket_from_sock(st->sock);
+
+	if (!smbdirect_connection_is_connected(sc))
+		return -ENOTCONN;
 
 	return smbdirect_connection_rdma_xmit(sc, buf, buflen,
 					      desc, desc_len, false);
@@ -271,7 +277,10 @@ static int smb_direct_rdma_read(struct ksmbd_transport *t,
 				unsigned int desc_len)
 {
 	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = st->socket;
+	struct smbdirect_socket *sc = smbdirect_socket_from_sock(st->sock);
+
+	if (!smbdirect_connection_is_connected(sc))
+		return -ENOTCONN;
 
 	return smbdirect_connection_rdma_xmit(sc, buf, buflen,
 					      desc, desc_len, true);
@@ -280,7 +289,7 @@ static int smb_direct_rdma_read(struct ksmbd_transport *t,
 static void smb_direct_disconnect(struct ksmbd_transport *t)
 {
 	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = st->socket;
+	struct smbdirect_socket *sc = smbdirect_socket_from_sock(st->sock);
 
 	ksmbd_debug(RDMA, "Disconnecting sc=%p\n", sc);
 
@@ -290,23 +299,23 @@ static void smb_direct_disconnect(struct ksmbd_transport *t)
 static void smb_direct_shutdown(struct ksmbd_transport *t)
 {
 	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = st->socket;
+	struct smbdirect_socket *sc = smbdirect_socket_from_sock(st->sock);
 
 	ksmbd_debug(RDMA, "smb-direct shutdown sc=%p\n", sc);
 
-	smbdirect_socket_shutdown(sc);
+	kernel_sock_shutdown(st->sock, SHUT_RDWR);
 }
 
 static int smb_direct_new_connection(struct smb_direct_listener *listener,
-				     struct smbdirect_socket *client_sc)
+				     struct socket *client_sock)
 {
 	struct smb_direct_transport *t;
 	struct task_struct *handler;
 	int ret;
 
-	t = alloc_transport(client_sc);
+	t = alloc_transport(client_sock);
 	if (!t) {
-		smbdirect_socket_release(client_sc);
+		sock_release(client_sock);
 		return -ENOMEM;
 	}
 
@@ -328,22 +337,21 @@ static int smb_direct_new_connection(struct smb_direct_listener *listener,
 static int smb_direct_listener_kthread_fn(void *p)
 {
 	struct smb_direct_listener *listener = (struct smb_direct_listener *)p;
-	struct smbdirect_socket *client_sc = NULL;
+	struct socket *client_sock = NULL;
 
 	while (!kthread_should_stop()) {
-		struct proto_accept_arg arg = { .err = -EINVAL, };
-		long timeo = MAX_SCHEDULE_TIMEOUT;
+		int ret;
 
-		if (!listener->socket)
+		if (!listener->sock)
 			break;
-		client_sc = smbdirect_socket_accept(listener->socket, timeo, &arg);
-		if (!client_sc && arg.err == -EINVAL)
+		ret = kernel_accept(listener->sock, &client_sock, 0);
+		if (ret == -EINVAL)
 			break;
-		if (!client_sc)
+		if (ret)
 			continue;
 
 		ksmbd_debug(CONN, "connect success: accepted new connection\n");
-		smb_direct_new_connection(listener, client_sc);
+		smb_direct_new_connection(listener, client_sock);
 	}
 
 	ksmbd_debug(CONN, "releasing socket\n");
@@ -354,8 +362,12 @@ static void smb_direct_listener_destroy(struct smb_direct_listener *listener)
 {
 	int ret;
 
-	if (listener->socket)
-		smbdirect_socket_shutdown(listener->socket);
+	if (listener->sock) {
+		ret = kernel_sock_shutdown(listener->sock, SHUT_RDWR);
+		if (ret)
+			pr_err("Failed to shutdown socket: %d %1pe\n",
+			       ret, ERR_PTR(ret));
+	}
 
 	if (listener->thread) {
 		ret = kthread_stop(listener->thread);
@@ -364,9 +376,9 @@ static void smb_direct_listener_destroy(struct smb_direct_listener *listener)
 		listener->thread = NULL;
 	}
 
-	if (listener->socket) {
-		smbdirect_socket_release(listener->socket);
-		listener->socket = NULL;
+	if (listener->sock) {
+		sock_release(listener->sock);
+		listener->sock = NULL;
 	}
 
 	listener->port = 0;
@@ -376,6 +388,7 @@ static int smb_direct_listen(struct smb_direct_listener *listener,
 			     int port)
 {
 	struct net *net = current->nsproxy->net_ns;
+	struct socket *ksmbd_socket;
 	struct task_struct *kthread;
 	struct sockaddr_in sin = {
 		.sin_family		= AF_INET,
@@ -410,13 +423,21 @@ static int smb_direct_listen(struct smb_direct_listener *listener,
 		return -ENODEV;
 	}
 
-	ret = smbdirect_socket_create_kern(net, &sc);
+	ret = sock_create_kern(net, PF_INET, SOCK_STREAM, IPPROTO_SMBDIRECT,
+			       &ksmbd_socket);
 	if (ret) {
-		pr_err("smbdirect_socket_create_kern() failed: %d %1pe\n",
+		pr_err("Can't create IPPROTO_SMBDIRECT socket: %d %1pe\n",
 		       ret, ERR_PTR(ret));
 		return ret;
 	}
 
+	sc = smbdirect_socket_from_sock(ksmbd_socket);
+	if (WARN_ON_ONCE(!sc)) {
+		ret = -EINVAL;
+		pr_err("smbdirect_socket_from_sock returned NULL\n");
+		goto err;
+	}
+
 	/*
 	 * Create the initial parameters
 	 */
@@ -450,22 +471,22 @@ static int smb_direct_listen(struct smb_direct_listener *listener,
 		goto err;
 	}
 
-	ret = smbdirect_socket_bind(sc, (struct sockaddr *)&sin);
+	ret = kernel_bind(ksmbd_socket, (struct sockaddr_unsized *)&sin, sizeof(sin));
 	if (ret) {
-		pr_err("smbdirect_socket_bind() failed: %d %1pe\n",
+		pr_err("Failed to bind IPPROTO_SMBDIRECT socket: %d %1pe\n",
 		       ret, ERR_PTR(ret));
 		goto err;
 	}
 
-	ret = smbdirect_socket_listen(sc, 10);
+	ret = kernel_listen(ksmbd_socket, 10);
 	if (ret) {
-		pr_err("Port[%d] smbdirect_socket_listen() failed: %d %1pe\n",
+		pr_err("Port[%d] kernel_listen() error: %d %1pe\n",
 		       port, ret, ERR_PTR(ret));
 		goto err;
 	}
 
 	listener->port = port;
-	listener->socket = sc;
+	listener->sock = ksmbd_socket;
 
 	kthread = kthread_run(smb_direct_listener_kthread_fn,
 			      listener,
@@ -489,7 +510,7 @@ int ksmbd_rdma_init(void)
 	int ret;
 
 	smb_direct_ib_listener = smb_direct_iw_listener = (struct smb_direct_listener) {
-		.socket = NULL,
+		.sock = NULL,
 	};
 
 	ret = smb_direct_listen(&smb_direct_ib_listener,
@@ -499,8 +520,8 @@ int ksmbd_rdma_init(void)
 		goto err;
 	}
 
-	ksmbd_debug(RDMA, "InfiniBand/RoCEv1/RoCEv2 RDMA listener. socket=%p\n",
-		    smb_direct_ib_listener.socket);
+	ksmbd_debug(RDMA, "InfiniBand/RoCEv1/RoCEv2 RDMA listener. sock=%p\n",
+		    smb_direct_ib_listener.sock);
 
 	ret = smb_direct_listen(&smb_direct_iw_listener,
 				SMB_DIRECT_PORT_IWARP);
@@ -509,8 +530,8 @@ int ksmbd_rdma_init(void)
 		goto err;
 	}
 
-	ksmbd_debug(RDMA, "iWarp RDMA listener. socket=%p\n",
-		    smb_direct_iw_listener.socket);
+	ksmbd_debug(RDMA, "iWarp RDMA listener. sock=%p\n",
+		    smb_direct_iw_listener.sock);
 
 	return 0;
 err:
-- 
2.43.0