[PATCH 1/1] Add vhost_blk driver

[Vitaly Mayatskikh]

This driver accelerates host side of virtio-blk.

Signed-off-by: Vitaly Mayatskikh <v.mayatskih@xxxxxxxxx>
---
 drivers/vhost/Kconfig  |  13 ++
 drivers/vhost/Makefile |   3 +
 drivers/vhost/blk.c    | 510 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 526 insertions(+)
 create mode 100644 drivers/vhost/blk.c

diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig
index b580885243f7..c4980d6af0ea 100644
--- a/drivers/vhost/Kconfig
+++ b/drivers/vhost/Kconfig
@@ -53,3 +53,16 @@ config VHOST_CROSS_ENDIAN_LEGACY
 	  adds some overhead, it is disabled by default.
 
 	  If unsure, say "N".
+
+config VHOST_BLK
+	tristate "Host kernel accelerator for virtio blk (EXPERIMENTAL)"
+	depends on BLOCK && EVENTFD
+	select VHOST
+	default n
+	help
+	 This kernel module can be loaded in host kernel to accelerate
+	 guest block with virtio_blk. Not to be confused with virtio_blk
+	 module itself which needs to be loaded in guest kernel.
+
+	 To compile this driver as a module, choose M here: the module will
+	 be called vhost_blk.
diff --git a/drivers/vhost/Makefile b/drivers/vhost/Makefile
index 6c6df24f770c..c8be36cd9214 100644
--- a/drivers/vhost/Makefile
+++ b/drivers/vhost/Makefile
@@ -8,6 +8,9 @@ vhost_scsi-y := scsi.o
 obj-$(CONFIG_VHOST_VSOCK) += vhost_vsock.o
 vhost_vsock-y := vsock.o
 
+obj-$(CONFIG_VHOST_BLK) += vhost_blk.o
+vhost_blk-y := blk.o
+
 obj-$(CONFIG_VHOST_RING) += vringh.o
 
 obj-$(CONFIG_VHOST)	+= vhost.o
diff --git a/drivers/vhost/blk.c b/drivers/vhost/blk.c
new file mode 100644
index 000000000000..aefb9a61fa0f
--- /dev/null
+++ b/drivers/vhost/blk.c
@@ -0,0 +1,510 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 IBM Corporation
+ * Author: Vitaly Mayatskikh <v.mayatskih@xxxxxxxxx>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ *
+ * virtio-blk server in host kernel.
+ */
+
+#include <linux/module.h>
+#include <linux/miscdevice.h>
+#include <linux/virtio_blk.h>
+#include <linux/vhost.h>
+#include <linux/fs.h>
+#include "vhost.h"
+
+enum {
+	VHOST_BLK_FEATURES =
+	VHOST_FEATURES |
+	(1ULL << VIRTIO_F_IOMMU_PLATFORM) |
+	(1ULL << VIRTIO_RING_F_INDIRECT_DESC) |
+	(1ULL << VIRTIO_RING_F_EVENT_IDX) |
+	(1ULL << VIRTIO_BLK_F_MQ)
+};
+
+#define VHOST_BLK_SET_BACKEND _IOW(VHOST_VIRTIO, 0x50, int)
+
+enum {
+	VHOST_BLK_VQ_MAX = 16,
+	VHOST_BLK_VQ_MAX_REQS = 128,
+};
+
+struct vhost_blk_req {
+	struct llist_node list;
+	int index;
+	struct vhost_blk_queue *q;
+	struct virtio_blk_outhdr hdr;
+	struct iovec *out_iov;
+	struct iovec *in_iov;
+	u8 out_num;
+	u8 in_num;
+	long len;
+	struct kiocb iocb;
+	struct iov_iter i;
+	int res;
+	void __user *status;
+};
+
+struct vhost_blk_queue {
+	int index;
+	struct vhost_blk *blk;
+	struct vhost_virtqueue vq;
+	struct vhost_work w;
+	struct llist_head wl;
+	struct vhost_blk_req req[VHOST_BLK_VQ_MAX_REQS];
+};
+
+struct vhost_blk {
+	struct vhost_dev dev;
+	struct file *backend;
+	int num_queues;
+	struct vhost_virtqueue *vqs[VHOST_BLK_VQ_MAX];
+	struct vhost_blk_queue queue[VHOST_BLK_VQ_MAX];
+};
+
+static void vhost_blk_flush(struct vhost_blk *blk)
+{
+	int i;
+
+	for (i = 0; i < blk->num_queues; i++)
+		vhost_poll_flush(&blk->queue[i].vq.poll);
+}
+
+
+static void vhost_blk_stop(struct vhost_blk *blk)
+{
+	struct vhost_virtqueue *vq;
+	int i;
+
+	for (i = 0; i < blk->num_queues; i++) {
+		vq = &blk->queue[i].vq;
+		mutex_lock(&vq->mutex);
+		rcu_assign_pointer(vq->private_data, NULL);
+		mutex_unlock(&vq->mutex);
+	}
+}
+
+static int vhost_blk_req_done(struct vhost_blk_req *req, unsigned char status)
+{
+	int ret;
+	int len = req->len;
+
+	pr_debug("%s vq[%d] req->index %d status %d len %d\n", __func__,
+		 req->q->index, req->index, status, len);
+	ret = put_user(status, (unsigned char __user *)req->status);
+
+	WARN(ret, "%s: vq[%d] req->index %d failed to write status\n", __func__,
+	     req->q->index, req->index);
+
+	vhost_add_used(&req->q->vq, req->index, len);
+
+	return ret;
+}
+
+static void vhost_blk_io_done_work(struct vhost_work *w)
+{
+	struct vhost_blk_queue *q = container_of(w, struct vhost_blk_queue, w);
+	struct llist_node *node;
+	struct vhost_blk_req *req, *tmp;
+
+	node = llist_del_all(&q->wl);
+	llist_for_each_entry_safe(req, tmp, node, list) {
+		vhost_blk_req_done(req, req->res);
+	}
+	vhost_signal(&q->blk->dev, &q->vq);
+}
+
+static void vhost_blk_iocb_complete(struct kiocb *iocb, long ret, long ret2)
+{
+	struct vhost_blk_req *req = container_of(iocb, struct vhost_blk_req,
+						 iocb);
+
+	pr_debug("%s vq[%d] req->index %d ret %ld ret2 %ld\n", __func__,
+		 req->q->index, req->index, ret, ret2);
+
+	req->res = (ret == req->len) ? VIRTIO_BLK_S_OK : VIRTIO_BLK_S_IOERR;
+	llist_add(&req->list, &req->q->wl);
+	vhost_vq_work_queue(&req->q->vq, &req->q->w);
+}
+
+static int vhost_blk_req_handle(struct vhost_blk_req *req)
+{
+	struct vhost_blk *blk = req->q->blk;
+	struct vhost_virtqueue *vq = &req->q->vq;
+	int type = le32_to_cpu(req->hdr.type);
+	int ret;
+	u8 status;
+
+	if ((type == VIRTIO_BLK_T_IN) || (type == VIRTIO_BLK_T_OUT)) {
+		bool write = (type == VIRTIO_BLK_T_OUT);
+		int nr_seg = (write ? req->out_num : req->in_num) - 1;
+		unsigned long sector = le64_to_cpu(req->hdr.sector);
+		ssize_t len, rem_len;
+
+		if (!req->q->blk->backend) {
+			vq_err(vq, "blk %p no backend!\n", req->q->blk);
+			ret = -EINVAL;
+			goto out_err;
+		}
+
+		len = iov_length(&vq->iov[1], nr_seg);
+		pr_debug("%s: [pid:%d %s] %s sector %lld, len %ld\n",
+			 __func__, current->pid, current->comm,
+			 write ? "WRITE" : "READ", req->hdr.sector, len);
+
+		req->len = len;
+		rem_len = len;
+		iov_iter_init(&req->i, (write ? WRITE : READ),
+			      write ? &req->out_iov[0] : &req->in_iov[0],
+			      nr_seg, len);
+
+		req->iocb.ki_pos = sector << 9;
+		req->iocb.ki_filp = blk->backend;
+		req->iocb.ki_complete = vhost_blk_iocb_complete;
+		req->iocb.ki_flags = IOCB_DIRECT;
+
+		if (write)
+			ret = call_write_iter(blk->backend, &req->iocb,
+					      &req->i);
+		else
+			ret = call_read_iter(blk->backend, &req->iocb,
+					     &req->i);
+
+		if (ret != -EIOCBQUEUED)
+			vhost_blk_iocb_complete(&req->iocb, ret, 0);
+
+		ret = 0;
+		goto out;
+	}
+
+	if (type == VIRTIO_BLK_T_GET_ID) {
+		char s[] = "vhost_blk";
+		size_t len = min_t(size_t, req->in_iov[0].iov_len,
+				   strlen(s));
+
+		ret = copy_to_user(req->in_iov[0].iov_base, s, len);
+		status = ret ? VIRTIO_BLK_S_IOERR : VIRTIO_BLK_S_OK;
+		if (put_user(status, (unsigned char __user *)req->status)) {
+			ret = -EFAULT;
+			goto out_err;
+		}
+		vhost_add_used_and_signal(&blk->dev, vq, req->index, 1);
+		ret = 0;
+		goto out;
+	} else {
+		pr_warn("Unsupported request type %d\n", type);
+		vhost_discard_vq_desc(vq, 1);
+		ret = -EINVAL;
+		return ret;
+	}
+out_err:
+	vhost_discard_vq_desc(vq, 1);
+out:
+	return ret;
+}
+
+static void vhost_blk_handle_guest_kick(struct vhost_work *work)
+{
+	struct vhost_virtqueue *vq;
+	struct vhost_blk_queue *q;
+	struct vhost_blk *blk;
+	struct vhost_blk_req *req;
+	int in, out;
+	int head;
+
+	vq = container_of(work, struct vhost_virtqueue, poll.work);
+	q = container_of(vq, struct vhost_blk_queue, vq);
+	blk = container_of(vq->dev, struct vhost_blk, dev);
+
+	vhost_disable_notify(&blk->dev, vq);
+	for (;;) {
+		in = out = -1;
+
+		head = vhost_get_vq_desc(vq, vq->iov,
+					 ARRAY_SIZE(vq->iov),
+					 &out, &in, NULL, NULL);
+
+		if (head < 0)
+			break;
+
+		if (head == vq->num) {
+			if (vhost_enable_notify(&blk->dev, vq)) {
+				vhost_disable_notify(&blk->dev, vq);
+				continue;
+			}
+			break;
+		}
+
+		req = &q->req[head];
+		req->index = head;
+		req->out_num = out;
+		req->in_num = in;
+		req->out_iov = &vq->iov[1];
+		req->in_iov = &vq->iov[out];
+		req->status = vq->iov[out + in - 1].iov_base;
+
+		if (copy_from_user(&req->hdr, vq->iov[0].iov_base,
+				   sizeof(req->hdr))) {
+			vq_err(vq, "Failed to get block header!\n");
+			vhost_discard_vq_desc(vq, 1);
+			continue;
+		}
+		if (vhost_blk_req_handle(req) < 0)
+			break;
+	}
+}
+
+static int vhost_blk_open(struct inode *inode, struct file *file)
+{
+	struct vhost_blk *blk;
+	struct vhost_blk_queue *q;
+	int i, j;
+
+	blk = kvzalloc(sizeof(*blk), GFP_KERNEL);
+	if (!blk)
+		return -ENOMEM;
+
+	for (i = 0; i < VHOST_BLK_VQ_MAX; i++) {
+		q = &blk->queue[i];
+		q->index = i;
+		q->blk = blk;
+		q->vq.handle_kick = vhost_blk_handle_guest_kick;
+		vhost_work_init(&q->w, vhost_blk_io_done_work);
+		blk->vqs[i] = &q->vq;
+		for (j = 0; j < VHOST_BLK_VQ_MAX_REQS; j++) {
+			q->req[j].index = j;
+			q->req[j].q = q;
+		}
+	}
+	vhost_dev_init(&blk->dev, (struct vhost_virtqueue **)&blk->vqs,
+		       VHOST_BLK_VQ_MAX);
+	file->private_data = blk;
+
+	return 0;
+}
+
+static int vhost_blk_release(struct inode *inode, struct file *f)
+{
+	struct vhost_blk *blk = f->private_data;
+
+	vhost_blk_stop(blk);
+	mutex_lock(&blk->dev.mutex);
+	vhost_blk_flush(blk);
+	vhost_dev_stop(&blk->dev);
+	vhost_dev_cleanup(&blk->dev);
+	vhost_blk_flush(blk);
+
+	if (blk->backend) {
+		fput(blk->backend);
+		blk->backend = NULL;
+	}
+
+	mutex_unlock(&blk->dev.mutex);
+	kvfree(blk);
+
+	return 0;
+}
+
+static int vhost_blk_set_features(struct vhost_blk *blk, u64 features)
+{
+	int i;
+	int ret = -EFAULT;
+
+	mutex_lock(&blk->dev.mutex);
+	if ((features & (1 << VHOST_F_LOG_ALL)) &&
+	    !vhost_log_access_ok(&blk->dev))
+		goto out_unlock;
+
+	if ((features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) {
+		if (vhost_init_device_iotlb(&blk->dev, true))
+			goto out_unlock;
+	}
+
+	for (i = 0; i < VHOST_BLK_VQ_MAX; ++i) {
+		struct vhost_virtqueue *vq = blk->vqs[i];
+
+		mutex_lock(&vq->mutex);
+		vq->acked_features = features & VHOST_BLK_FEATURES;
+		mutex_unlock(&vq->mutex);
+	}
+	ret = 0;
+out_unlock:
+	mutex_unlock(&blk->dev.mutex);
+
+	return ret;
+}
+
+static long vhost_blk_reset_owner(struct vhost_blk *blk)
+{
+	long err;
+	struct vhost_umem *umem;
+
+	mutex_lock(&blk->dev.mutex);
+	err = vhost_dev_check_owner(&blk->dev);
+	if (err)
+		goto done;
+	umem = vhost_dev_reset_owner_prepare();
+	if (!umem) {
+		err = -ENOMEM;
+		goto done;
+	}
+	vhost_blk_stop(blk);
+	vhost_blk_flush(blk);
+	vhost_dev_reset_owner(&blk->dev, umem);
+done:
+	mutex_unlock(&blk->dev.mutex);
+	return err;
+}
+
+static long vhost_blk_set_backend(struct vhost_blk *blk, int fd)
+{
+	struct file *backend;
+	int ret, i;
+	struct vhost_virtqueue *vq;
+
+	mutex_lock(&blk->dev.mutex);
+	ret = vhost_dev_check_owner(&blk->dev);
+	if (ret)
+		goto out_dev;
+
+	backend = fget(fd);
+	if (IS_ERR(backend)) {
+		ret = PTR_ERR(backend);
+		goto out_dev;
+	}
+
+	if (backend == blk->backend) {
+		ret = 0;
+		goto out_file;
+	}
+
+	if (blk->backend)
+		fput(blk->backend);
+	blk->backend = backend;
+	for (i = 0; i < blk->num_queues; i++) {
+		vq = &blk->queue[i].vq;
+		if (!vhost_vq_access_ok(vq)) {
+			ret = -EFAULT;
+			goto out_file;
+		}
+		mutex_lock(&vq->mutex);
+		rcu_assign_pointer(vq->private_data, backend);
+		ret = vhost_vq_init_access(vq);
+		mutex_unlock(&vq->mutex);
+		if (ret) {
+			pr_err("vhost_vq_init_access failed: %d\n", ret);
+			goto out_file;
+		}
+
+	}
+	ret = 0;
+	goto out_dev;
+out_file:
+	fput(backend);
+	blk->backend = NULL;
+out_dev:
+	mutex_unlock(&blk->dev.mutex);
+	vhost_blk_flush(blk);
+	return ret;
+}
+
+static long vhost_blk_pass_ioctl(struct vhost_blk *blk, unsigned int ioctl,
+				 void __user *argp)
+{
+	long ret;
+
+	mutex_lock(&blk->dev.mutex);
+	ret = vhost_dev_ioctl(&blk->dev, ioctl, argp);
+	if (ret == -ENOIOCTLCMD)
+		ret = vhost_vring_ioctl(&blk->dev, ioctl, argp);
+	else
+		vhost_blk_flush(blk);
+	mutex_unlock(&blk->dev.mutex);
+	return ret;
+}
+
+static long vhost_blk_ioctl(struct file *f, unsigned int ioctl,
+			    unsigned long arg)
+{
+	struct vhost_blk *blk = f->private_data;
+	void __user *argp = (void __user *)arg;
+	int fd;
+	u64 __user *featurep = argp;
+	u64 features;
+	long ret;
+	struct vhost_vring_state s;
+
+	switch (ioctl) {
+	case VHOST_SET_MEM_TABLE:
+		vhost_blk_stop(blk);
+		ret = vhost_blk_pass_ioctl(blk, ioctl, argp);
+		break;
+	case VHOST_SET_VRING_NUM:
+		if (copy_from_user(&s, argp, sizeof(s)))
+			return -EFAULT;
+		ret = vhost_blk_pass_ioctl(blk, ioctl, argp);
+		if (!ret)
+			blk->num_queues = s.index + 1;
+		break;
+	case VHOST_BLK_SET_BACKEND:
+		if (copy_from_user(&fd, argp, sizeof(fd)))
+			return -EFAULT;
+		ret = vhost_blk_set_backend(blk, fd);
+		break;
+	case VHOST_GET_FEATURES:
+		features = VHOST_BLK_FEATURES;
+		if (copy_to_user(featurep, &features, sizeof(features)))
+			return -EFAULT;
+		ret = 0;
+		break;
+	case VHOST_SET_FEATURES:
+		if (copy_from_user(&features, featurep, sizeof(features)))
+			return -EFAULT;
+		if (features & ~VHOST_BLK_FEATURES)
+			return -EOPNOTSUPP;
+		ret = vhost_blk_set_features(blk, features);
+		break;
+	case VHOST_RESET_OWNER:
+		ret = vhost_blk_reset_owner(blk);
+		break;
+	default:
+		ret = vhost_blk_pass_ioctl(blk, ioctl, argp);
+		break;
+	}
+	return ret;
+}
+
+static const struct file_operations vhost_blk_fops = {
+	.owner          = THIS_MODULE,
+	.open           = vhost_blk_open,
+	.release        = vhost_blk_release,
+	.llseek		= noop_llseek,
+	.unlocked_ioctl = vhost_blk_ioctl,
+};
+
+static struct miscdevice vhost_blk_misc = {
+	MISC_DYNAMIC_MINOR,
+	"vhost-blk",
+	&vhost_blk_fops,
+};
+
+static int vhost_blk_init(void)
+{
+	return misc_register(&vhost_blk_misc);
+}
+module_init(vhost_blk_init);
+
+static void vhost_blk_exit(void)
+{
+	misc_deregister(&vhost_blk_misc);
+}
+
+module_exit(vhost_blk_exit);
+
+MODULE_VERSION("1.0");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Vitaly Mayatskikh");
+MODULE_DESCRIPTION("Host kernel accelerator for virtio blk");
+MODULE_ALIAS("devname:vhost-blk");
-- 
2.17.1