[PATCH] nvme-pci: adaptively poll busy queues from threaded interrupts

[Fengnan Chang]

On systems with multiple fast NVMe devices, hard IRQ completion
processing can become CPU-bound and cap 4K random-read throughput
before the devices are saturated.

Add adaptive polling for busy interrupt-driven queues. Track the SQ head
reported by completions while the CQE is still owned by the host. After
the hard IRQ reaps pending CQEs, wake the IRQ thread and mask the queue
interrupt when the number of SQ entries not yet consumed reaches a high
watermark. Poll in the IRQ thread until the queue drops below a low
watermark or a bounded loop count is reached, then reenable the
interrupt.

Only enable adaptive polling when the controller has independent
vectors, as masking a shared single vector could suppress unrelated
queues. Expose the enable switch, watermarks, loop bound, and sleep
interval as module parameters.

Signed-off-by: Fengnan Chang <changfengnan@xxxxxxxxxxxxx>
---
 drivers/nvme/host/pci.c | 212 +++++++++++++++++++++++++++++++++++++---
 1 file changed, 198 insertions(+), 14 deletions(-)

diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 69932d640b537..be3a3ea0196f5 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -9,6 +9,7 @@
 #include <linux/blkdev.h>
 #include <linux/blk-mq-dma.h>
 #include <linux/blk-integrity.h>
+#include <linux/delay.h>
 #include <linux/dmi.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -79,8 +80,136 @@ struct quirk_entry {
 	u32 disabled_quirks;
 };
 
-static int use_threaded_interrupts;
-module_param(use_threaded_interrupts, int, 0444);
+static int use_adaptive_polling;
+module_param(use_adaptive_polling, int, 0444);
+MODULE_PARM_DESC(use_adaptive_polling,
+		 "use adaptive completion polling on busy interrupt queues");
+
+static unsigned int adaptive_poll_start_pending = 10;
+static unsigned int adaptive_poll_stop_pending = 3;
+static unsigned int adaptive_poll_max_loops = 100;
+static unsigned int adaptive_poll_sleep_min_us = 20;
+static unsigned int adaptive_poll_sleep_max_us = 30;
+
+static int nvme_adaptive_poll_start_pending_set(const char *val, const struct kernel_param *kp)
+{
+	unsigned int pending;
+	int ret;
+
+	ret = kstrtouint(val, 10, &pending);
+	if (ret)
+		return ret;
+	if (!pending || pending < READ_ONCE(adaptive_poll_stop_pending))
+		return -EINVAL;
+
+	WRITE_ONCE(*(unsigned int *)kp->arg, pending);
+	return 0;
+}
+
+static const struct kernel_param_ops nvme_adaptive_poll_start_pending_ops = {
+	.set = nvme_adaptive_poll_start_pending_set,
+	.get = param_get_uint,
+};
+
+module_param_cb(adaptive_poll_start_pending, &nvme_adaptive_poll_start_pending_ops,
+		&adaptive_poll_start_pending, 0644);
+MODULE_PARM_DESC(adaptive_poll_start_pending, "SQ pending high watermark for adaptive polling");
+
+static int nvme_adaptive_poll_stop_pending_set(const char *val, const struct kernel_param *kp)
+{
+	unsigned int pending;
+	int ret;
+
+	ret = kstrtouint(val, 10, &pending);
+	if (ret)
+		return ret;
+	if (!pending || pending > READ_ONCE(adaptive_poll_start_pending))
+		return -EINVAL;
+
+	WRITE_ONCE(*(unsigned int *)kp->arg, pending);
+	return 0;
+}
+
+static const struct kernel_param_ops nvme_adaptive_poll_stop_pending_ops = {
+	.set = nvme_adaptive_poll_stop_pending_set,
+	.get = param_get_uint,
+};
+
+module_param_cb(adaptive_poll_stop_pending, &nvme_adaptive_poll_stop_pending_ops,
+		&adaptive_poll_stop_pending, 0644);
+MODULE_PARM_DESC(adaptive_poll_stop_pending, "SQ pending low watermark for adaptive polling");
+
+static int nvme_adaptive_poll_max_loops_set(const char *val, const struct kernel_param *kp)
+{
+	unsigned int loops;
+	int ret;
+
+	ret = kstrtouint(val, 10, &loops);
+	if (ret)
+		return ret;
+	if (!loops)
+		return -EINVAL;
+
+	WRITE_ONCE(*(unsigned int *)kp->arg, loops);
+	return 0;
+}
+
+static const struct kernel_param_ops nvme_adaptive_poll_max_loops_ops = {
+	.set = nvme_adaptive_poll_max_loops_set,
+	.get = param_get_uint,
+};
+
+module_param_cb(adaptive_poll_max_loops, &nvme_adaptive_poll_max_loops_ops,
+		&adaptive_poll_max_loops, 0644);
+MODULE_PARM_DESC(adaptive_poll_max_loops, "Maximum adaptive polling loops");
+
+static int nvme_adaptive_poll_sleep_min_us_set(const char *val, const struct kernel_param *kp)
+{
+	unsigned int usecs;
+	int ret;
+
+	ret = kstrtouint(val, 10, &usecs);
+	if (ret)
+		return ret;
+	if (!usecs || usecs > READ_ONCE(adaptive_poll_sleep_max_us))
+		return -EINVAL;
+
+	WRITE_ONCE(*(unsigned int *)kp->arg, usecs);
+	return 0;
+}
+
+static const struct kernel_param_ops nvme_adaptive_poll_sleep_min_us_ops = {
+	.set = nvme_adaptive_poll_sleep_min_us_set,
+	.get = param_get_uint,
+};
+
+module_param_cb(adaptive_poll_sleep_min_us, &nvme_adaptive_poll_sleep_min_us_ops,
+		&adaptive_poll_sleep_min_us, 0644);
+MODULE_PARM_DESC(adaptive_poll_sleep_min_us, "Minimum adaptive polling delay in microseconds");
+
+static int nvme_adaptive_poll_sleep_max_us_set(const char *val, const struct kernel_param *kp)
+{
+	unsigned int usecs;
+	int ret;
+
+	ret = kstrtouint(val, 10, &usecs);
+	if (ret)
+		return ret;
+	if (!usecs || usecs < READ_ONCE(adaptive_poll_sleep_min_us))
+		return -EINVAL;
+
+	WRITE_ONCE(*(unsigned int *)kp->arg, usecs);
+	return 0;
+}
+
+static const struct kernel_param_ops nvme_adaptive_poll_sleep_max_us_ops = {
+	.set = nvme_adaptive_poll_sleep_max_us_set,
+	.get = param_get_uint,
+};
+
+module_param_cb(adaptive_poll_sleep_max_us, &nvme_adaptive_poll_sleep_max_us_ops,
+		&adaptive_poll_sleep_max_us, 0644);
+MODULE_PARM_DESC(adaptive_poll_sleep_max_us, "Maximum adaptive polling delay in microseconds");
 
 static bool use_cmb_sqes = true;
 module_param(use_cmb_sqes, bool, 0444);
@@ -377,6 +506,7 @@ struct nvme_queue {
 	u16 cq_vector;
 	u16 sq_tail;
 	u16 last_sq_tail;
+	u16 sq_head;
 	u16 cq_head;
 	u16 qid;
 	u8 cq_phase;
@@ -1607,7 +1737,7 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
 }
 
 static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
-			        struct io_comp_batch *iob)
+				struct io_comp_batch *iob, u16 *sq_head)
 {
 	bool found = false;
 
@@ -1618,6 +1748,8 @@ static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
 		 * the cqe requires a full read memory barrier
 		 */
 		dma_rmb();
+		if (sq_head)
+			*sq_head = le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].sq_head);
 		nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head);
 		nvme_update_cq_head(nvmeq);
 	}
@@ -1631,8 +1763,11 @@ static irqreturn_t nvme_irq(int irq, void *data)
 {
 	struct nvme_queue *nvmeq = data;
 	DEFINE_IO_COMP_BATCH(iob);
+	u16 sq_head;
+
+	if (nvme_poll_cq(nvmeq, &iob, &sq_head)) {
+		WRITE_ONCE(nvmeq->sq_head, sq_head);
 
-	if (nvme_poll_cq(nvmeq, &iob)) {
 		if (!rq_list_empty(&iob.req_list))
 			nvme_pci_complete_batch(&iob);
 		return IRQ_HANDLED;
@@ -1640,13 +1775,56 @@ static irqreturn_t nvme_irq(int irq, void *data)
 	return IRQ_NONE;
 }
 
-static irqreturn_t nvme_irq_check(int irq, void *data)
+static unsigned int nvme_sq_pending(struct nvme_queue *nvmeq)
+{
+	int pending = READ_ONCE(nvmeq->sq_tail) - READ_ONCE(nvmeq->sq_head);
+
+	if (pending < 0)
+		pending += nvmeq->q_depth;
+	return pending;
+}
+
+static bool nvme_adaptive_poll_should_start(struct nvme_queue *nvmeq)
+{
+	return nvme_sq_pending(nvmeq) >= READ_ONCE(adaptive_poll_start_pending);
+}
+
+static bool nvme_adaptive_poll_should_stop(struct nvme_queue *nvmeq)
+{
+	return nvme_sq_pending(nvmeq) < READ_ONCE(adaptive_poll_stop_pending);
+}
+
+static irqreturn_t nvme_adaptive_poll(int irq, void *data)
 {
 	struct nvme_queue *nvmeq = data;
+	unsigned int loops = 0;
+	unsigned int max_loops = READ_ONCE(adaptive_poll_max_loops);
+	unsigned int sleep_min = READ_ONCE(adaptive_poll_sleep_min_us);
+	unsigned int sleep_max = READ_ONCE(adaptive_poll_sleep_max_us);
 
-	if (nvme_cqe_pending(nvmeq))
-		return IRQ_WAKE_THREAD;
-	return IRQ_NONE;
+	do {
+		if (nvme_cqe_pending(nvmeq))
+			nvme_irq(irq, data);
+		if (nvme_adaptive_poll_should_stop(nvmeq))
+			break;
+		usleep_range(sleep_min, sleep_max);
+	} while (++loops < max_loops);
+
+	enable_irq(irq);
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t nvme_irq_adaptive_poll(int irq, void *data)
+{
+	struct nvme_queue *nvmeq = data;
+	irqreturn_t ret;
+
+	ret = nvme_irq(irq, data);
+	if (!nvme_adaptive_poll_should_start(nvmeq))
+		return ret;
+
+	disable_irq_nosync(irq);
+	return IRQ_WAKE_THREAD;
 }
 
 /*
@@ -1663,7 +1841,7 @@ static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
 	irq = pci_irq_vector(pdev, nvmeq->cq_vector);
 	disable_irq(irq);
 	spin_lock(&nvmeq->cq_poll_lock);
-	nvme_poll_cq(nvmeq, NULL);
+	nvme_poll_cq(nvmeq, NULL, NULL);
 	spin_unlock(&nvmeq->cq_poll_lock);
 	enable_irq(irq);
 }
@@ -1678,7 +1856,7 @@ static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 		return 0;
 
 	spin_lock(&nvmeq->cq_poll_lock);
-	found = nvme_poll_cq(nvmeq, iob);
+	found = nvme_poll_cq(nvmeq, iob, NULL);
 	spin_unlock(&nvmeq->cq_poll_lock);
 
 	return found;
@@ -2075,7 +2253,7 @@ static void nvme_reap_pending_cqes(struct nvme_dev *dev)
 
 	for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
 		spin_lock(&dev->queues[i].cq_poll_lock);
-		nvme_poll_cq(&dev->queues[i], NULL);
+		nvme_poll_cq(&dev->queues[i], NULL, NULL);
 		spin_unlock(&dev->queues[i].cq_poll_lock);
 	}
 }
@@ -2171,9 +2349,14 @@ static int queue_request_irq(struct nvme_queue *nvmeq)
 	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
 	int nr = nvmeq->dev->ctrl.instance;
 
-	if (use_threaded_interrupts) {
-		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
-				nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
+	/*
+	 * Adaptive polling masks the entire vector, so don't use it when
+	 * multiple queues share the single available vector.
+	 */
+	if (use_adaptive_polling && nvmeq->dev->num_vecs > 1) {
+		return pci_request_irq(pdev, nvmeq->cq_vector,
+				nvme_irq_adaptive_poll, nvme_adaptive_poll, nvmeq,
+				"nvme%dq%d", nr, nvmeq->qid);
 	} else {
 		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
 				NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
@@ -2186,6 +2369,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
 
 	nvmeq->sq_tail = 0;
 	nvmeq->last_sq_tail = 0;
+	nvmeq->sq_head = 0;
 	nvmeq->cq_head = 0;
 	nvmeq->cq_phase = 1;
 	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];