[PATCH 2/3] nvme: Add hysteresis and idle detection to link rate switching

[Liao Xuan]

The rate switching decision currently toggles the link rate in every
monitoring window based on the I/O of that window alone, which can
cause excessive rate switching when the workload fluctuates around the
threshold.

Add counters that require the threshold to be exceeded for a
consecutive number of windows before the rate is upgraded, and to stay
below the threshold for a consecutive number of windows before it is
downgraded. The upgrade happens immediately (threshold 0 by default)
so that latency is not sacrificed, while the downgrade requires 10
consecutive windows.

Monitoring is stopped after 10 seconds of inactivity and is re-armed by
the next I/O.

Signed-off-by: Liao Xuan <liaoxuan@xxxxxxxxxxxxxx>
---
 drivers/nvme/host/nvme.h         |  5 +++
 drivers/nvme/host/speed_switch.c | 52 ++++++++++++++++++++++++++++----
 2 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index ff06f85d1..5d5034601 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -346,6 +346,11 @@ struct nvme_speed_switch {
 	u8 min_speed;
 	u32 monitor_interval;	/* ms */
 	u32 threshold;		/* KB */
+	u32 up_cnt;
+	u32 down_cnt;
+	u32 up_threshold;
+	u32 down_threshold;
+	u32 idle_cnt;
 	struct timer_list timer;
 	struct work_struct work;
 	struct nvme_speed_switch_stats __percpu *stats;
diff --git a/drivers/nvme/host/speed_switch.c b/drivers/nvme/host/speed_switch.c
index 88c046fcd..22f180c81 100644
--- a/drivers/nvme/host/speed_switch.c
+++ b/drivers/nvme/host/speed_switch.c
@@ -228,10 +228,15 @@ static void nvme_clear_io_stats(struct nvme_speed_switch *sw)
 
 /*
  * Aggregate the per-CPU I/O counters of the last monitoring window and
- * decide the target rate: the maximum rate if the window exceeded the
- * threshold, the minimum rate otherwise.
+ * decide the target rate.  Hysteresis counters are used to avoid
+ * excessive rate switching when the workload fluctuates around the
+ * threshold: the rate is only upgraded after the threshold was exceeded
+ * for a number of consecutive windows, and only downgraded after it
+ * stayed below the threshold for a number of consecutive windows.  When
+ * no I/O was observed for 10 seconds the monitoring is stopped.
  */
-static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw)
+static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw,
+					  bool *io_activity)
 {
 	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
 	u64 read_bytes = 0, write_bytes = 0;
@@ -247,31 +252,62 @@ static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw)
 		stat->write_bytes = 0;
 	}
 
+	if (read_bytes || write_bytes) {
+		*io_activity = true;
+		sw->idle_cnt = 0;
+	} else {
+		sw->idle_cnt++;
+	}
+
+	/* Stop monitoring after 10 seconds of inactivity (100 windows of 100ms). */
+	if (sw->idle_cnt >= 100)
+		*io_activity = false;
+
 	read_kb = read_bytes / 1024;
 	write_kb = write_bytes / 1024;
 	dev_dbg(ctrl->device,
 		"I/O in window: read=%lu KB, write=%lu KB, threshold=%u KB\n",
 		read_kb, write_kb, READ_ONCE(sw->threshold));
 
-	if (read_kb >= sw->threshold || write_kb >= sw->threshold)
+	if (read_kb >= sw->threshold || write_kb >= sw->threshold) {
+		sw->up_cnt++;
+		sw->down_cnt = 0;
+	} else {
+		sw->down_cnt++;
+		sw->up_cnt = 0;
+	}
+
+	if (sw->up_cnt > READ_ONCE(sw->up_threshold)) {
+		sw->up_cnt = 0;
 		return sw->max_speed;
+	}
+
+	if (sw->down_cnt > READ_ONCE(sw->down_threshold) || !*io_activity) {
+		sw->down_cnt = 0;
+		return sw->min_speed;
+	}
 
-	return sw->min_speed;
+	return READ_ONCE(sw->cur_speed);
 }
 
 static void nvme_speed_switch_timer_fn(struct timer_list *t)
 {
 	struct nvme_speed_switch *sw = container_of(t, struct nvme_speed_switch, timer);
+	bool io_activity = true;
 
 	if (!READ_ONCE(sw->enabled)) {
 		atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
 		return;
 	}
 
-	sw->target_speed = nvme_check_io_and_decide_speed(sw);
+	sw->target_speed = nvme_check_io_and_decide_speed(sw, &io_activity);
 	if (sw->target_speed != READ_ONCE(sw->cur_speed))
 		schedule_work(&sw->work);
 
+	if (!io_activity) {
+		atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
+		return;
+	}
 	mod_timer(t, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
 }
 
@@ -279,6 +315,10 @@ static void nvme_speed_switch_params_init(struct nvme_speed_switch *sw)
 {
 	sw->monitor_interval = 100;
 	sw->min_speed = PCI_EXP_LNKSTA_CLS_2_5GB;
+	/* Upgrade immediately once the threshold is exceeded. */
+	sw->up_threshold = 0;
+	/* Downgrade after 10 consecutive below-threshold windows. */
+	sw->down_threshold = 10;
 }
 
 void nvme_speed_switch_init(struct nvme_ctrl *ctrl)
-- 
2.43.0