[PATCH 1/2] sched/psi: Preserve pressure accounting across RT trigger registration

[Hui Su]

Creating a privileged trigger samples per-cpu pressure before an existing
rtpoll worker can consume it. Preserve state changes for existing triggers,
and initialize the new trigger window after the registration sample so it
does not account pressure that predates registration.

Evaluate triggers whenever their update is due, even if the current sample
has no changed states. Advance rtpoll_total after each evaluation so a
consumed sample is not counted again.

Signed-off-by: Hui Su <sh_def@xxxxxxx>
---
 include/linux/psi_types.h |  1 +
 kernel/sched/psi.c        | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h
index dd10c22299ab8..5507a31477a43 100644
--- a/include/linux/psi_types.h
+++ b/include/linux/psi_types.h
@@ -197,6 +197,7 @@ struct psi_group {
 	struct list_head rtpoll_triggers;
 	u32 rtpoll_nr_triggers[NR_PSI_STATES - 1];
 	u32 rtpoll_states;
+	u32 rtpoll_pending_states;
 	u64 rtpoll_min_period;
 
 	/* Total stall times at the start of RT polling monitor activation */
diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c
index 7c53814233345..733ecacc2adb4 100644
--- a/kernel/sched/psi.c
+++ b/kernel/sched/psi.c
@@ -219,6 +219,7 @@ static void group_init(struct psi_group *group)
 	mutex_init(&group->rtpoll_trigger_lock);
 	INIT_LIST_HEAD(&group->rtpoll_triggers);
 	group->rtpoll_min_period = U32_MAX;
+	group->rtpoll_pending_states = 0;
 	group->rtpoll_next_update = ULLONG_MAX;
 	init_waitqueue_head(&group->rtpoll_wait);
 	timer_setup(&group->rtpoll_timer, poll_timer_fn, 0);
@@ -648,7 +649,7 @@ static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long dela
 static void psi_rtpoll_work(struct psi_group *group)
 {
 	bool force_reschedule = false;
-	u32 changed_states;
+	u32 changed_states, sampled_states;
 	u64 now;
 
 	mutex_lock(&group->rtpoll_trigger_lock);
@@ -691,8 +692,11 @@ static void psi_rtpoll_work(struct psi_group *group)
 		force_reschedule = true;
 	}
 
-
-	collect_percpu_times(group, PSI_POLL, &changed_states);
+	/* Trigger registration may have consumed an activity delta. */
+	changed_states = group->rtpoll_pending_states;
+	group->rtpoll_pending_states = 0;
+	collect_percpu_times(group, PSI_POLL, &sampled_states);
+	changed_states |= sampled_states;
 
 	if (changed_states & group->rtpoll_states) {
 		/* Initialize trigger windows when entering rtpolling mode */
@@ -714,11 +718,8 @@ static void psi_rtpoll_work(struct psi_group *group)
 	}
 
 	if (now >= group->rtpoll_next_update) {
-		if (changed_states & group->rtpoll_states) {
-			update_triggers(group, now, PSI_POLL);
-			memcpy(group->rtpoll_total, group->total[PSI_POLL],
-				   sizeof(group->rtpoll_total));
-		}
+		update_triggers(group, now, PSI_POLL);
+		memcpy(group->rtpoll_total, group->total[PSI_POLL], sizeof(group->rtpoll_total));
 		group->rtpoll_next_update = now + group->rtpoll_min_period;
 	}
 
@@ -1334,7 +1335,7 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
 {
 	struct psi_trigger *t;
 	enum psi_states state;
-	u32 threshold_us;
+	u32 threshold_us, changed_states, existing_states;
 	bool privileged;
 	u32 window_us;
 
@@ -1386,8 +1387,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
 	t->state = state;
 	t->threshold = threshold_us * NSEC_PER_USEC;
 	t->win.size = window_us * NSEC_PER_USEC;
-	window_reset(&t->win, sched_clock(),
-			group->total[PSI_POLL][t->state], 0);
+	if (!privileged)
+		window_reset(&t->win, sched_clock(), group->total[PSI_POLL][t->state], 0);
 
 	t->event = 0;
 	t->last_event_time = 0;
@@ -1400,13 +1401,19 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
 	if (privileged) {
 		mutex_lock(&group->rtpoll_trigger_lock);
 
+		existing_states = group->rtpoll_states;
 		list_add(&t->node, &group->rtpoll_triggers);
 		group->rtpoll_min_period = min(group->rtpoll_min_period,
 			div_u64(t->win.size, UPDATES_PER_WINDOW));
 		group->rtpoll_nr_triggers[t->state]++;
 		group->rtpoll_states |= (1 << t->state);
+		collect_percpu_times(group, PSI_POLL, &changed_states);
+		group->rtpoll_pending_states |= changed_states & existing_states;
+		window_reset(&t->win, sched_clock(), group->total[PSI_POLL][t->state], 0);
 
 		*need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task);
+		if (!*need_rtpoll_worker)
+			psi_schedule_rtpoll_work(group, 1, true);
 
 		mutex_unlock(&group->rtpoll_trigger_lock);
 	} else {
@@ -1458,8 +1465,10 @@ void psi_trigger_destroy(struct psi_trigger *t)
 
 			list_del(&t->node);
 			group->rtpoll_nr_triggers[t->state]--;
-			if (!group->rtpoll_nr_triggers[t->state])
+			if (!group->rtpoll_nr_triggers[t->state]) {
 				group->rtpoll_states &= ~(1 << t->state);
+				group->rtpoll_pending_states &= ~(1 << t->state);
+			}
 			/*
 			 * Reset min update period for the remaining triggers
 			 * iff the destroying trigger had the min window size.
@@ -1473,6 +1482,7 @@ void psi_trigger_destroy(struct psi_trigger *t)
 			/* Destroy rtpoll_task when the last trigger is destroyed */
 			if (group->rtpoll_states == 0) {
 				group->rtpoll_until = 0;
+				group->rtpoll_pending_states = 0;
 				task_to_destroy = rcu_dereference_protected(
 						group->rtpoll_task,
 						lockdep_is_held(&group->rtpoll_trigger_lock));
-- 
2.55.0