[PATCH 13/17] sched: Merge latency_offset into slice

[Peter Zijlstra]

Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
---
 include/linux/sched.h |    2 --
 kernel/sched/core.c   |   17 +++++++----------
 kernel/sched/fair.c   |   29 ++++++++++++-----------------
 kernel/sched/sched.h  |    2 +-
 4 files changed, 20 insertions(+), 30 deletions(-)

--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -573,8 +573,6 @@ struct sched_entity {
 	/* cached value of my_q->h_nr_running */
 	unsigned long			runnable_weight;
 #endif
-	/* preemption offset in ns */
-	long				latency_offset;
 
 #ifdef CONFIG_SMP
 	/*
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1285,9 +1285,10 @@ static void set_load_weight(struct task_
 	}
 }
 
-static void set_latency_offset(struct task_struct *p)
+static inline void set_latency_prio(struct task_struct *p, int prio)
 {
-	p->se.latency_offset = calc_latency_offset(p->latency_prio - MAX_RT_PRIO);
+	p->latency_prio = prio;
+	set_latency_fair(&p->se, prio - MAX_RT_PRIO);
 }
 
 #ifdef CONFIG_UCLAMP_TASK
@@ -4442,7 +4443,7 @@ static void __sched_fork(unsigned long c
 	p->se.vlag			= 0;
 	INIT_LIST_HEAD(&p->se.group_node);
 
-	set_latency_offset(p);
+	set_latency_prio(p, p->latency_prio);
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
 	p->se.cfs_rq			= NULL;
@@ -4694,9 +4695,7 @@ int sched_fork(unsigned long clone_flags
 
 		p->prio = p->normal_prio = p->static_prio;
 		set_load_weight(p, false);
-
-		p->latency_prio = NICE_TO_PRIO(0);
-		set_latency_offset(p);
+		set_latency_prio(p, NICE_TO_PRIO(0));
 
 		/*
 		 * We don't need the reset flag anymore after the fork. It has
@@ -7469,10 +7468,8 @@ static void __setscheduler_params(struct
 static void __setscheduler_latency(struct task_struct *p,
 				   const struct sched_attr *attr)
 {
-	if (attr->sched_flags & SCHED_FLAG_LATENCY_NICE) {
-		p->latency_prio = NICE_TO_PRIO(attr->sched_latency_nice);
-		set_latency_offset(p);
-	}
+	if (attr->sched_flags & SCHED_FLAG_LATENCY_NICE)
+		set_latency_prio(p, NICE_TO_PRIO(attr->sched_latency_nice));
 }
 
 /*
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -919,12 +919,19 @@ int sched_update_scaling(void)
 }
 #endif
 
-long calc_latency_offset(int prio)
+void set_latency_fair(struct sched_entity *se, int prio)
 {
 	u32 weight = sched_prio_to_weight[prio];
 	u64 base = sysctl_sched_base_slice;
 
-	return div_u64(base << SCHED_FIXEDPOINT_SHIFT, weight);
+	/*
+	 * For EEVDF the virtual time slope is determined by w_i (iow.
+	 * nice) while the request time r_i is determined by
+	 * latency-nice.
+	 *
+	 * Smaller request gets better latency.
+	 */
+	se->slice = div_u64(base << SCHED_FIXEDPOINT_SHIFT, weight);
 }
 
 /*
@@ -937,13 +944,6 @@ static void update_deadline(struct cfs_r
 		return;
 
 	/*
-	 * For EEVDF the virtual time slope is determined by w_i (iow.
-	 * nice) while the request time r_i is determined by
-	 * latency-nice.
-	 */
-	se->slice = se->latency_offset;
-
-	/*
 	 * EEVDF: vd_i = ve_i + r_i / w_i
 	 */
 	se->deadline = se->vruntime + calc_delta_fair(se->slice, se);
@@ -12231,7 +12231,7 @@ void init_tg_cfs_entry(struct task_group
 
 	se->my_q = cfs_rq;
 
-	se->latency_offset = calc_latency_offset(tg->latency_prio - MAX_RT_PRIO);
+	set_latency_fair(se, tg->latency_prio - MAX_RT_PRIO);
 
 	/* guarantee group entities always have weight */
 	update_load_set(&se->load, NICE_0_LOAD);
@@ -12365,7 +12365,6 @@ int sched_group_set_idle(struct task_gro
 
 int sched_group_set_latency(struct task_group *tg, int prio)
 {
-	long latency_offset;
 	int i;
 
 	if (tg == &root_task_group)
@@ -12379,13 +12378,9 @@ int sched_group_set_latency(struct task_
 	}
 
 	tg->latency_prio = prio;
-	latency_offset = calc_latency_offset(prio - MAX_RT_PRIO);
 
-	for_each_possible_cpu(i) {
-		struct sched_entity *se = tg->se[i];
-
-		WRITE_ONCE(se->latency_offset, latency_offset);
-	}
+	for_each_possible_cpu(i)
+		set_latency_fair(tg->se[i], prio - MAX_RT_PRIO);
 
 	mutex_unlock(&shares_mutex);
 	return 0;
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2477,7 +2477,7 @@ extern unsigned int sysctl_numa_balancin
 extern unsigned int sysctl_numa_balancing_hot_threshold;
 #endif
 
-extern long calc_latency_offset(int prio);
+extern void set_latency_fair(struct sched_entity *se, int prio);
 
 #ifdef CONFIG_SCHED_HRTICK