[PATCH 2/3] sched/fair: Add load_weight->runnable_load_{sum|avg}

[Tejun Heo]

Currently, runnable_load_avg, which represents the portion of load avg
only from tasks which are currently active, is tracked by cfs_rq but
not by sched_entity.  We want to propagate runnable_load_avg of a
nested cfs_rq without affecting load_avg propagation.  To implement an
equivalent propagation channel, sched_entity needs to track
runnable_load_avg too.

This patch moves cfs_rq->runnable_load_{sum|avg} into struct
load_weight which is already used to track load_avg and shared by both
cfs_rq and sched_entity.

This patch only changes where runnable_load_{sum|avg} are located and
doesn't cause any actual behavior changes.  The fields are still only
used for cfs_rqs.

Signed-off-by: Tejun Heo <tj@xxxxxxxxxx>
Cc: Vincent Guittot <vincent.guittot@xxxxxxxxxx>
Cc: Ingo Molnar <mingo@xxxxxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: Mike Galbraith <efault@xxxxxx>
Cc: Paul Turner <pjt@xxxxxxxxxx>
Cc: Chris Mason <clm@xxxxxx>
---
 include/linux/sched.h |    2 +
 kernel/sched/debug.c  |    2 -
 kernel/sched/fair.c   |   63 +++++++++++++++++++++++++-------------------------
 kernel/sched/sched.h  |    2 -
 4 files changed, 35 insertions(+), 34 deletions(-)

--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -314,9 +314,11 @@ struct load_weight {
 struct sched_avg {
 	u64				last_update_time;
 	u64				load_sum;
+	u64				runnable_load_sum;
 	u32				util_sum;
 	u32				period_contrib;
 	unsigned long			load_avg;
+	unsigned long			runnable_load_avg;
 	unsigned long			util_avg;
 };
 
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -516,7 +516,7 @@ void print_cfs_rq(struct seq_file *m, in
 	SEQ_printf(m, "  .%-30s: %lu\n", "load_avg",
 			cfs_rq->avg.load_avg);
 	SEQ_printf(m, "  .%-30s: %lu\n", "runnable_load_avg",
-			cfs_rq->runnable_load_avg);
+			cfs_rq->avg.runnable_load_avg);
 	SEQ_printf(m, "  .%-30s: %lu\n", "util_avg",
 			cfs_rq->avg.util_avg);
 	SEQ_printf(m, "  .%-30s: %ld\n", "removed_load_avg",
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2656,10 +2656,10 @@ calc_cfs_shares(struct cfs_rq *cfs_rq, s
 	case shares_runnable:
 		/*
 		 * Instead of the correct cfs_rq->avg.load_avg we use
-		 * cfs_rq->runnable_load_avg, which does not include the
-		 * blocked load.
+		 * cfs_rq->avg.runnable_load_avg, which does not include
+		 * the blocked load.
 		 */
-		load = cfs_rq->runnable_load_avg;
+		load = cfs_rq->avg.runnable_load_avg;
 		break;
 
 	case shares_avg:
@@ -2877,7 +2877,7 @@ static u32 __compute_runnable_contrib(u6
  */
 static __always_inline int
 __update_load_avg(u64 now, int cpu, struct sched_avg *sa,
-		  unsigned long weight, int running, struct cfs_rq *cfs_rq)
+		  unsigned long weight, int running, bool update_runnable)
 {
 	u64 delta, scaled_delta, periods;
 	u32 contrib;
@@ -2923,10 +2923,8 @@ __update_load_avg(u64 now, int cpu, stru
 		scaled_delta_w = cap_scale(delta_w, scale_freq);
 		if (weight) {
 			sa->load_sum += weight * scaled_delta_w;
-			if (cfs_rq) {
-				cfs_rq->runnable_load_sum +=
-						weight * scaled_delta_w;
-			}
+			if (update_runnable)
+				sa->runnable_load_sum += weight * scaled_delta_w;
 		}
 		if (running)
 			sa->util_sum += scaled_delta_w * scale_cpu;
@@ -2938,9 +2936,9 @@ __update_load_avg(u64 now, int cpu, stru
 		delta %= 1024;
 
 		sa->load_sum = decay_load(sa->load_sum, periods + 1);
-		if (cfs_rq) {
-			cfs_rq->runnable_load_sum =
-				decay_load(cfs_rq->runnable_load_sum, periods + 1);
+		if (update_runnable) {
+			sa->runnable_load_sum =
+				decay_load(sa->runnable_load_sum, periods + 1);
 		}
 		sa->util_sum = decay_load((u64)(sa->util_sum), periods + 1);
 
@@ -2949,8 +2947,8 @@ __update_load_avg(u64 now, int cpu, stru
 		contrib = cap_scale(contrib, scale_freq);
 		if (weight) {
 			sa->load_sum += weight * contrib;
-			if (cfs_rq)
-				cfs_rq->runnable_load_sum += weight * contrib;
+			if (update_runnable)
+				sa->runnable_load_sum += weight * contrib;
 		}
 		if (running)
 			sa->util_sum += contrib * scale_cpu;
@@ -2960,8 +2958,8 @@ __update_load_avg(u64 now, int cpu, stru
 	scaled_delta = cap_scale(delta, scale_freq);
 	if (weight) {
 		sa->load_sum += weight * scaled_delta;
-		if (cfs_rq)
-			cfs_rq->runnable_load_sum += weight * scaled_delta;
+		if (update_runnable)
+			sa->runnable_load_sum += weight * scaled_delta;
 	}
 	if (running)
 		sa->util_sum += scaled_delta * scale_cpu;
@@ -2970,9 +2968,9 @@ __update_load_avg(u64 now, int cpu, stru
 
 	if (decayed) {
 		sa->load_avg = div_u64(sa->load_sum, LOAD_AVG_MAX);
-		if (cfs_rq) {
-			cfs_rq->runnable_load_avg =
-				div_u64(cfs_rq->runnable_load_sum, LOAD_AVG_MAX);
+		if (update_runnable) {
+			sa->runnable_load_avg =
+				div_u64(sa->runnable_load_sum, LOAD_AVG_MAX);
 		}
 		sa->util_avg = sa->util_sum / LOAD_AVG_MAX;
 	}
@@ -3075,7 +3073,7 @@ void set_task_rq_fair(struct sched_entit
 		n_last_update_time = next->avg.last_update_time;
 #endif
 		__update_load_avg(p_last_update_time, cpu_of(rq_of(prev)),
-				  &se->avg, 0, 0, NULL);
+				  &se->avg, 0, 0, false);
 		se->avg.last_update_time = n_last_update_time;
 	}
 }
@@ -3129,8 +3127,9 @@ update_tg_cfs_load(struct cfs_rq *cfs_rq
 	 */
 	if (se->on_rq) {
 		/* Update parent cfs_rq runnable_load_avg */
-		add_positive(&cfs_rq->runnable_load_avg, delta);
-		cfs_rq->runnable_load_sum = cfs_rq->runnable_load_avg * LOAD_AVG_MAX;
+		add_positive(&cfs_rq->avg.runnable_load_avg, delta);
+		cfs_rq->avg.runnable_load_sum =
+			cfs_rq->avg.runnable_load_avg * LOAD_AVG_MAX;
 	}
 }
 
@@ -3264,7 +3263,7 @@ update_cfs_rq_load_avg(u64 now, struct c
 	}
 
 	decayed = __update_load_avg(now, cpu_of(rq_of(cfs_rq)), sa,
-		scale_load_down(cfs_rq->load.weight), cfs_rq->curr != NULL, cfs_rq);
+		scale_load_down(cfs_rq->load.weight), cfs_rq->curr != NULL, true);
 
 #ifndef CONFIG_64BIT
 	smp_wmb();
@@ -3299,7 +3298,7 @@ static inline void update_load_avg(struc
 	if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) {
 		__update_load_avg(now, cpu, &se->avg,
 			  se->on_rq * scale_load_down(se->load.weight),
-			  cfs_rq->curr == se, NULL);
+			  cfs_rq->curr == se, false);
 	}
 
 	decayed  = update_cfs_rq_load_avg(now, cfs_rq, true);
@@ -3355,8 +3354,8 @@ enqueue_entity_load_avg(struct cfs_rq *c
 {
 	struct sched_avg *sa = &se->avg;
 
-	cfs_rq->runnable_load_avg += sa->load_avg;
-	cfs_rq->runnable_load_sum += sa->load_sum;
+	cfs_rq->avg.runnable_load_avg += sa->load_avg;
+	cfs_rq->avg.runnable_load_sum += sa->load_sum;
 
 	if (!sa->last_update_time) {
 		attach_entity_load_avg(cfs_rq, se);
@@ -3368,10 +3367,12 @@ enqueue_entity_load_avg(struct cfs_rq *c
 static inline void
 dequeue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	cfs_rq->runnable_load_avg =
-		max_t(long, cfs_rq->runnable_load_avg - se->avg.load_avg, 0);
-	cfs_rq->runnable_load_sum =
-		max_t(s64,  cfs_rq->runnable_load_sum - se->avg.load_sum, 0);
+	struct sched_avg *sa = &se->avg;
+
+	cfs_rq->avg.runnable_load_avg =
+		max_t(long, cfs_rq->avg.runnable_load_avg - sa->load_avg, 0);
+	cfs_rq->avg.runnable_load_sum =
+		max_t(s64,  cfs_rq->avg.runnable_load_sum - sa->load_sum, 0);
 }
 
 #ifndef CONFIG_64BIT
@@ -3405,7 +3406,7 @@ void sync_entity_load_avg(struct sched_e
 	u64 last_update_time;
 
 	last_update_time = cfs_rq_last_update_time(cfs_rq);
-	__update_load_avg(last_update_time, cpu_of(rq_of(cfs_rq)), &se->avg, 0, 0, NULL);
+	__update_load_avg(last_update_time, cpu_of(rq_of(cfs_rq)), &se->avg, 0, 0, false);
 }
 
 /*
@@ -3433,7 +3434,7 @@ void remove_entity_load_avg(struct sched
 
 static inline unsigned long cfs_rq_runnable_load_avg(struct cfs_rq *cfs_rq)
 {
-	return cfs_rq->runnable_load_avg;
+	return cfs_rq->avg.runnable_load_avg;
 }
 
 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -422,8 +422,6 @@ struct cfs_rq {
 	 * CFS load tracking
 	 */
 	struct sched_avg avg;
-	u64 runnable_load_sum;
-	unsigned long runnable_load_avg;
 #ifdef CONFIG_FAIR_GROUP_SCHED
 	unsigned long tg_load_avg_contrib;
 	unsigned long propagate_avg;