[PATCH v2 2/2] sched/cache: Drive cache task tick from execution context

[Hui Su]

Cache-aware scheduling accounts CPU runtime to the mm of the task
actually executing. update_se() passes the execution task to
account_mm_sched() for this purpose.

With proxy execution, however, sched_tick() invokes task_tick() for the
scheduling context in rq->donor. task_tick_cache() is currently called
from task_tick_fair(), so it is skipped when a fair task executes on
behalf of an RT or deadline donor.

In that case account_mm_sched() continues to advance runtime accounting
for rq->curr, while task_tick_cache() does not advance the corresponding
mm scan epoch. Once the epoch becomes stale, account_mm_sched() can
invalidate the mm's preferred LLC.

Move cache tick handling one level up into sched_tick(), alongside NUMA
tick handling, and run it when the execution context is a fair task.

Do the same in sched_tick_remote() so full-dynticks CPUs retain cache
tick handling after it is removed from task_tick_fair().

Fixes: df0d98475954 ("sched/cache: Introduce infrastructure for cache-aware load balancing")
Suggested-by: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx>
Signed-off-by: Hui Su <sh_def@xxxxxxx>
---
 kernel/sched/core.c  | 16 ++++++++++------
 kernel/sched/fair.c  |  6 ++----
 kernel/sched/sched.h |  1 +
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 15fcc218d2fe..dfef97ef8217 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5795,9 +5795,11 @@ void sched_tick(void)
 
 	donor->sched_class->task_tick(rq, donor, 0);
 
-	if (rq->curr->sched_class == &fair_sched_class &&
-	    static_branch_unlikely(&sched_numa_balancing))
-		task_tick_numa(rq, rq->curr);
+	if (rq->curr->sched_class == &fair_sched_class) {
+		if (static_branch_unlikely(&sched_numa_balancing))
+			task_tick_numa(rq, rq->curr);
+		task_tick_cache(rq, rq->curr);
+	}
 
 	if (sched_feat(LATENCY_WARN))
 		resched_latency = cpu_resched_latency(rq);
@@ -5896,9 +5898,11 @@ static void sched_tick_remote(struct work_struct *work)
 			}
 			curr->sched_class->task_tick(rq, curr, 0);
 
-			if (curr->sched_class == &fair_sched_class &&
-			    static_branch_unlikely(&sched_numa_balancing))
-				task_tick_numa(rq, curr);
+			if (curr->sched_class == &fair_sched_class) {
+				if (static_branch_unlikely(&sched_numa_balancing))
+					task_tick_numa(rq, curr);
+				task_tick_cache(rq, curr);
+			}
 
 			calc_load_nohz_remote(rq);
 		}
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 55f0460e4ae3..f335479ba562 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1774,7 +1774,7 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
 	}
 }
 
-static void task_tick_cache(struct rq *rq, struct task_struct *p)
+void task_tick_cache(struct rq *rq, struct task_struct *p)
 {
 	struct callback_head *work = &p->cache_work;
 	struct mm_struct *mm = p->mm;
@@ -1996,7 +1996,7 @@ static inline void account_mm_sched(struct rq *rq, struct task_struct *p,
 
 void init_sched_mm(struct task_struct *p) { }
 
-static void task_tick_cache(struct rq *rq, struct task_struct *p) { }
+void task_tick_cache(struct rq *rq, struct task_struct *p) { }
 
 static inline int get_pref_llc(struct task_struct *p,
 			       struct mm_struct *mm)
@@ -15042,8 +15042,6 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
 	if (queued)
 		return;
 
-	task_tick_cache(rq, curr);
-
 	update_misfit_status(curr, rq);
 	check_update_overutilized_status(task_rq(curr));
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4d619f272b15..5d1f5ee47bf1 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -4153,6 +4153,7 @@ void sched_domains_free_llc_id(int cpu);
 
 extern void init_sched_mm(struct task_struct *p);
 void task_tick_numa(struct rq *rq, struct task_struct *p);
+void task_tick_cache(struct rq *rq, struct task_struct *p);
 
 extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
 extern int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se);
-- 
2.54.0