[PATCH v3 06/20] sched/core: allow only preferred CPUs in is_cpu_allowed

[Shrikanth Hegde]

When possible, choose a preferred CPUs to pick.

Push task mechanism uses stopper thread which going to call
select_fallback_rq and use this mechanism to pick only a preferred CPU.

When task is affined only to non-preferred CPUs it should continue to
run there. Detect that by checking if cpus_ptr and cpu_preferred_mask
intersect or not.

Signed-off-by: Shrikanth Hegde <sshegde@xxxxxxxxxxxxx>
---
 kernel/sched/core.c  | 27 +++++++++++++++++++++++++--
 kernel/sched/sched.h |  5 +++++
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3ae5f19c1b7e..292d4e7db0fd 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2468,6 +2468,8 @@ static inline bool rq_has_pinned_tasks(struct rq *rq)
  */
 static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
 {
+	bool task_has_preferred_cpu = false;
+
 	/* When not in the task's cpumask, no point in looking further. */
 	if (!task_allowed_on_cpu(p, cpu))
 		return false;
@@ -2476,9 +2478,26 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
 	if (is_migration_disabled(p))
 		return cpu_online(cpu);
 
+	/*
+	 * This is essential to maintain user affinities when preferred
+	 * CPUs change. A task pinned on non-preferred CPU should continue
+	 * to run there, since this is non-user triggered.
+	 *
+	 * For majority of the cases this would still keep select_fallback_rq
+	 * as O(N). task_has_preferred_cpus which is O(N) is called only if
+	 * !cpu_preferred. Then task running there is expected to move out.
+	 * So subsequent it should run on preferred CPU. This becomes O(N**2)
+	 * only for tasks pinned only non preferred CPUs. That is rare case.
+	 */
+	task_has_preferred_cpu = !cpu_preferred(cpu) && task_has_preferred_cpus(p);
+
 	/* Non kernel threads are not allowed during either online or offline. */
-	if (!(p->flags & PF_KTHREAD))
-		return cpu_active(cpu);
+	if (!(p->flags & PF_KTHREAD)) {
+		if (!cpu_active(cpu))
+			return false;
+		if (task_has_preferred_cpu)
+			return false;
+	}
 
 	/* KTHREAD_IS_PER_CPU is always allowed. */
 	if (kthread_is_per_cpu(p))
@@ -2488,6 +2507,10 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
 	if (cpu_dying(cpu))
 		return false;
 
+	/* Try on preferred CPU first if possible*/
+	if (task_has_preferred_cpu)
+		return false;
+
 	/* But are allowed during online. */
 	return cpu_online(cpu);
 }
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index ffe77b2b6296..faf36bc7bd12 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -4130,4 +4130,9 @@ DEFINE_CLASS_IS_UNCONDITIONAL(sched_change)
 
 #include "ext.h"
 
+static inline bool task_has_preferred_cpus(struct task_struct *p)
+{
+	return cpumask_intersects(p->cpus_ptr, cpu_preferred_mask);
+}
+
 #endif /* _KERNEL_SCHED_SCHED_H */
-- 
2.47.3