[PATCH v3 18/21] sched/cache: Allow the user space to turn on and off cache aware scheduling

[Tim Chen]

From: Chen Yu <yu.c.chen@xxxxxxxxx>

Provide a debugfs knob to allow the user to turn off and on the
cache aware scheduling at runtime.

Signed-off-by: Chen Yu <yu.c.chen@xxxxxxxxx>
Signed-off-by: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx>
---

Notes:
    v2->v3:
    Split into a new patch for better review, use kstrtobool_from_user()
    to get the user input.  (Peter Zijlstra)

 kernel/sched/debug.c    | 45 ++++++++++++++++++++++++++++
 kernel/sched/sched.h    |  7 +++--
 kernel/sched/topology.c | 65 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 115 insertions(+), 2 deletions(-)

diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 41caa22e0680..bae747eddc59 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -215,6 +215,46 @@ static const struct file_operations sched_scaling_fops = {
 	.release	= single_release,
 };
 
+#ifdef CONFIG_SCHED_CACHE
+static ssize_t
+sched_cache_enable_write(struct file *filp, const char __user *ubuf,
+			 size_t cnt, loff_t *ppos)
+{
+	bool val;
+	int ret;
+
+	ret = kstrtobool_from_user(ubuf, cnt, &val);
+	if (ret)
+		return ret;
+
+	sysctl_sched_cache_user = val;
+
+	sched_cache_active_set_unlocked();
+
+	return cnt;
+}
+
+static int sched_cache_enable_show(struct seq_file *m, void *v)
+{
+	seq_printf(m, "%d\n", sysctl_sched_cache_user);
+	return 0;
+}
+
+static int sched_cache_enable_open(struct inode *inode,
+				   struct file *filp)
+{
+	return single_open(filp, sched_cache_enable_show, NULL);
+}
+
+static const struct file_operations sched_cache_enable_fops = {
+	.open           = sched_cache_enable_open,
+	.write          = sched_cache_enable_write,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = single_release,
+};
+#endif
+
 #ifdef CONFIG_PREEMPT_DYNAMIC
 
 static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
@@ -523,6 +563,11 @@ static __init int sched_init_debug(void)
 	debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold);
 #endif /* CONFIG_NUMA_BALANCING */
 
+#ifdef CONFIG_SCHED_CACHE
+	debugfs_create_file("llc_enabled", 0644, debugfs_sched, NULL,
+			    &sched_cache_enable_fops);
+#endif
+
 	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
 
 	debugfs_fair_server_init();
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 59ac04625842..adf3428745dd 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3917,12 +3917,15 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct
 
 #ifdef CONFIG_SCHED_CACHE
 DECLARE_STATIC_KEY_FALSE(sched_cache_present);
-extern int max_llcs;
+DECLARE_STATIC_KEY_FALSE(sched_cache_active);
+extern int max_llcs, sysctl_sched_cache_user;
 
 static inline bool sched_cache_enabled(void)
 {
-	return static_branch_unlikely(&sched_cache_present);
+	return static_branch_unlikely(&sched_cache_active);
 }
+
+extern void sched_cache_active_set_unlocked(void);
 #endif
 extern void init_sched_mm(struct task_struct *p);
 
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 9104fed25351..e86dea1b9e86 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -801,7 +801,16 @@ enum s_alloc {
 };
 
 #ifdef CONFIG_SCHED_CACHE
+/* hardware support for cache aware scheduling */
 DEFINE_STATIC_KEY_FALSE(sched_cache_present);
+/*
+ * Indicator of whether cache aware scheduling
+ * is active, used by the scheduler.
+ */
+DEFINE_STATIC_KEY_FALSE(sched_cache_active);
+/* user wants cache aware scheduling [0 or 1] */
+int sysctl_sched_cache_user = 1;
+
 static bool alloc_sd_pref(const struct cpumask *cpu_map,
 			  struct s_data *d)
 {
@@ -833,6 +842,60 @@ static bool alloc_sd_pref(const struct cpumask *cpu_map,
 
 	return false;
 }
+
+static void _sched_cache_active_set(bool enable, bool locked)
+{
+	if (enable) {
+		if (locked)
+			static_branch_enable_cpuslocked(&sched_cache_active);
+		else
+			static_branch_enable(&sched_cache_active);
+	} else {
+		if (locked)
+			static_branch_disable_cpuslocked(&sched_cache_active);
+		else
+			static_branch_disable(&sched_cache_active);
+	}
+}
+
+/*
+ * Enable/disable cache aware scheduling according to
+ * user input and the presence of hardware support.
+ */
+static void sched_cache_active_set(bool locked)
+{
+	/* hardware does not support */
+	if (!static_branch_likely(&sched_cache_present)) {
+		_sched_cache_active_set(false, locked);
+		return;
+	}
+
+	/*
+	 * user wants it or not ?
+	 * TBD: read before writing the static key.
+	 * It is not in the critical path, leave as-is
+	 * for now.
+	 */
+	if (sysctl_sched_cache_user) {
+		_sched_cache_active_set(true, locked);
+		if (sched_debug())
+			pr_info("%s: enabling cache aware scheduling\n", __func__);
+	} else {
+		_sched_cache_active_set(false, locked);
+		if (sched_debug())
+			pr_info("%s: disabling cache aware scheduling\n", __func__);
+	}
+}
+
+static void sched_cache_active_set_locked(void)
+{
+	return sched_cache_active_set(true);
+}
+
+void sched_cache_active_set_unlocked(void)
+{
+	return sched_cache_active_set(false);
+}
 #else
 static bool alloc_sd_pref(const struct cpumask *cpu_map,
 			  struct s_data *d)
@@ -2809,6 +2872,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 		static_branch_enable_cpuslocked(&sched_cache_present);
 	else
 		static_branch_disable_cpuslocked(&sched_cache_present);
+
+	sched_cache_active_set_locked();
 #endif
 	__free_domain_allocs(&d, alloc_state, cpu_map);
 
-- 
2.32.0