[PATCH 4/6] sched/numa: make free-space tiering promotion bypass configurable

[Li Zhe]

Memory tiering normally promotes any recently accessed slow-tier page if
the target fast-tier node has enough free space. This preserves the
existing policy of filling available fast memory before relying on
hotness and rate-limit control.

That is not always the desired policy. Some workloads may intentionally
place memory across CXL-attached memory and DRAM to use aggregate memory
bandwidth. With the free-space bypass, cold one-shot accesses and bursts
of hot accesses can refill the fast tier and pull the workload away from
that distribution whenever DRAM has free space.

Add kernel.numa_balancing_promote_free_space_bypass. It defaults to 1,
preserving the existing behavior. When cleared, enough free space in the
target fast-tier node no longer bypasses the normal hotness and
rate-limit controls.

Signed-off-by: Li Zhe <lizhe.67@xxxxxxxxxxxxx>
---
 Documentation/admin-guide/sysctl/kernel.rst | 16 ++++++++++++++
 include/linux/mmzone.h                      |  5 ++---
 kernel/sched/fair.c                         | 23 +++++++++++++++------
 3 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index ffea61d448eb..4a035b18aff4 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -765,6 +765,22 @@ will be limited to be no more than the set value.
 A rule of thumb is to set this to less than 1/10 of the PMEM node
 write bandwidth.
 
+numa_balancing_promote_free_space_bypass
+========================================
+
+Controls whether memory tiering promotion may bypass the normal
+hint-fault hotness check and promotion rate limit when the target
+fast-tier node has enough free space.
+
+The default value is 1.  In this mode, if the fast tier has enough free
+space, recently accessed slow-tier pages can be promoted without the
+hotness check or the promotion rate limit, so the kernel can fill
+available fast-tier capacity.
+
+When set to 0, enough free space no longer bypasses these controls.
+A slow-tier page must pass the normal hotness and rate-limit checks before
+promotion.
+
 oops_all_cpu_backtrace
 ======================
 
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 94f9c3ff5416..5d583b17dfa2 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -297,9 +297,8 @@ enum node_stat_item {
 	 */
 	PGPROMOTE_CANDIDATE,
 	/**
-	 * Not rate-limited (NRL) candidate pages for those can be promoted
-	 * without considering hot threshold because of enough free pages in
-	 * fast-tier node.  These promotions bypass the regular hotness checks
+	 * Not rate-limited (NRL) candidate pages that are promoted through the
+	 * free-space bypass. These promotions bypass the regular hotness checks
 	 * and do NOT influence the promotion rate-limiter or
 	 * threshold-adjustment logic.
 	 * This is for statistics/monitoring purposes.
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index ec8a6bfeb2c7..2e11107bf5ca 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -134,6 +134,8 @@ static unsigned int sysctl_sched_cfs_bandwidth_slice		= 5000UL;
 #ifdef CONFIG_NUMA_BALANCING
 /* Restrict the NUMA promotion throughput (MB/s) for each target node. */
 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536;
+/* Preserve the existing default of using free fast-tier space eagerly. */
+static unsigned int sysctl_numa_balancing_promote_free_space_bypass = 1;
 #endif
 
 #ifdef CONFIG_SYSCTL
@@ -157,6 +159,15 @@ static const struct ctl_table sched_fair_sysctls[] = {
 		.proc_handler	= proc_dointvec_minmax,
 		.extra1		= SYSCTL_ZERO,
 	},
+	{
+		.procname	= "numa_balancing_promote_free_space_bypass",
+		.data		= &sysctl_numa_balancing_promote_free_space_bypass,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= proc_douintvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
 #endif /* CONFIG_NUMA_BALANCING */
 };
 
@@ -2816,11 +2827,10 @@ static inline bool cpupid_valid(int cpupid)
 }
 
 /*
- * For memory tiering mode, if there are enough free pages (more than
- * enough watermark defined here) in fast memory node, to take full
- * advantage of fast memory capacity, all recently accessed slow
- * memory pages will be migrated to fast memory node without
- * considering hot threshold.
+ * For memory tiering mode, if the target fast memory node has enough free
+ * pages, recently accessed slow memory pages can bypass the normal hotness
+ * check and rate limit so the fast tier can be filled. The bypass can be
+ * disabled by numa_balancing_promote_free_space_bypass.
  */
 static bool pgdat_free_space_enough(struct pglist_data *pgdat)
 {
@@ -2941,7 +2951,8 @@ bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio,
 		long nr = folio_nr_pages(folio);
 
 		pgdat = NODE_DATA(dst_nid);
-		if (pgdat_free_space_enough(pgdat)) {
+		if (sysctl_numa_balancing_promote_free_space_bypass &&
+		    pgdat_free_space_enough(pgdat)) {
 			/* workload changed, reset hot threshold */
 			pgdat->nbp_threshold = 0;
 			mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE_NRL, nr);
-- 
2.20.1