[RFC PATCH v3 07/17] sched/topology: Add per-sd scratch for cache-aware load balancing

[Jianyong Wu]

Cache-aware load balancing needs arrays of eligible LLC/node IDs and
affinity weights when scoring candidate runqueues. Selecting a source
runqueue by LLC load also needs a temporary CPU mask to avoid sampling
the same LLC more than once.

Allocate these buffers on the bottom sched domain of each CPU, next to
llc_counts, rather than allocating memory in the load balance path.
Size both affinity arrays by llc_max and the independent statistics mask
by cpumask_size().

Signed-off-by: Jianyong Wu <wujianyong@xxxxxxxx>
---
 include/linux/sched/topology.h |  7 +++++++
 kernel/sched/topology.c        | 37 +++++++++++++++++++++++++++++++++-
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index b5d9d7c2b8ad..e41fc64391c3 100644
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -121,6 +121,13 @@ struct sched_domain {
 	unsigned int llc_max;
 	unsigned int *llc_counts __counted_by_ptr(llc_max);
 	unsigned long llc_bytes;
+	/*
+	 * Scratch for the load balance affinity score, llc_max entries each.
+	 * Only the bottom sd owns them, like llc_counts above.
+	 */
+	int *affi_ids __counted_by_ptr(llc_max);
+	int *affi_weights __counted_by_ptr(llc_max);
+	struct cpumask *llc_stats_mask;
 #endif
 
 #ifdef CONFIG_SCHEDSTATS
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 37f1eeae9b7b..7d65a4d7b0dd 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -643,6 +643,9 @@ static void destroy_sched_domain(struct sched_domain *sd)
 #ifdef CONFIG_SCHED_CACHE
 	/* only the bottom sd has llc_counts array */
 	kfree(sd->llc_counts);
+	kfree(sd->affi_ids);
+	kfree(sd->affi_weights);
+	kfree(sd->llc_stats_mask);
 #endif
 	kfree(sd);
 }
@@ -846,10 +849,16 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
 			sd->llc_counts = tmp->llc_counts;
 			sd->llc_max = tmp->llc_max;
 			sd->llc_bytes = tmp->llc_bytes;
+			sd->affi_ids = tmp->affi_ids;
+			sd->affi_weights = tmp->affi_weights;
+			sd->llc_stats_mask = tmp->llc_stats_mask;
 			/* make sure destroy_sched_domain() does not free it */
 			tmp->llc_counts = NULL;
 			tmp->llc_max = 0;
 			tmp->llc_bytes = 0;
+			tmp->affi_ids = NULL;
+			tmp->affi_weights = NULL;
+			tmp->llc_stats_mask = NULL;
 #endif
 			/*
 			 * sched groups hold the flags of the child sched
@@ -1307,6 +1316,8 @@ static bool alloc_sd_llc(const struct cpumask *cpu_map,
 {
 	struct sched_domain *sd, *top_llc, *parent;
 	unsigned int *p;
+	int *p_ids, *p_w;
+	struct cpumask *stats_mask;
 	int i;
 
 	for_each_cpu(i, cpu_map) {
@@ -1316,8 +1327,20 @@ static bool alloc_sd_llc(const struct cpumask *cpu_map,
 
 		p = kcalloc_node(max_lid + 1, sizeof(unsigned int),
 				 GFP_KERNEL, cpu_to_node(i));
-		if (!p)
+		/* Scratch for the load balance affinity score. */
+		p_ids = kcalloc_node(max_lid + 1, sizeof(int),
+				     GFP_KERNEL, cpu_to_node(i));
+		p_w = kcalloc_node(max_lid + 1, sizeof(int),
+				   GFP_KERNEL, cpu_to_node(i));
+		stats_mask = kmalloc_node(cpumask_size(), GFP_KERNEL,
+					 cpu_to_node(i));
+		if (!p || !p_ids || !p_w || !stats_mask) {
+			kfree(p);
+			kfree(p_ids);
+			kfree(p_w);
+			kfree(stats_mask);
 			goto err;
+		}
 
 		top_llc = sd;
 		/*
@@ -1333,9 +1356,15 @@ static bool alloc_sd_llc(const struct cpumask *cpu_map,
 			sd->llc_max = max_lid + 1;
 			sd->llc_counts = p;
 			sd->llc_bytes = get_effective_llc_bytes(i, top_llc);
+			sd->affi_ids = p_ids;
+			sd->affi_weights = p_w;
+			sd->llc_stats_mask = stats_mask;
 		} else {
 			/* avoid memory leak */
 			kfree(p);
+			kfree(p_ids);
+			kfree(p_w);
+			kfree(stats_mask);
 		}
 	}
 
@@ -1345,9 +1374,15 @@ static bool alloc_sd_llc(const struct cpumask *cpu_map,
 		sd = *per_cpu_ptr(d->sd, i);
 		if (sd) {
 			kfree(sd->llc_counts);
+			kfree(sd->affi_ids);
+			kfree(sd->affi_weights);
+			kfree(sd->llc_stats_mask);
 			sd->llc_counts = NULL;
 			sd->llc_max = 0;
 			sd->llc_bytes = 0;
+			sd->affi_ids = NULL;
+			sd->affi_weights = NULL;
+			sd->llc_stats_mask = NULL;
 		}
 	}
 
-- 
2.34.1