[RFC 24/60] cosched: Do minimal pre-SMP coscheduler initialization

[Jan H. SchÃnherr]

The scheduler is operational before we have the necessary information
about scheduling domains, which would allow us to set up the runqueue
hierarchy. Because of that, we have to postpone the "real"
initialization a bit. We cannot not totally skip all initialization,
though, because all the adapted scheduler code paths do not really
like NULL pointers within the extended coscheduling related data
structures.

Perform a minimal setup of the coscheduling data structures at the
bottom level (i.e., CPU level), so that all traversals terminate
correctly. There is no hierarchy yet.

Signed-off-by: Jan H. SchÃnherr <jschoenh@xxxxxxxxx>
---
 kernel/sched/core.c    |  2 ++
 kernel/sched/cosched.c | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++
 kernel/sched/sched.h   |  6 ++++
 3 files changed, 93 insertions(+)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 48e37c3baed1..a235b6041cb5 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6058,6 +6058,8 @@ void __init sched_init(void)
 		atomic_set(&rq->nr_iowait, 0);
 	}
 
+	cosched_init_bottom();
+
 	set_load_weight(&init_task, false);
 
 	/*
diff --git a/kernel/sched/cosched.c b/kernel/sched/cosched.c
index 3bd4557ca5b7..03ba86676b90 100644
--- a/kernel/sched/cosched.c
+++ b/kernel/sched/cosched.c
@@ -7,3 +7,88 @@
  */
 
 #include "sched.h"
+
+static int mask_to_node(const struct cpumask *span)
+{
+	int node = cpu_to_node(cpumask_first(span));
+
+	if (cpumask_subset(span, cpumask_of_node(node)))
+		return node;
+
+	return NUMA_NO_NODE;
+}
+
+static void init_sdrq_data(struct sdrq_data *data, struct sdrq_data *parent,
+			   const struct cpumask *span, int level)
+{
+	struct rq *rq = container_of(data, struct rq, sdrq_data);
+	int cpu = cpumask_first(span);
+	int weight = cpumask_weight(span);
+
+	data->numa_node = weight == 1 ? cpu_to_node(cpu) : mask_to_node(span);
+	data->leader = cpu;
+	data->level = level >= 0 ? level : parent->level - 1;
+	data->parent = parent;
+	data->span = span;
+	data->span_weight = weight;
+	data->current_sdrq = &rq->cfs.sdrq;
+
+	/*
+	 * Bottom level runqueues have already been initialized and are live,
+	 * do not initialize them again.
+	 */
+	if (!data->level)
+		return;
+
+	/* Initialize the subset of struct rq in which we are interested. */
+	raw_spin_lock_init(&rq->lock);
+	rq->cpu = cpu;
+	INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
+	rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
+}
+
+static void init_sdrq(struct task_group *tg, struct sdrq *sdrq,
+		      struct sdrq *sd_parent, struct sdrq *tg_parent,
+		      struct sdrq_data *data)
+{
+	struct rq *rq = container_of(data, struct rq, sdrq_data);
+	struct cfs_rq *cfs_rq = container_of(sdrq, struct cfs_rq, sdrq);
+
+	/* Attention: tg->parent may not yet be set. Check tg_parent instead. */
+
+	sdrq->sd_parent = sd_parent;
+	sdrq->tg_parent = tg_parent;
+	sdrq->data = data;
+	sdrq->is_root = 1;
+	sdrq->cfs_rq = cfs_rq;
+	sdrq->tg_se = cfs_rq->my_se;
+
+	INIT_LIST_HEAD(&sdrq->tg_children);
+	INIT_LIST_HEAD(&sdrq->children);
+	if (sd_parent)
+		list_add_tail(&sdrq->siblings, &sd_parent->children);
+
+	/*
+	 * If we are not at the bottom level in hierarchy, we need to setup
+	 * a SD-SE, so that the level below us can be represented within this
+	 * level.
+	 */
+	if (data->level) {
+		sdrq->sd_se = &sdrq->__sd_se;
+		init_tg_cfs_entry(tg, NULL, sdrq->sd_se, rq, sdrq->cfs_rq);
+	}
+}
+
+void cosched_init_bottom(void)
+{
+	int cpu;
+
+	raw_spin_lock_init(&root_task_group.lock);
+	for_each_possible_cpu(cpu) {
+		struct sdrq_data *data = &cpu_rq(cpu)->sdrq_data;
+		struct sdrq *sdrq = &root_task_group.cfs_rq[cpu]->sdrq;
+
+		init_sdrq_data(data, NULL, cpumask_of(cpu), 0);
+		init_sdrq(&root_task_group, sdrq, NULL, NULL, data);
+	}
+}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 1bce6061ac45..21b7c6cf8b87 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1129,6 +1129,12 @@ static inline struct cfs_rq *taskgroup_next_cfsrq(struct task_group *tg,
 }
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 
+#ifdef CONFIG_COSCHEDULING
+void cosched_init_bottom(void);
+#else /* !CONFIG_COSCHEDULING */
+static inline void cosched_init_bottom(void) { }
+#endif /* !CONFIG_COSCHEDULING */
+
 #ifdef CONFIG_SCHED_SMT
 
 extern struct static_key_false sched_smt_present;
-- 
2.9.3.1.gcba166c.dirty