[v2 2/9] sched/clock: interface to allow timestamps early in boot

[Pavel Tatashin]

In Linux printk() can output timestamps next to every line.  This is very
useful for tracking regressions, and finding places that can be optimized.
However, the timestamps are available only later in boot. On smaller
machines it is insignificant amount of time, but on larger it can be many
seconds or even minutes into the boot process.

This patch adds an interface for platforms with unstable sched clock to
show timestamps early in boot. In order to get this functionality a
platform must do:

- Implement u64 sched_clock_early()
  Clock that returns monotonic time

- Call sched_clock_early_init()
  Tells sched clock that the early clock can be used

- Call sched_clock_early_fini()
  Tells sched clock that the early clock is finished, and sched clock
  should hand over the operation to permanent clock.

Signed-off-by: Pavel Tatashin <pasha.tatashin@xxxxxxxxxx>
---
 include/linux/sched/clock.h |  4 +++
 kernel/sched/clock.c        | 61 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)

diff --git a/include/linux/sched/clock.h b/include/linux/sched/clock.h
index 4a68c67..9b06b3a 100644
--- a/include/linux/sched/clock.h
+++ b/include/linux/sched/clock.h
@@ -67,6 +67,10 @@ static inline u64 local_clock(void)
 extern void sched_clock_idle_sleep_event(void);
 extern void sched_clock_idle_wakeup_event(u64 delta_ns);
 
+void sched_clock_early_init(void);
+void sched_clock_early_fini(void);
+u64 sched_clock_early(void);
+
 /*
  * As outlined in clock.c, provides a fast, high resolution, nanosecond
  * time source that is monotonic per cpu argument and has bounded drift
diff --git a/kernel/sched/clock.c b/kernel/sched/clock.c
index 2bc1090..6a6d57b 100644
--- a/kernel/sched/clock.c
+++ b/kernel/sched/clock.c
@@ -79,6 +79,15 @@ unsigned long long __weak sched_clock(void)
 
 __read_mostly int sched_clock_running;
 
+/*
+ * We start with sched clock early static branch enabled, and global status
+ * disabled.  Early in boot it is decided whether to enable the global
+ * status as well (set sched_clock_early_running to true), and later, when
+ * early clock is no longer needed, the static branch is disabled.
+ */
+static DEFINE_STATIC_KEY_TRUE(__use_sched_clock_early);
+static bool __read_mostly sched_clock_early_running;
+
 void sched_clock_init(void)
 {
 	sched_clock_running = 1;
@@ -188,6 +197,12 @@ void sched_clock_init_late(void)
 	 */
 	smp_mb(); /* matches {set,clear}_sched_clock_stable() */
 
+	/*
+	 * It is guaranteed early clock is not running anymore. This function is
+	 * called from sched_init_smp(), and early clock must finish before smp.
+	 */
+	static_branch_disable(&__use_sched_clock_early);
+
 	if (__sched_clock_stable_early)
 		__set_sched_clock_stable();
 }
@@ -320,6 +335,11 @@ u64 sched_clock_cpu(int cpu)
 	if (sched_clock_stable())
 		return sched_clock() + raw_offset;
 
+	if (static_branch_unlikely(&__use_sched_clock_early)) {
+		if (sched_clock_early_running)
+			return sched_clock_early();
+	}
+
 	if (unlikely(!sched_clock_running))
 		return 0ull;
 
@@ -379,6 +399,47 @@ void sched_clock_idle_wakeup_event(u64 delta_ns)
 }
 EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
 
+u64 __weak sched_clock_early(void)
+{
+	return 0;
+}
+
+/*
+ * Is called when sched_clock_early() is about to be finished, notifies sched
+ * clock that after this call sched_clock_early() can't be used.
+ *
+ * Must be called before smp_init() as sched_clock_early() is supposed to be
+ * used only during early boot, and are not guarantee to handle multi-CPU
+ * environment properly.
+ */
+void __init sched_clock_early_fini(void)
+{
+	struct sched_clock_data *scd = this_scd();
+	u64 now_early = sched_clock_early();
+	u64 now_sched = sched_clock();
+
+	/*
+	 * Set both: gtod_offset and raw_offset because we could switch to
+	 * either unstable clock or stable clock.
+	 */
+	gtod_offset = now_early - scd->tick_gtod;
+	raw_offset = now_early - now_sched;
+
+	sched_clock_early_running = false;
+}
+
+/*
+ * Notifies sched clock that early boot clocksource is available, it means that
+ * the current platform has implemented sched_clock_early().
+ *
+ * The early clock is running until we switch to a stable clock, or when we
+ * learn that the stable clock is not available.
+ */
+void __init sched_clock_early_init(void)
+{
+	sched_clock_early_running = true;
+}
+
 #else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
 
 u64 sched_clock_cpu(int cpu)
-- 
1.8.3.1