[RFC PATCH 3/4] timekeeping: Add absolute reference for feed-forward clock discipline

[David Woodhouse]

From: David Woodhouse <dwmw@xxxxxxxxxxxx>

Add timekeeping_set_reference() which allows an external clock source
(e.g. vmclock from a hypervisor) to provide an absolute time reference.
The reference defines a linear TSC-to-time mapping that the tick
mechanism clamps to, replacing the relative ntp_error accumulator for
the dithering decision.

When a reference is active:
 - ntp_tick is set to match the reference rate via the normal NTP path
 - The dithering decision (mult vs mult+1) uses an absolute comparison
   against the reference line instead of the ntp_error accumulator
 - timekeeping_apply_adjustment() still runs for vDSO monotonicity
 - adjtimex reads back the correct frequency

The reference is automatically cleared when:
 - adjtimex ADJ_FREQUENCY is called (NTP takes over)
 - The clocksource changes

This eliminates the ~26 PPB systematic drift caused by the interaction
between timekeeping_apply_adjustment()'s monotonicity correction and
the ntp_error accumulator, which depends on interrupt latency.

Signed-off-by: David Woodhouse <dwmw@xxxxxxxxxxxx>
---
 include/linux/timekeeping_reference.h | 35 ++++++++++++
 kernel/time/ntp.c                     | 33 ++++++++++++
 kernel/time/ntp_internal.h            |  2 +
 kernel/time/timekeeping.c             | 76 ++++++++++++++++++++++++++-
 4 files changed, 145 insertions(+), 1 deletion(-)
 create mode 100644 include/linux/timekeeping_reference.h

diff --git a/include/linux/timekeeping_reference.h b/include/linux/timekeeping_reference.h
new file mode 100644
index 000000000000..0cf248ace241
--- /dev/null
+++ b/include/linux/timekeeping_reference.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_TIMEKEEPING_REFERENCE_H
+#define _LINUX_TIMEKEEPING_REFERENCE_H
+
+#include <linux/clocksource_ids.h>
+#include <linux/types.h>
+
+struct timekeeper;
+
+/**
+ * struct tk_reference - Absolute time reference for feed-forward timekeeping
+ * @cs_id:		Clocksource counter this reference applies to
+ * @counter_value:	Counter reading at the reference point
+ * @cycle_interval:	Counter cycles per tick (for ntp_tick computation)
+ * @time_sec:		Seconds (UTC) at the reference point
+ * @time_frac_sec:	Fractional seconds (units of 1/2^64 second)
+ * @period_frac_sec:	Counter period (units of 1/2^(64+shift) seconds)
+ * @period_shift:	Additional shift for period fixed-point
+ */
+struct tk_reference {
+	enum clocksource_ids	cs_id;
+	u64			counter_value;
+	u64			cycle_interval;
+	u64			time_sec;
+	u64			time_frac_sec;
+	u64			period_frac_sec;
+	u8			period_shift;
+};
+
+int timekeeping_set_reference(const struct tk_reference *ref);
+bool timekeeping_has_reference(void);
+void timekeeping_clear_reference(void);
+bool timekeeping_ref_ahead(struct timekeeper *tk);
+
+#endif /* _LINUX_TIMEKEEPING_REFERENCE_H */
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index 3c318c96f35d..cdd63589160d 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -21,6 +21,7 @@
 #include <linux/timekeeper_internal.h>
 
 #include "ntp_internal.h"
+#include <linux/timekeeping_reference.h>
 #include "timekeeping_internal.h"
 
 /**
@@ -364,6 +365,37 @@ u64 ntp_tick_length(unsigned int tkid)
 	return tk_ntp_data[tkid].tick_length;
 }
 
+void ntp_set_tick_length(unsigned int tkid, u64 tick_length)
+{
+	struct ntp_data *ntpdata = &tk_ntp_data[tkid];
+	u64 base;
+
+	/*
+	 * Reverse ntp_update_frequency() to find the time_freq that
+	 * produces this tick_length, keeping everything consistent.
+	 *
+	 * tick_length = ((tick_usec * 1000 * USER_HZ) << 32 +
+	 *                ntp_tick_adj + time_freq) / NTP_INTERVAL_FREQ
+	 *
+	 * time_freq = tick_length * NTP_INTERVAL_FREQ -
+	 *             (tick_usec * 1000 * USER_HZ) << 32 - ntp_tick_adj
+	 */
+	base = (u64)(ntpdata->tick_usec * NSEC_PER_USEC * USER_HZ) << NTP_SCALE_SHIFT;
+	base += ntpdata->ntp_tick_adj;
+
+	ntpdata->time_freq = (s64)(tick_length * NTP_INTERVAL_FREQ - base);
+	ntp_update_frequency(ntpdata);
+}
+
+void ntp_set_time_offset(unsigned int tkid, s64 offset_ns)
+{
+	struct ntp_data *ntpdata = &tk_ntp_data[tkid];
+
+	ntpdata->time_offset = div_s64((s64)offset_ns << NTP_SCALE_SHIFT,
+				       NTP_INTERVAL_FREQ);
+	ntpdata->time_adjust = 0;
+}
+
 /**
  * ntp_get_next_leap - Returns the next leapsecond in CLOCK_REALTIME ktime_t
  * @tkid:	Timekeeper ID
@@ -736,6 +768,7 @@ static inline void process_adjtimex_modes(struct ntp_data *ntpdata, const struct
 		ntpdata->time_freq = max(ntpdata->time_freq, -MAXFREQ_SCALED);
 		/* Update pps_freq */
 		pps_set_freq(ntpdata);
+		timekeeping_clear_reference();
 	}
 
 	if (txc->modes & ADJ_MAXERROR)
diff --git a/kernel/time/ntp_internal.h b/kernel/time/ntp_internal.h
index b36a8090fc9c..4531c162d229 100644
--- a/kernel/time/ntp_internal.h
+++ b/kernel/time/ntp_internal.h
@@ -7,6 +7,8 @@ extern bool ntp_synced(void);
 extern void ntp_clear(unsigned int tkid);
 /* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */
 extern u64 ntp_tick_length(unsigned int tkid);
+extern void ntp_set_tick_length(unsigned int tkid, u64 length);
+extern void ntp_set_time_offset(unsigned int tkid, s64 offset_ns);
 extern ktime_t ntp_get_next_leap(unsigned int tkid);
 extern int second_overflow(unsigned int tkid, time64_t secs);
 extern int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct timespec64 *ts,
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 1935881041d0..1225efdf5dc0 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -27,6 +27,8 @@
 #include "tick-internal.h"
 #include "timekeeping_internal.h"
 #include "ntp_internal.h"
+#include <linux/timekeeping_reference.h>
+#include <linux/math64.h>
 #include <linux/vmclock_host.h>
 
 void (*vmclock_host_update_fn)(struct timekeeper *tk);
@@ -396,6 +398,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
 	tk->skip_second_overflow = 0;
 
 	tk->cs_id = clock->id;
+	timekeeping_clear_reference();
 
 	/* Coupled clockevent data */
 	if (IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED) &&
@@ -2323,9 +2326,77 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
 	tk->tkr_mono.xtime_nsec -= offset;
 }
 
+static struct tk_reference tk_ref;
+static bool tk_ref_valid;
+
+int timekeeping_set_reference(const struct tk_reference *ref)
+{
+	struct timekeeper *tk = &tk_core.timekeeper;
+	__uint128_t product;
+	u64 delta, ref_frac, ref_ns;
+	s64 offset_ns;
+
+	tk_ref = *ref;
+	if (!tk_ref.cycle_interval)
+		tk_ref.cycle_interval = tk->cycle_interval;
+
+	/* Reject if the clocksource doesn't match */
+	if (tk->cs_id != ref->cs_id)
+		return -ENODEV;
+
+	tk_ref_valid = true;
+	ntp_set_tick_length(TIMEKEEPER_CORE,
+		mul_u64_u64_shr(ref->period_frac_sec,
+				(u64)tk_ref.cycle_interval * NSEC_PER_SEC,
+				32 + ref->period_shift));
+
+	/* Compute phase offset: (reference_time - xtime) in ns */
+	delta = tk->tkr_mono.cycle_last - tk_ref.counter_value;
+	product = (__uint128_t)delta * tk_ref.period_frac_sec;
+	product >>= tk_ref.period_shift;
+	product += tk_ref.time_frac_sec;
+	ref_frac = (u64)product;
+	ref_ns = mul_u64_u64_shr(ref_frac, NSEC_PER_SEC, 64);
+
+	if (tk_ref.time_sec + (u64)(product >> 64) == tk->xtime_sec) {
+		offset_ns = (s64)ref_ns -
+			(s64)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
+		ntp_set_time_offset(TIMEKEEPER_CORE, offset_ns);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(timekeeping_set_reference);
+
+bool timekeeping_has_reference(void) { return tk_ref_valid; }
+void timekeeping_clear_reference(void) { tk_ref_valid = false; }
+
+bool timekeeping_ref_ahead(struct timekeeper *tk)
+{
+	u64 delta, ref_frac, ref_sec, ref_shifted_ns;
+	__uint128_t product;
+
+	if (tk->cs_id != tk_ref.cs_id)
+		return false;
+	delta = tk->tkr_mono.cycle_last - tk_ref.counter_value;
+	product = (__uint128_t)delta * tk_ref.period_frac_sec;
+	product >>= tk_ref.period_shift;
+	product += tk_ref.time_frac_sec;
+	ref_sec = tk_ref.time_sec + (u64)(product >> 64);
+	ref_frac = (u64)product;
+	ref_shifted_ns = mul_u64_u64_shr(ref_frac,
+				(u64)NSEC_PER_SEC << tk->tkr_mono.shift, 64);
+	if (tk->xtime_sec > ref_sec)
+		return true;
+	if (tk->xtime_sec == ref_sec &&
+	    tk->tkr_mono.xtime_nsec > ref_shifted_ns)
+		return true;
+	return false;
+}
 /*
  * Adjust the timekeeper's multiplier to the correct frequency
  * and also to reduce the accumulated error value.
+
  */
 static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
 {
@@ -2352,7 +2423,10 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
 	 * tick division, the clock will slow down. Otherwise it will stay
 	 * ahead until the tick length changes to a non-divisible value.
 	 */
-	tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;
+	if (timekeeping_has_reference())
+		tk->ntp_err_mult = timekeeping_ref_ahead(tk) ? 0 : 1;
+	else
+		tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;
 	mult += tk->ntp_err_mult;
 
 	timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult);
-- 
2.51.0