[patch 30/48] hrtimer: Separate remove/enqueue handling for local timers

[Thomas Gleixner]

As the base switch can be avoided completely when the base stays the same
the remove/enqueue handling can be more streamlined.

Split it out into a separate function which handles both in one go which is
way more efficient and makes the code simpler to follow.

Signed-off-by: Thomas Gleixner <tglx@xxxxxxxxxx>
---
 kernel/time/hrtimer.c |   72 +++++++++++++++++++++++++++++---------------------
 1 file changed, 43 insertions(+), 29 deletions(-)

--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -1147,13 +1147,11 @@ static void __remove_hrtimer(struct hrti
 }
 
 static inline bool remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
-				 bool restart, bool keep_base)
+				  bool newstate)
 {
-	bool queued_state = timer->is_queued;
-
 	lockdep_assert_held(&base->cpu_base->lock);
 
-	if (queued_state) {
+	if (timer->is_queued) {
 		bool reprogram;
 
 		debug_hrtimer_deactivate(timer);
@@ -1168,23 +1166,35 @@ static inline bool remove_hrtimer(struct
 		 */
 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
 
-		/*
-		 * If the timer is not restarted then reprogramming is
-		 * required if the timer is local. If it is local and about
-		 * to be restarted, avoid programming it twice (on removal
-		 * and a moment later when it's requeued).
-		 */
-		if (!restart)
-			queued_state = HRTIMER_STATE_INACTIVE;
-		else
-			reprogram &= !keep_base;
-
-		__remove_hrtimer(timer, base, queued_state, reprogram);
+		__remove_hrtimer(timer, base, newstate, reprogram);
 		return true;
 	}
 	return false;
 }
 
+static inline bool
+remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
+			     const enum hrtimer_mode mode, ktime_t expires, u64 delta_ns)
+{
+	/* Remove it from the timer queue if active */
+	if (timer->is_queued) {
+		debug_hrtimer_deactivate(timer);
+		timerqueue_del(&base->active, &timer->node);
+	}
+
+	/* Set the new expiry time */
+	hrtimer_set_expires_range_ns(timer, expires, delta_ns);
+
+	debug_activate(timer, mode, timer->is_queued);
+	base->cpu_base->active_bases |= 1 << base->index;
+
+	/* Pairs with the lockless read in hrtimer_is_queued() */
+	WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED);
+
+	/* Returns true if this is the first expiring timer */
+	return timerqueue_add(&base->active, &timer->node);
+}
+
 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
 					    const enum hrtimer_mode mode)
 {
@@ -1267,7 +1277,7 @@ static bool __hrtimer_start_range_ns(str
 				     const enum hrtimer_mode mode, struct hrtimer_clock_base *base)
 {
 	struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases);
-	bool is_pinned, first, was_first, was_armed, keep_base = false;
+	bool is_pinned, first, was_first, keep_base = false;
 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;
 
 	was_first = cpu_base->next_timer == timer;
@@ -1283,6 +1293,12 @@ static bool __hrtimer_start_range_ns(str
 		keep_base = hrtimer_keep_base(timer, is_local, was_first, is_pinned);
 	}
 
+	/* Calculate absolute expiry time for relative timers */
+	if (mode & HRTIMER_MODE_REL)
+		tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid));
+	/* Compensate for low resolution granularity */
+	tim = hrtimer_update_lowres(timer, tim, mode);
+
 	/*
 	 * Remove an active timer from the queue. In case it is not queued
 	 * on the current CPU, make sure that remove_hrtimer() updates the
@@ -1297,22 +1313,20 @@ static bool __hrtimer_start_range_ns(str
 	 * @keep_base is also true if the timer callback is running on a
 	 * remote CPU and for local pinned timers.
 	 */
-	was_armed = remove_hrtimer(timer, base, true, keep_base);
-
-	if (mode & HRTIMER_MODE_REL)
-		tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid));
-
-	tim = hrtimer_update_lowres(timer, tim, mode);
+	if (likely(keep_base)) {
+		first = remove_and_enqueue_same_base(timer, base, mode, tim, delta_ns);
+	} else {
+		/* Keep the ENQUEUED state in case it is queued */
+		bool was_armed = remove_hrtimer(timer, base, HRTIMER_STATE_ENQUEUED);
 
-	hrtimer_set_expires_range_ns(timer, tim, delta_ns);
+		hrtimer_set_expires_range_ns(timer, tim, delta_ns);
 
-	/* Switch the timer base, if necessary: */
-	if (!keep_base) {
+		/* Switch the timer base, if necessary: */
 		base = switch_hrtimer_base(timer, base, is_pinned);
 		cpu_base = base->cpu_base;
-	}
 
-	first = enqueue_hrtimer(timer, base, mode, was_armed);
+		first = enqueue_hrtimer(timer, base, mode, was_armed);
+	}
 
 	/*
 	 * If the hrtimer interrupt is running, then it will reevaluate the
@@ -1432,7 +1446,7 @@ int hrtimer_try_to_cancel(struct hrtimer
 	base = lock_hrtimer_base(timer, &flags);
 
 	if (!hrtimer_callback_running(timer)) {
-		ret = remove_hrtimer(timer, base, false, false);
+		ret = remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE);
 		if (ret)
 			trace_hrtimer_cancel(timer);
 	}