[PATCH 17/25] hrtimer: Implementation of softirq hrtimer handling

[Anna-Maria Gleixner]

From: Anna-Maria Gleixner <anna-maria@xxxxxxxxxxxxx>

hrtimers are executed always in hard irq context. If a hrtimer callback
function needs to be exectued in softirq context, the detour using tasklets
is required. To facilitate this, also in regards to real time specific
handling of hrtimers, new clock ids ease the use of hrtimers in softirq
context.

Every clock ID is available for soft and hard hrtimers. The hrtimers are
handled the same way when they are enqueued. When the hrtimer_interrupt
raises, a check is implemented, if the HRTIMER_SOFTIRQ has to be raised as
well. If it is raised, the soft hrtimers are not taken into account when
for example _hrtimer_get_next_event() is called. At the end of the softirq,
all hrtimer_cpu_base struct members are updated, so that the soft hrtimers
are also taken into account.

Suggested-by: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Signed-off-by: Anna-Maria Gleixner <anna-maria@xxxxxxxxxxxxx>
---
 include/linux/hrtimer.h |    8 ++-
 kernel/time/hrtimer.c   |  125 ++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 122 insertions(+), 11 deletions(-)

--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -161,6 +161,8 @@ enum  hrtimer_base_type {
  * @clock_was_set_seq:	Sequence counter of clock was set events
  * @migration_enabled:	The migration of hrtimers to other cpus is enabled
  * @nohz_active:	The nohz functionality is enabled
+ * @softirq_activated:	displays, if the softirq is raised - update of softirq
+ *			related settings is not required then.
  * @in_hrtirq:		hrtimer_interrupt() is currently executing
  * @hres_active:	State of high resolution mode
  * @hang_detected:	The last hrtimer interrupt detected a hang
@@ -169,8 +171,10 @@ enum  hrtimer_base_type {
  * @nr_hangs:		Total number of hrtimer interrupt hangs
  * @max_hang_time:	Maximum time spent in hrtimer_interrupt
  * @expires_next:	absolute time of the next event, is required for remote
- *			hrtimer enqueue
+ *			hrtimer enqueue; it is the total first expiry time (hard
+ *			and soft hrtimer are taken into account)
  * @next_timer:		Pointer to the first expiring timer
+ * @softirq_expires_next: Time to check, if soft queues needs also to be expired
  * @clock_base:		array of clock bases for this cpu
  *
  * Note: next_timer is just an optimization for __remove_hrtimer().
@@ -184,6 +188,7 @@ struct hrtimer_cpu_base {
 	unsigned int			clock_was_set_seq;
 	bool				migration_enabled;
 	bool				nohz_active;
+	bool				softirq_activated;
 	unsigned int			hres_active	: 1,
 					in_hrtirq	: 1,
 					hang_detected	: 1;
@@ -195,6 +200,7 @@ struct hrtimer_cpu_base {
 #endif
 	ktime_t				expires_next;
 	struct hrtimer			*next_timer;
+	ktime_t				softirq_expires_next;
 	struct hrtimer_clock_base	clock_base[HRTIMER_MAX_CLOCK_BASES];
 } ____cacheline_aligned;
 
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -499,7 +499,6 @@ static inline void hrtimer_update_next_t
 	cpu_base->next_timer = timer;
 }
 
-#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
 					 unsigned int active,
 					 ktime_t expires_next)
@@ -540,12 +539,23 @@ static ktime_t __hrtimer_get_next_event(
 
 	hrtimer_update_next_timer(cpu_base, NULL);
 
+	if (!cpu_base->softirq_activated) {
+		active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
+		expires_next = __hrtimer_next_event_base(cpu_base, active,
+							 expires_next);
+		cpu_base->softirq_expires_next = expires_next;
+	}
+
 	active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
 	expires_next = __hrtimer_next_event_base(cpu_base, active, expires_next);
 
+	/*
+	 * cpu_base->expires_next is not updated here. It is set only
+	 * in hrtimer_reprogramming path!
+	 */
+
 	return expires_next;
 }
-#endif
 
 static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
 {
@@ -969,6 +979,49 @@ static inline ktime_t hrtimer_update_low
 	return tim;
 }
 
+static void hrtimer_reprogram_softirq(struct hrtimer *timer)
+{
+	struct hrtimer_clock_base *base = timer->base;
+	struct hrtimer_cpu_base *cpu_base = base->cpu_base;
+	ktime_t expires;
+
+	/*
+	 * The softirq timer is not rearmed, when the softirq was raised
+	 * and has not yet run to completion.
+	 */
+	if (cpu_base->softirq_activated)
+		return;
+
+	expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
+
+	if (!ktime_before(expires, cpu_base->softirq_expires_next))
+		return;
+
+	cpu_base->softirq_expires_next = expires;
+
+	if (!ktime_before(expires, cpu_base->expires_next))
+		return;
+	hrtimer_reprogram(timer);
+}
+
+static void hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base,
+					 bool reprogram)
+{
+	ktime_t expires;
+
+	expires = __hrtimer_get_next_event(cpu_base);
+
+	if (!reprogram || !ktime_before(expires, cpu_base->expires_next))
+		return;
+	/*
+	 * next_timer can be used here, because
+	 * hrtimer_get_next_event() updated the next
+	 * timer. expires_next is only set when reprogramming function
+	 * is called.
+	 */
+	hrtimer_reprogram(cpu_base->next_timer);
+}
+
 static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
 				    u64 delta_ns, const enum hrtimer_mode mode,
 				    struct hrtimer_clock_base *base)
@@ -1007,9 +1060,12 @@ void hrtimer_start_range_ns(struct hrtim
 
 	base = lock_hrtimer_base(timer, &flags);
 
-	if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
-		hrtimer_reprogram(timer);
-
+	if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base)) {
+		if (timer->base->index < HRTIMER_BASE_MONOTONIC_SOFT)
+			hrtimer_reprogram(timer);
+		else
+			hrtimer_reprogram_softirq(timer);
+	}
 	unlock_hrtimer_base(timer, &flags);
 }
 EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
@@ -1206,7 +1262,8 @@ EXPORT_SYMBOL_GPL(hrtimer_active);
 
 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
 			  struct hrtimer_clock_base *base,
-			  struct hrtimer *timer, ktime_t *now)
+			  struct hrtimer *timer, ktime_t *now,
+			  bool hardirq)
 {
 	enum hrtimer_restart (*fn)(struct hrtimer *);
 	int restart;
@@ -1241,11 +1298,19 @@ static void __run_hrtimer(struct hrtimer
 	 * protected against migration to a different CPU even if the lock
 	 * is dropped.
 	 */
-	raw_spin_unlock(&cpu_base->lock);
+	if (hardirq)
+		raw_spin_unlock(&cpu_base->lock);
+	else
+		raw_spin_unlock_irq(&cpu_base->lock);
+
 	trace_hrtimer_expire_entry(timer, now);
 	restart = fn(timer);
 	trace_hrtimer_expire_exit(timer);
-	raw_spin_lock(&cpu_base->lock);
+
+	if (hardirq)
+		raw_spin_lock(&cpu_base->lock);
+	else
+		raw_spin_lock_irq(&cpu_base->lock);
 
 	/*
 	 * Note: We clear the running state after enqueue_hrtimer and
@@ -1309,11 +1374,28 @@ static void __hrtimer_run_queues(struct
 			if (basenow < hrtimer_get_softexpires_tv64(timer))
 				break;
 
-			__run_hrtimer(cpu_base, base, timer, &basenow);
+			__run_hrtimer(cpu_base, base, timer, &basenow,
+				      active_mask == HRTIMER_ACTIVE_HARD);
 		}
 	}
 }
 
+static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h)
+{
+	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
+	ktime_t now;
+
+	raw_spin_lock_irq(&cpu_base->lock);
+
+	now = hrtimer_update_base(cpu_base);
+	__hrtimer_run_queues(cpu_base, now, HRTIMER_ACTIVE_SOFT);
+
+	cpu_base->softirq_activated = 0;
+	hrtimer_update_softirq_timer(cpu_base, true);
+
+	raw_spin_unlock_irq(&cpu_base->lock);
+}
+
 #ifdef CONFIG_HIGH_RES_TIMERS
 
 /*
@@ -1343,9 +1425,15 @@ void hrtimer_interrupt(struct clock_even
 	 */
 	cpu_base->expires_next = KTIME_MAX;
 
+	if (!ktime_before(now, cpu_base->softirq_expires_next)) {
+		cpu_base->softirq_expires_next = KTIME_MAX;
+		cpu_base->softirq_activated = 1;
+		raise_softirq_irqoff(HRTIMER_SOFTIRQ);
+	}
+
 	__hrtimer_run_queues(cpu_base, now, HRTIMER_ACTIVE_HARD);
 
-	/* Reevaluate the clock bases for the next expiry */
+	/* Reevaluate the hard interrupt clock bases for the next expiry */
 	expires_next = __hrtimer_get_next_event(cpu_base);
 	/*
 	 * Store the new expiry value so the migration code can verify
@@ -1448,6 +1536,13 @@ void hrtimer_run_queues(void)
 
 	raw_spin_lock(&cpu_base->lock);
 	now = hrtimer_update_base(cpu_base);
+
+	if (!ktime_before(now, cpu_base->softirq_expires_next)) {
+		cpu_base->softirq_expires_next = KTIME_MAX;
+		cpu_base->softirq_activated = 1;
+		raise_softirq_irqoff(HRTIMER_SOFTIRQ);
+	}
+
 	__hrtimer_run_queues(cpu_base, now, HRTIMER_ACTIVE_HARD);
 	raw_spin_unlock(&cpu_base->lock);
 }
@@ -1629,6 +1724,7 @@ int hrtimers_prepare_cpu(unsigned int cp
 	cpu_base->cpu = cpu;
 	cpu_base->hres_active = 0;
 	cpu_base->expires_next = KTIME_MAX;
+	cpu_base->softirq_expires_next = KTIME_MAX;
 	return 0;
 }
 
@@ -1672,6 +1768,7 @@ int hrtimers_dead_cpu(unsigned int scpu)
 	BUG_ON(cpu_online(scpu));
 	tick_cancel_sched_timer(scpu);
 
+	local_bh_disable();
 	local_irq_disable();
 	old_base = &per_cpu(hrtimer_bases, scpu);
 	new_base = this_cpu_ptr(&hrtimer_bases);
@@ -1687,12 +1784,19 @@ int hrtimers_dead_cpu(unsigned int scpu)
 				     &new_base->clock_base[i]);
 	}
 
+	/*
+	 * The migration might have changed the first expiring softirq
+	 * timer on this CPU. Update it.
+	 */
+	hrtimer_update_softirq_timer(new_base, false);
+
 	raw_spin_unlock(&old_base->lock);
 	raw_spin_unlock(&new_base->lock);
 
 	/* Check, if we got expired work to do */
 	__hrtimer_peek_ahead_timers();
 	local_irq_enable();
+	local_bh_enable();
 	return 0;
 }
 
@@ -1707,6 +1811,7 @@ void __init hrtimers_init(void)
 	BUILD_BUG_ON_NOT_POWER_OF_2(HRTIMER_BASE_SOFT_MASK);
 
 	hrtimers_prepare_cpu(smp_processor_id());
+	open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
 }
 
 /**