[PATCH 5/5] cputime: Generic on-demand virtual cputime accounting

[Frederic Weisbecker]

If we want to stop the tick further idle, we need to be
able to account the cputime without using the tick.

Virtual based cputime accounting solves that problem by
hooking into kernel/user boundaries.

However implementing CONFIG_VIRT_CPU_ACCOUNTING requires
to set low level hooks and involves more overhead. But
we already have a generic user hooks subsystem that is
required for RCU needs by archs which will want to
shut down the tick outside idle.

This patch implements a generic virtual based cputime
accounting that relies on these generic user hooks.

There are some upsides of doing this:

- This requires no arch code to implement CONFIG_VIRT_CPU_ACCOUNTING
if user hooks are already built (already necessary for RCU in full
tickless mode).

- We can rely on the generic user hooks subsystem to dynamically
(de)activate the hooks, so that we can switch anytime between virtual
and tick based accounting. This way we don't have the overhead
of the virtual accounting when the tick is running periodically.

And a few downsides:

- It relies on jiffies and the hooks are set in high level code. This
results in less precise cputime accounting than with a true native
virtual based cputime accounting which hooks on low level code and use
a cpu hardware clock. Precision is not the goal of this though.

- There is probably more overhead than a native virtual based cputime
accounting. But this relies on hooks that are already set anyway.

Signed-off-by: Frederic Weisbecker <fweisbec@xxxxxxxxx>
Cc: Alessio Igor Bogani <abogani@xxxxxxxxxx>
Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
Cc: Avi Kivity <avi@xxxxxxxxxx>
Cc: Chris Metcalf <cmetcalf@xxxxxxxxxx>
Cc: Christoph Lameter <cl@xxxxxxxxx>
Cc: Geoff Levand <geoff@xxxxxxxxxxxxx>
Cc: Gilad Ben Yossef <gilad@xxxxxxxxxxxxx>
Cc: Hakan Akkan <hakanakkan@xxxxxxxxx>
Cc: H. Peter Anvin <hpa@xxxxxxxxx>
Cc: Ingo Molnar <mingo@xxxxxxxxxx>
Cc: Kevin Hilman <khilman@xxxxxx>
Cc: Max Krasnyansky <maxk@xxxxxxxxxxxx>
Cc: Paul E. McKenney <paulmck@xxxxxxxxxxxxxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: Stephen Hemminger <shemminger@xxxxxxxxxx>
Cc: Steven Rostedt <rostedt@xxxxxxxxxxx>
Cc: Sven-Thorsten Dietrich <thebigcorporation@xxxxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
---
 include/asm-generic/cputime.h |    2 +
 include/linux/kernel_stat.h   |   12 ++++-
 include/linux/user_hooks.h    |   18 +++++++
 init/Kconfig                  |   11 ++++-
 kernel/sched/cputime.c        |  112 +++++++++++++++++++++++++++++++++++++++++
 kernel/user_hooks.c           |   10 ++--
 6 files changed, 158 insertions(+), 7 deletions(-)

diff --git a/include/asm-generic/cputime.h b/include/asm-generic/cputime.h
index 212c8bb..2a78aa7 100644
--- a/include/asm-generic/cputime.h
+++ b/include/asm-generic/cputime.h
@@ -66,9 +66,11 @@ typedef u64 __nocast cputime64_t;
 #define cputime64_to_clock_t(__ct)	\
 	jiffies_64_to_clock_t(cputime64_to_jiffies64(__ct))
 
+#ifndef CONFIG_VIRT_CPU_ACCOUNTING_GEN
 static inline bool accounting_vtime(void)
 {
 	return false;
 }
+#endif
 
 #endif
diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
index 1270b86..6e509a9 100644
--- a/include/linux/kernel_stat.h
+++ b/include/linux/kernel_stat.h
@@ -130,13 +130,23 @@ extern void account_process_tick(struct task_struct *, int user);
 extern void account_steal_ticks(unsigned long ticks);
 extern void account_idle_ticks(unsigned long ticks);
 
+
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
 extern void account_switch_vtime(struct task_struct *prev);
-extern bool account_process_tick_vtime(struct task_struct *p, int user_tick);
+extern void account_process_tick_vtime(struct task_struct *p, int user_tick);
 #else
 static inline void account_switch_vtime(struct task_struct *prev) { }
 static inline void account_process_tick_vtime(struct task_struct *p,
 					      int user_tick) { }
 #endif
 
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
+void account_system_vtime(struct task_struct *tsk);
+void account_user_vtime(struct task_struct *tsk);
+bool accounting_vtime(void);
+#else
+static inline void account_system_vtime(struct task_struct *tsk) { }
+static inline void account_user_vtime(struct task_struct *tsk) { }
+#endif
+
 #endif /* _LINUX_KERNEL_STAT_H */
diff --git a/include/linux/user_hooks.h b/include/linux/user_hooks.h
index 720292d..07385f1 100644
--- a/include/linux/user_hooks.h
+++ b/include/linux/user_hooks.h
@@ -3,6 +3,24 @@
 
 #ifdef CONFIG_USER_HOOKS
 #include <linux/sched.h>
+#include <linux/percpu.h>
+
+struct user_hooks {
+	bool hooking;
+	bool in_user;
+};
+
+DECLARE_PER_CPU(struct user_hooks, user_hooks);
+
+static inline bool in_user(void)
+{
+	return __get_cpu_var(user_hooks).in_user;
+}
+
+static inline bool user_hooks_hooking(void)
+{
+	return __get_cpu_var(user_hooks).hooking;
+}
 
 extern void user_enter(void);
 extern void user_exit(void);
diff --git a/init/Kconfig b/init/Kconfig
index 21da9b7..b0ed659 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -269,7 +269,9 @@ config POSIX_MQUEUE_SYSCTL
 
 config VIRT_CPU_ACCOUNTING
 	bool "Deterministic task and CPU time accounting"
-	depends on HAVE_VIRT_CPU_ACCOUNTING && !IRQ_TIME_ACCOUNTING
+	depends on HAVE_VIRT_CPU_ACCOUNTING || HAVE_USER_HOOKS
+	depends on !IRQ_TIME_ACCOUNTING
+	select VIRT_CPU_ACCOUNTING_GEN if !HAVE_VIRT_CPU_ACCOUNTING
 	default y if PPC64
 	help
 	  Select this option to enable more accurate task and CPU time
@@ -280,6 +282,13 @@ config VIRT_CPU_ACCOUNTING
 	  stolen time on logically-partitioned systems running on
 	  IBM POWER5-based machines.
 
+config VIRT_CPU_ACCOUNTING_GEN
+	select USER_HOOKS
+	bool
+	help
+	  Implement a generic virtual based cputime accounting by using
+	  the user hooks subsystem.
+
 config BSD_PROCESS_ACCT
 	bool "BSD Process Accounting"
 	help
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index 53b03cc..284d136 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -2,6 +2,7 @@
 #include <linux/sched.h>
 #include <linux/tsacct_kern.h>
 #include <linux/kernel_stat.h>
+#include <linux/user_hooks.h>
 #include "sched.h"
 
 
@@ -490,3 +491,114 @@ void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
 	*ut = sig->prev_utime;
 	*st = sig->prev_stime;
 }
+
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
+static DEFINE_PER_CPU(long, last_jiffies) = INITIAL_JIFFIES;
+
+static cputime_t get_vtime_delta(void)
+{
+	long delta;
+
+	delta = jiffies - __this_cpu_read(last_jiffies);
+	__this_cpu_add(last_jiffies, delta);
+
+	return jiffies_to_cputime(delta);
+}
+
+void account_system_vtime(struct task_struct *tsk)
+{
+	cputime_t delta_cpu = get_vtime_delta();
+
+	account_system_time(tsk, irq_count(), delta_cpu, cputime_to_scaled(delta_cpu));
+}
+
+void account_user_vtime(struct task_struct *tsk)
+{
+	cputime_t delta_cpu = get_vtime_delta();
+
+	account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu));
+}
+
+static void account_idle_vtime(void)
+{
+	cputime_t delta_cpu = get_vtime_delta();
+
+	account_idle_time(delta_cpu);
+}
+
+void account_vtime(struct task_struct *tsk)
+{
+	unsigned long count = irq_count();
+
+	if (!count) {
+		/*
+		 * If we interrupted user, in_user() is 1
+		 * because the user hooks subsys don't hook
+		 * on irq entry/exit. This way we know if
+		 * we need to flush user time on kernel entry.
+		 */
+		if (in_user())
+			account_user_vtime(tsk);
+	} else {
+		if (count == HARDIRQ_OFFSET ||
+		    count == SOFTIRQ_OFFSET) {
+			if (is_idle_task(tsk))
+				account_idle_vtime();
+			else
+				account_system_vtime(tsk);
+		}
+	}
+}
+
+void account_switch_vtime(struct task_struct *prev)
+{
+	if (is_idle_task(prev))
+		account_idle_vtime();
+	else
+		account_system_vtime(prev);
+}
+
+/*
+ * This is a kind of hack: if we flush user time only on
+ * irq entry, we miss the jiffies update and the time is spuriously
+ * accounted to system time.
+ */
+void account_process_tick_vtime(struct task_struct *p, int user_tick)
+{
+	if (in_user())
+		account_user_vtime(p);
+}
+
+bool accounting_vtime(void)
+{
+	return user_hooks_hooking();
+}
+
+static int __cpuinit vtime_cpu_notify(struct notifier_block *self,
+				      unsigned long action, void *hcpu)
+{
+	long cpu = (long)hcpu;
+	long *last_jiffies_cpu = per_cpu_ptr(&last_jiffies, cpu);
+
+	switch (action) {
+	case CPU_UP_PREPARE:
+	case CPU_UP_PREPARE_FROZEN:
+		/*
+		 * CHECKME: ensure that's visible by the CPU
+		 * once it wakes up
+		 */
+		*last_jiffies_cpu = jiffies;
+	default:
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
+static int __init init_vtime(void)
+{
+	cpu_notifier(vtime_cpu_notify, 0);
+	return 0;
+}
+early_initcall(init_vtime);
+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
diff --git a/kernel/user_hooks.c b/kernel/user_hooks.c
index 63174b0..31f5d7e 100644
--- a/kernel/user_hooks.c
+++ b/kernel/user_hooks.c
@@ -1,12 +1,8 @@
 #include <linux/user_hooks.h>
 #include <linux/rcupdate.h>
 #include <linux/sched.h>
-#include <linux/percpu.h>
+#include <linux/kernel_stat.h>
 
-struct user_hooks {
-	bool hooking;
-	bool in_user;
-};
 
 DEFINE_PER_CPU(struct user_hooks, user_hooks) = {
 #ifdef CONFIG_USER_HOOKS_FORCE
@@ -24,6 +20,8 @@ void user_enter(void)
 	uh = &__get_cpu_var(user_hooks);
 	if (uh->hooking && !uh->in_user) {
 		uh->in_user = true;
+		if (accounting_vtime())
+			account_system_vtime(current);
 		rcu_user_enter();
 	}
 	local_irq_restore(flags);
@@ -39,6 +37,8 @@ void user_exit(void)
 	if (uh->in_user) {
 		uh->in_user = false;
 		rcu_user_exit();
+		if (accounting_vtime())
+			account_user_vtime(current);
 	}
 	local_irq_restore(flags);
 }
-- 
1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/