[PATCH v3 5/5] x86/idle: Disable IBRS entering idle and enable it on wakeup

[Tim Chen]

Clear IBRS on idle entry and set it on idle exit into kernel on mwait.
When we are in mwait, we are not running but if we leave IBRS on,
it will affect the performance on the sibling hardware thread.  So
we disable IBRS and reenable it when we wake up.

Thanks to Peter Zijlstra and Andrea Arcangeli's suggestion of using
static key to improve IBRS toggling.

Signed-off-by: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx>
---
 arch/x86/include/asm/mwait.h    | 13 +++++++++++++
 arch/x86/kernel/cpu/spec_ctrl.c | 27 ++++++++++++++++++++++++++-
 arch/x86/kernel/process.c       |  9 +++++++--
 3 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
index 39a2fb2..870299a 100644
--- a/arch/x86/include/asm/mwait.h
+++ b/arch/x86/include/asm/mwait.h
@@ -6,6 +6,7 @@
 #include <linux/sched/idle.h>
 
 #include <asm/cpufeature.h>
+#include <asm/spec_ctrl.h>
 
 #define MWAIT_SUBSTATE_MASK		0xf
 #define MWAIT_CSTATE_MASK		0xf
@@ -106,9 +107,21 @@ static inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
 			mb();
 		}
 
+	       /*
+		* CPUs run faster with speculation protection
+		* disabled.  All CPU threads in a core must
+		* disable speculation protection for it to be
+		* disabled.  Disable it while we are idle so the
+		* other hyperthread can run fast.
+		*
+		* Interrupts have been disabled at this point.
+		*/
+
+		spec_ctrl_unprotected_begin();
 		__monitor((void *)&current_thread_info()->flags, 0, 0);
 		if (!need_resched())
 			__mwait(eax, ecx);
+		spec_ctrl_unprotected_end();
 	}
 	current_clr_polling();
 }
diff --git a/arch/x86/kernel/cpu/spec_ctrl.c b/arch/x86/kernel/cpu/spec_ctrl.c
index 2f3d6ca..843b4e6 100644
--- a/arch/x86/kernel/cpu/spec_ctrl.c
+++ b/arch/x86/kernel/cpu/spec_ctrl.c
@@ -4,12 +4,16 @@
 #include <asm/cpufeature.h>
 
 static bool ibrs_admin_enabled;
+DEFINE_STATIC_KEY_FALSE(spec_ctrl_dynamic_ibrs);
 
 void spec_ctrl_scan_feature(struct cpuinfo_x86 *c)
 {
 	if (boot_cpu_has(X86_FEATURE_SPEC_CTRL)) {
-		if (ibrs_admin_enabled)
+		if (ibrs_admin_enabled) {
 			set_cpu_cap(c, X86_FEATURE_SPEC_CTRL_IBRS);
+			if (!c->cpu_index)
+				static_branch_enable(&spec_ctrl_dynamic_ibrs);
+		}
 	}
 }
 
@@ -21,3 +25,24 @@ static int __init check_ibrs_param(char *str)
 	return 0;
 }
 early_param("spectre_v2", check_ibrs_param);
+
+/*
+ * Interrupts must be disabled to begin unprotected speculation.
+ * Otherwise interrupts could come in and start running in unprotected mode.
+ */
+
+void spec_ctrl_unprotected_begin(void)
+{
+	/* should use lockdep_assert_irqs_disabled() when available */
+	WARN_ON_ONCE(!irqs_disabled());
+	if (static_branch_unlikely(&spec_ctrl_dynamic_ibrs))
+		__enable_indirect_speculation();
+}
+EXPORT_SYMBOL_GPL(spec_ctrl_unprotected_begin);
+
+void spec_ctrl_unprotected_end(void)
+{
+	if (static_branch_unlikely(&spec_ctrl_dynamic_ibrs))
+		__disable_indirect_speculation();
+}
+EXPORT_SYMBOL_GPL(spec_ctrl_unprotected_end);
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 3cb2486..87a1121 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -39,6 +39,7 @@
 #include <asm/switch_to.h>
 #include <asm/desc.h>
 #include <asm/prctl.h>
+#include <asm/spec_ctrl.h>
 
 /*
  * per-CPU TSS segments. Threads are completely 'soft' on Linux,
@@ -461,11 +462,15 @@ static __cpuidle void mwait_idle(void)
 			mb(); /* quirk */
 		}
 
+		spec_ctrl_unprotected_begin();
 		__monitor((void *)&current_thread_info()->flags, 0, 0);
-		if (!need_resched())
+		if (!need_resched()) {
 			__sti_mwait(0, 0);
-		else
+			spec_ctrl_unprotected_end();
+		} else {
+			spec_ctrl_unprotected_end();
 			local_irq_enable();
+		}
 		trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
 	} else {
 		local_irq_enable();
-- 
2.9.4