[PATCH 23/35] x86/speculation: Add basic speculation control code

[Peter Zijlstra]

From: Thomas Gleixner <tglx@xxxxxxxxxxxxx>

Add the minimal infrastructure to control the speculation control feature.

 - Integrate it into the spectre_v2 coammand line parser and the mitigation
   selector function. The conditional selector function is a placeholder
   right now, which needs to be expanded with CPU specific decision
   functions.

 - Provide a static key for the actual code control.

 - Provide a init function which is called after jump label patching is
   functional.

 - Provide an interface for the late micro code loader to allow late
   discovery of the IBRS support. Not yet functional.

[peterz: fixed Makefile]

Signed-off-by: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
---
 Documentation/admin-guide/kernel-parameters.txt |    1 
 arch/x86/include/asm/nospec-branch.h            |    5 +++
 arch/x86/kernel/cpu/Makefile                    |    1 
 arch/x86/kernel/cpu/bugs.c                      |   26 +++++++++++++++++-
 arch/x86/kernel/cpu/specctrl.c                  |   33 ++++++++++++++++++++++++
 5 files changed, 64 insertions(+), 2 deletions(-)

--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -3932,6 +3932,7 @@
 			retpoline	  - replace indirect branches
 			retpoline,generic - google's original retpoline
 			retpoline,amd     - AMD-specific minimal thunk
+			ibrs		  - Intel: Indirect Branch Restricted Speculation
 
 			Not specifying this option is equivalent to
 			spectre_v2=auto.
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -214,5 +214,10 @@ static inline void vmexit_fill_RSB(void)
 		      : "r" (loops) : "memory" );
 #endif
 }
+
+bool specctrl_force_enable_ibrs(void);
+bool specctrl_cond_enable_ibrs(bool full_retpoline);
+bool is_skylake_era(void);
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -24,6 +24,7 @@ obj-y			+= match.o
 obj-y			+= bugs.o
 obj-$(CONFIG_CPU_FREQ)	+= aperfmperf.o
 obj-y			+= cpuid-deps.o
+obj-y			+= specctrl.o
 
 obj-$(CONFIG_PROC_FS)	+= proc.o
 obj-$(CONFIG_X86_FEATURE_NAMES) += capflags.o powerflags.o
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -79,6 +79,7 @@ enum spectre_v2_mitigation_cmd {
 	SPECTRE_V2_CMD_RETPOLINE,
 	SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 	SPECTRE_V2_CMD_RETPOLINE_AMD,
+	SPECTRE_V2_CMD_IBRS,
 };
 
 static const char *spectre_v2_strings[] = {
@@ -87,6 +88,7 @@ static const char *spectre_v2_strings[]
 	[SPECTRE_V2_RETPOLINE_MINIMAL_AMD]	= "Vulnerable: Minimal AMD ASM retpoline",
 	[SPECTRE_V2_RETPOLINE_GENERIC]		= "Mitigation: Full generic retpoline",
 	[SPECTRE_V2_RETPOLINE_AMD]		= "Mitigation: Full AMD retpoline",
+	[SPECTRE_V2_IBRS]			= "Mitigation: Indirect Branch Restricted Speculation",
 };
 
 #undef pr_fmt
@@ -144,6 +146,8 @@ static enum spectre_v2_mitigation_cmd __
 		} else if (match_option(arg, ret, "retpoline,generic")) {
 			spec2_print_if_insecure("generic retpoline selected on command line.");
 			return SPECTRE_V2_CMD_RETPOLINE_GENERIC;
+		} else if (match_option(arg, ret, "ibrs")) {
+			return SPECTRE_V2_CMD_IBRS;
 		} else if (match_option(arg, ret, "auto")) {
 			return SPECTRE_V2_CMD_AUTO;
 		}
@@ -156,8 +160,8 @@ static enum spectre_v2_mitigation_cmd __
 	return SPECTRE_V2_CMD_NONE;
 }
 
-/* Check for Skylake-like CPUs (for RSB handling) */
-static bool __init is_skylake_era(void)
+/* Check for Skylake-like CPUs (for RSB and IBRS handling) */
+bool __init is_skylake_era(void)
 {
 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
 	    boot_cpu_data.x86 == 6) {
@@ -175,6 +179,7 @@ static bool __init is_skylake_era(void)
 
 static void __init spectre_v2_select_mitigation(void)
 {
+	bool full_retpoline = IS_ENABLED(CONFIG_RETPOLINE) && retp_compiler();
 	enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 	enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 
@@ -190,9 +195,25 @@ static void __init spectre_v2_select_mit
 	case SPECTRE_V2_CMD_NONE:
 		return;
 
+	case SPECTRE_V2_CMD_IBRS:
+		/* Command line requested IBRS. Try to enable it */
+		if (specctrl_force_enable_ibrs()) {
+			mode = SPECTRE_V2_IBRS;
+			goto set_mode;
+		}
+		/* FALLTRHU */
+
 	case SPECTRE_V2_CMD_FORCE:
 		/* FALLTRHU */
 	case SPECTRE_V2_CMD_AUTO:
+		/*
+		 * Check whether the CPU prefers to have IBRS or IBRS is
+		 * the only available mitigation.
+		 */
+		if (specctrl_cond_enable_ibrs(full_retpoline)) {
+			mode = SPECTRE_V2_IBRS;
+			goto set_mode;
+		}
 		goto retpoline_auto;
 
 	case SPECTRE_V2_CMD_RETPOLINE_AMD:
@@ -229,6 +250,7 @@ static void __init spectre_v2_select_mit
 		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 	}
 
+set_mode:
 	spectre_v2_enabled = mode;
 	pr_info("%s\n", spectre_v2_strings[mode]);
 
--- /dev/null
+++ b/arch/x86/kernel/cpu/specctrl.c
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
+#include <asm/nospec-branch.h>
+
+static inline void specctrl_enable_ibrs(void)
+{
+	setup_force_cpu_cap(X86_FEATURE_IBRS);
+}
+
+bool __init specctrl_force_enable_ibrs(void)
+{
+	if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+		return false;
+	specctrl_enable_ibrs();
+	return true;
+}
+
+bool __init specctrl_cond_enable_ibrs(bool full_retpoline)
+{
+	if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+		return false;
+	/*
+	 * IBRS is only required by SKL or as fallback if retpoline is not
+	 * fully supported.
+	 */
+	if (!is_skylake_era() && full_retpoline)
+		return false;
+
+	specctrl_enable_ibrs();
+	return true;
+}