[PATCH v2 01/12] x86/retpoline: Define retpoline indirect thunk and macros

[Andi Kleen]

From: Dave Hansen <dave.hansen@xxxxxxxxxxxxxxx>

From: David Woodhouse <dwmw@xxxxxxxxxxxx>

retpoline is a special sequence on Intel CPUs to stop speculation for
indirect branches.

Provide assembler infrastructure to use retpoline by the compiler
and for assembler. We add the out of line trampoline used by the
compiler, and NOSPEC_JUMP / NOSPEC_CALL macros for assembler

[Originally from David and Tim, heavily hacked by AK]

v2: Add CONFIG_RETPOLINE option
Signed-off-by: David Woodhouse <dwmw@xxxxxxxxxxxx>
Signed-off-by: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx>
Signed-off-by: Andi Kleen <ak@xxxxxxxxxxxxxxx>
---
 arch/x86/Kconfig                |  8 +++++
 arch/x86/include/asm/jump-asm.h | 70 +++++++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/vmlinux.lds.S   |  1 +
 arch/x86/lib/Makefile           |  1 +
 arch/x86/lib/retpoline.S        | 35 +++++++++++++++++++++
 5 files changed, 115 insertions(+)
 create mode 100644 arch/x86/include/asm/jump-asm.h
 create mode 100644 arch/x86/lib/retpoline.S

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index d4fc98c50378..8b0facfa35be 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,14 @@ config GOLDFISH
        def_bool y
        depends on X86_GOLDFISH
 
+config RETPOLINE
+	bool "Avoid speculative indirect branches in kernel"
+	default y
+	help
+	  Compile kernel with the retpoline compiler options to guard against
+	  kernel to user data leaks by avoiding speculative indirect
+	  branches. Requires a new enough compiler. The kernel may run slower.
+
 config INTEL_RDT
 	bool "Intel Resource Director Technology support"
 	default n
diff --git a/arch/x86/include/asm/jump-asm.h b/arch/x86/include/asm/jump-asm.h
new file mode 100644
index 000000000000..936fa620f346
--- /dev/null
+++ b/arch/x86/include/asm/jump-asm.h
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef JUMP_ASM_H
+#define JUMP_ASM_H 1
+
+#ifdef __ASSEMBLY__
+
+#ifdef CONFIG_RETPOLINE
+
+/*
+ * Jump to an indirect pointer without speculation.
+ *
+ * The out of line __x86.indirect_thunk has special code sequences
+ * to stop speculation.
+ */
+
+.macro NOSPEC_JMP target
+	push	\target
+	jmp	__x86.indirect_thunk
+.endm
+
+
+/*
+ * Call an indirect pointer without speculation.
+ */
+
+.macro NOSPEC_CALL target
+	jmp     1221f
+1222:
+	push	\target
+	jmp	__x86.indirect_thunk
+1221:
+	call	1222b
+.endm
+
+#else /* CONFIG_RETPOLINE */
+
+.macro NOSPEC_JMP target
+	jmp *\target
+.endm
+
+.macro NOSPEC_CALL target
+	call *\target
+.endm
+
+#endif /* !CONFIG_RETPOLINE */
+
+#else /* __ASSEMBLY__ */
+
+#ifdef CONFIG_RETPOLINE
+
+#define NOSPEC_JMP(t) \
+	"push " t "; "				\
+	"jmp __x86.indirect_thunk; "
+
+#define NOSPEC_CALL(t) \
+	"	jmp 1221f; "			\
+	"1222:	push " t ";"			\
+	"	jmp __x86.indirect_thunk;"	\
+	"1221:	call 1222b;"
+
+#else /* CONFIG_RETPOLINE */
+
+#define NOSPEC_JMP(t) "jmp *" t "; "
+#define NOSPEC_CALL(t) "call *" t "; "
+
+#endif /* !CONFIG_RETPOLINE */
+
+#endif /* !__ASSEMBLY */
+
+#endif
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 1e413a9326aa..2e64241a6664 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -103,6 +103,7 @@ SECTIONS
 		/* bootstrapping code */
 		HEAD_TEXT
 		. = ALIGN(8);
+		*(.text.__x86.indirect_thunk)
 		TEXT_TEXT
 		SCHED_TEXT
 		CPUIDLE_TEXT
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 7b181b61170e..f23934bbaf4e 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-$(CONFIG_RETPOLINE) += retpoline.o
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
 
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
new file mode 100644
index 000000000000..cb40781adbfe
--- /dev/null
+++ b/arch/x86/lib/retpoline.S
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * Out of line jump trampoline for calls that disable speculation.
+ *
+ * This is a special sequence that prevents the CPU speculating
+ * for indirect calls.
+ *
+ * This can be called by gcc generated code, or with the asm macros
+ * in asm/jump-asm.h
+ */
+
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/export.h>
+
+	.section	.text.__x86.indirect_thunk,"ax"
+
+ENTRY(__x86.indirect_thunk)
+	CFI_STARTPROC
+	call	retpoline_call_target
+2:
+	lfence		/* stop speculation */
+	jmp	2b
+retpoline_call_target:
+#ifdef CONFIG_64BIT
+	lea	8(%rsp), %rsp
+#else
+	lea	4(%esp), %esp
+#endif
+	ret
+	CFI_ENDPROC
+ENDPROC(__x86.indirect_thunk)
+
+	EXPORT_SYMBOL(__x86.indirect_thunk)
-- 
2.14.3