[patch 30/60] x86/mm/kpti: Add infrastructure for page table isolation

From: Thomas Gleixner
Date: Mon Dec 04 2017 - 12:03:15 EST


From: Thomas Gleixner <tglx@xxxxxxxxxxxxx>

Add the initial files for kernel page table isolation, with a minimal init
function and the boot time detection for this misfeature.

Signed-off-by: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
---
Documentation/admin-guide/kernel-parameters.txt | 2
arch/x86/boot/compressed/pagetable.c | 3
arch/x86/entry/calling.h | 7 ++
arch/x86/include/asm/kpti.h | 14 ++++
arch/x86/mm/Makefile | 7 +-
arch/x86/mm/init.c | 2
arch/x86/mm/kpti.c | 76 ++++++++++++++++++++++++
include/linux/kpti.h | 11 +++
init/main.c | 2
9 files changed, 121 insertions(+), 3 deletions(-)

--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2711,6 +2711,8 @@
steal time is computed, but won't influence scheduler
behaviour

+ nokpti [X86-64] Disable kernel page table isolation
+
nolapic [X86-32,APIC] Do not enable or use the local APIC.

nolapic_timer [X86-32,APIC] Do not use the local APIC timer.
--- a/arch/x86/boot/compressed/pagetable.c
+++ b/arch/x86/boot/compressed/pagetable.c
@@ -23,6 +23,9 @@
*/
#undef CONFIG_AMD_MEM_ENCRYPT

+/* No KERNEL_PAGE_TABLE_ISOLATION support needed either: */
+#undef CONFIG_KERNEL_PAGE_TABLE_ISOLATION
+
#include "misc.h"

/* These actually do the work of building the kernel identity maps. */
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -205,18 +205,23 @@ For 32-bit we have the following convent
.endm

.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req
+ ALTERNATIVE "jmp .Lend_\@", "", X86_BUG_CPU_SECURE_MODE_KPTI
mov %cr3, \scratch_reg
ADJUST_KERNEL_CR3 \scratch_reg
mov \scratch_reg, %cr3
+.Lend_\@:
.endm

.macro SWITCH_TO_USER_CR3 scratch_reg:req
+ ALTERNATIVE "jmp .Lend_\@", "", X86_BUG_CPU_SECURE_MODE_KPTI
mov %cr3, \scratch_reg
ADJUST_USER_CR3 \scratch_reg
mov \scratch_reg, %cr3
+.Lend_\@:
.endm

.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
+ ALTERNATIVE "jmp .Ldone_\@", "", X86_BUG_CPU_SECURE_MODE_KPTI
movq %cr3, \scratch_reg
movq \scratch_reg, \save_reg
/*
@@ -233,11 +238,13 @@ For 32-bit we have the following convent
.endm

.macro RESTORE_CR3 save_reg:req
+ ALTERNATIVE "jmp .Lend_\@", "", X86_BUG_CPU_SECURE_MODE_KPTI
/*
* The CR3 write could be avoided when not changing its value,
* but would require a CR3 read *and* a scratch register.
*/
movq \save_reg, %cr3
+.Lend_\@:
.endm

#else /* CONFIG_KERNEL_PAGE_TABLE_ISOLATION=n: */
--- /dev/null
+++ b/arch/x86/include/asm/kpti.h
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+#ifndef _ASM_X86_KPTI_H
+#define _ASM_X86_KPTI_H
+#ifndef __ASSEMBLY__
+
+#ifdef CONFIG_KERNEL_PAGE_TABLE_ISOLATION
+extern void kpti_init(void);
+extern void kpti_check_boottime_disable(void);
+#else
+static inline void kpti_check_boottime_disable(void) { }
+#endif
+
+#endif /* __ASSEMBLY__ */
+#endif /* _ASM_X86_KPTI_H */
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -43,9 +43,10 @@ obj-$(CONFIG_AMD_NUMA) += amdtopology.o
obj-$(CONFIG_ACPI_NUMA) += srat.o
obj-$(CONFIG_NUMA_EMU) += numa_emulation.o

-obj-$(CONFIG_X86_INTEL_MPX) += mpx.o
-obj-$(CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS) += pkeys.o
-obj-$(CONFIG_RANDOMIZE_MEMORY) += kaslr.o
+obj-$(CONFIG_X86_INTEL_MPX) += mpx.o
+obj-$(CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS) += pkeys.o
+obj-$(CONFIG_RANDOMIZE_MEMORY) += kaslr.o
+obj-$(CONFIG_KERNEL_PAGE_TABLE_ISOLATION) += kpti.o

obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt.o
obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_boot.o
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -20,6 +20,7 @@
#include <asm/kaslr.h>
#include <asm/hypervisor.h>
#include <asm/cpufeature.h>
+#include <asm/kpti.h>

/*
* We need to define the tracepoints somewhere, and tlb.c
@@ -630,6 +631,7 @@ void __init init_mem_mapping(void)
{
unsigned long end;

+ kpti_check_boottime_disable();
probe_page_size_mask();
setup_pcid();

--- /dev/null
+++ b/arch/x86/mm/kpti.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * This code is based in part on work published here:
+ *
+ * https://github.com/IAIK/KAISER
+ *
+ * The original work was written by and and signed off by for the Linux
+ * kernel by:
+ *
+ * Signed-off-by: Richard Fellner <richard.fellner@xxxxxxxxxxxxxxxxx>
+ * Signed-off-by: Moritz Lipp <moritz.lipp@xxxxxxxxxxxxxx>
+ * Signed-off-by: Daniel Gruss <daniel.gruss@xxxxxxxxxxxxxx>
+ * Signed-off-by: Michael Schwarz <michael.schwarz@xxxxxxxxxxxxxx>
+ *
+ * Major changes to the original code by: Dave Hansen <dave.hansen@xxxxxxxxx>
+ * Mostly rewritten by Thomas Gleixner <tglx@xxxxxxxxxxxxx> and
+ * Andy Lutomirsky <luto@xxxxxxxxxxxxxx>
+ */
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/bug.h>
+#include <linux/init.h>
+#include <linux/spinlock.h>
+#include <linux/mm.h>
+#include <linux/uaccess.h>
+
+#include <asm/cpufeature.h>
+#include <asm/hypervisor.h>
+#include <asm/cmdline.h>
+#include <asm/kpti.h>
+#include <asm/pgtable.h>
+#include <asm/pgalloc.h>
+#include <asm/tlbflush.h>
+#include <asm/desc.h>
+
+#undef pr_fmt
+#define pr_fmt(fmt) "Kernel/User page tables isolation: " fmt
+
+void __init kpti_check_boottime_disable(void)
+{
+ bool enable = true;
+
+ if (cmdline_find_option_bool(boot_command_line, "nokpti")) {
+ pr_info("disabled on command line.\n");
+ enable = false;
+ }
+ if (hypervisor_is_type(X86_HYPER_XEN_PV)) {
+ pr_info("disabled on XEN_PV.\n");
+ enable = false;
+ }
+ if (enable)
+ setup_force_cpu_bug(X86_BUG_CPU_SECURE_MODE_KPTI);
+}
+
+/*
+ * Initialize kernel page table isolation
+ */
+void __init kpti_init(void)
+{
+ if (!static_cpu_has_bug(X86_BUG_CPU_SECURE_MODE_KPTI))
+ return;
+
+ pr_info("enabled\n");
+}
--- /dev/null
+++ b/include/linux/kpti.h
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
+#ifndef _INCLUDE_KPTI_H
+#define _INCLUDE_KPTI_H
+
+#ifdef CONFIG_KERNEL_PAGE_TABLE_ISOLATION
+#include <asm/kpti.h>
+#else
+static inline void kpti_init(void) { }
+#endif
+
+#endif
--- a/init/main.c
+++ b/init/main.c
@@ -76,6 +76,7 @@
#include <linux/slab.h>
#include <linux/perf_event.h>
#include <linux/ptrace.h>
+#include <linux/kpti.h>
#include <linux/blkdev.h>
#include <linux/elevator.h>
#include <linux/sched_clock.h>
@@ -505,6 +506,7 @@ static void __init mm_init(void)
pgtable_init();
vmalloc_init();
ioremap_huge_init();
+ kpti_init();
}

asmlinkage __visible void __init start_kernel(void)