[PATCH] x86/boot: Use 5-level paging trampoline in place if possible

[Ard Biesheuvel]

The traditional x86 decompressor contains a rather nasty hack in
find_trampoline_placement() to temporarily use some system RAM below 1M
for a 32-bit addressable trampoline, which it needs in order to be able
to switch from 4 levels of paging to 5 or vice versa.

It does this even when running from 32-bit addressable memory itself, in
which case none of this is needed - the trampoline code can be fixed up
to execute correctly from anywhere below 4G, and so it can execute in
place from the decompressor's initial placement. The additional root
level page table allocation can be used directly if it resides below the
4G mark as well.

The traditional decompressor is no longer used by the EFI stub, and is
mostly relied upon by non-EFI bootloaders such as coreboot which run
entirely in 32-bit mode, and are therefore guaranteed to place the
decompressor in 32-bit addressable memory. In those cases, on CPUs that
support it, 5-level paging will always get enabled, by invoking this
code (unless it is explicitly disabled on the command line) but the
trampoline allocation is never needed.

kexec also relies on the traditional decompressor and does typically
load the kernel above 4G, but in that case, the trampoline is only
needed when either the first or the second kernel explicitly disables
5-level paging, otherwise it will just remain enabled and no trampoline
is needed.

So avoid the trampoline hack unless it is really required.

While at it, bring the C prototype of trampoline_32bit_src() in line
with the asm implementation, and drop a couple of redundant
declarations.

Cc: "H. Peter Anvin" <hpa@xxxxxxxxx>
Cc: Kiryl Shutsemau <kas@xxxxxxxxxx>
Signed-off-by: Ard Biesheuvel <ardb@xxxxxxxxxx>
---
 arch/x86/boot/compressed/pgtable_64.c   | 40 +++++++++++---------
 arch/x86/include/asm/boot.h             |  2 +-
 drivers/firmware/efi/libstub/x86-stub.h |  3 --
 3 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
index 0e89e197e112..0f6e2ae5c1a1 100644
--- a/arch/x86/boot/compressed/pgtable_64.c
+++ b/arch/x86/boot/compressed/pgtable_64.c
@@ -102,7 +102,7 @@ static unsigned long find_trampoline_placement(void)
 
 asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable)
 {
-	void (*toggle_la57)(void *cr3);
+	void (*toggle_la57)(void *cr3) = trampoline_32bit_src;
 	bool l5_required = false;
 
 	/* Initialize boot_params. Required for cmdline_find_option_bool(). */
@@ -135,18 +135,22 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable)
 	if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))
 		return;
 
-	trampoline_32bit = (unsigned long *)find_trampoline_placement();
+	if ((unsigned long)pgtable > U32_MAX) {
+		trampoline_32bit = (unsigned long *)find_trampoline_placement();
 
-	/* Preserve trampoline memory */
-	memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE);
+		/* Preserve trampoline memory */
+		memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE);
 
-	/* Clear trampoline memory first */
-	memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);
+		/* Clear trampoline memory first */
+		memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);
 
-	/* Copy trampoline code in place */
-	toggle_la57 = memcpy(trampoline_32bit +
-			TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
-			&trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
+		/* Copy trampoline code in place */
+		toggle_la57 = memcpy(trampoline_32bit +
+				TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
+				&trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
+	} else {
+		trampoline_32bit = memset(pgtable, 0, PAGE_SIZE);
+	}
 
 	/*
 	 * Avoid the need for a stack in the 32-bit trampoline code, by using
@@ -189,12 +193,14 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable)
 
 	toggle_la57(trampoline_32bit);
 
-	/*
-	 * Move the top level page table out of trampoline memory.
-	 */
-	memcpy(pgtable, trampoline_32bit, PAGE_SIZE);
-	native_write_cr3((unsigned long)pgtable);
+	if (pgtable != trampoline_32bit) {
+		/*
+		 * Move the top level page table out of trampoline memory.
+		 */
+		memcpy(pgtable, trampoline_32bit, PAGE_SIZE);
+		native_write_cr3((unsigned long)pgtable);
 
-	/* Restore trampoline memory */
-	memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
+		/* Restore trampoline memory */
+		memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
+	}
 }
diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h
index f7b67cb73915..bc9739a2633a 100644
--- a/arch/x86/include/asm/boot.h
+++ b/arch/x86/include/asm/boot.h
@@ -93,7 +93,7 @@ extern struct boot_params *boot_params_ptr;
 extern unsigned long *trampoline_32bit;
 extern const u16 trampoline_ljmp_imm_offset;
 
-void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
+void trampoline_32bit_src(void *cr3);
 
 #endif
 
diff --git a/drivers/firmware/efi/libstub/x86-stub.h b/drivers/firmware/efi/libstub/x86-stub.h
index 1c20e99a6494..514560712610 100644
--- a/drivers/firmware/efi/libstub/x86-stub.h
+++ b/drivers/firmware/efi/libstub/x86-stub.h
@@ -2,9 +2,6 @@
 
 #include <linux/efi.h>
 
-extern void trampoline_32bit_src(void *, bool);
-extern const u16 trampoline_ljmp_imm_offset;
-
 efi_status_t efi_adjust_memory_range_protection(unsigned long start,
 						unsigned long size);
 
-- 
2.53.0.473.g4a7958ca14-goog