[PATCH -tip] x86/stackprotector: Move stack canary to struct pcpu_hot

[Uros Bizjak]

Move stack canary from __stack_chk_guard to struct pcpu_hot and
alias __stack_chk_guard to point to the new location in the
linker script.

__stack_chk_guard is one of the hottest data structures on x86, so
moving it there makes sense even if its benefit cannot be measured
explicitly.

Signed-off-by: Uros Bizjak <ubizjak@xxxxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Cc: Ingo Molnar <mingo@xxxxxxxxxx>
Cc: Borislav Petkov <bp@xxxxxxxxx>
Cc: Dave Hansen <dave.hansen@xxxxxxxxxxxxxxx>
Cc: "H. Peter Anvin" <hpa@xxxxxxxxx>
Cc: Brian Gerst <brgerst@xxxxxxxxx>
Cc: Ard Biesheuvel <ardb@xxxxxxxxxx>
---
 arch/x86/include/asm/current.h | 13 +++++++++++++
 arch/x86/kernel/cpu/common.c   |  1 -
 arch/x86/kernel/vmlinux.lds.S  |  2 ++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index bf5953883ec3..e4ff1d15b465 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -15,6 +15,9 @@ struct task_struct;
 struct pcpu_hot {
 	union {
 		struct {
+#ifdef CONFIG_STACKPROTECTOR
+			unsigned long		stack_canary;
+#endif
 			struct task_struct	*current_task;
 			int			preempt_count;
 			int			cpu_number;
@@ -35,6 +38,16 @@ struct pcpu_hot {
 };
 static_assert(sizeof(struct pcpu_hot) == 64);
 
+/*
+ * stack_canary should be at the beginning of struct pcpu_hot to avoid:
+ *
+ * Invalid absolute R_X86_64_32S relocation: __stack_chk_guard
+ *
+ * error when aliasing __stack_chk_guard to struct pcpu_hot
+ * - see arch/x86/kernel/vmlinux.lds.S.
+ */
+static_assert(offsetof(struct pcpu_hot, stack_canary) == 0);
+
 DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);
 
 /* const-qualified alias to pcpu_hot, aliased by linker. */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 21078907af57..9e54c1b585d2 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -2088,7 +2088,6 @@ void syscall_init(void)
 #endif /* CONFIG_X86_64 */
 
 #ifdef CONFIG_STACKPROTECTOR
-DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
 #ifndef CONFIG_SMP
 EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
 #endif
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 1769a7126224..cabb86d505fc 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -467,6 +467,8 @@ SECTIONS
 . = ASSERT((_end - LOAD_OFFSET <= KERNEL_IMAGE_SIZE),
 	   "kernel image bigger than KERNEL_IMAGE_SIZE");
 
+PROVIDE(__stack_chk_guard = pcpu_hot);
+
 /* needed for Clang - see arch/x86/entry/entry.S */
 PROVIDE(__ref_stack_chk_guard = __stack_chk_guard);
 
-- 
2.42.0