[PATCH 4.1 23/65] x86/entry: Stop using PER_CPU_VAR(kernel_stack)

From: Pavel Tatashin
Date: Mon Mar 05 2018 - 19:38:01 EST


From: Denys Vlasenko <dvlasenk@xxxxxxxxxx>

PER_CPU_VAR(kernel_stack) is redundant:

- On the 64-bit build, we can use PER_CPU_VAR(cpu_tss + TSS_sp0).
- On the 32-bit build, we can use PER_CPU_VAR(cpu_current_top_of_stack).

PER_CPU_VAR(kernel_stack) will be deleted by a separate change.

Signed-off-by: Denys Vlasenko <dvlasenk@xxxxxxxxxx>
Cc: Alexei Starovoitov <ast@xxxxxxxxxxxx>
Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
Cc: Andy Lutomirski <luto@xxxxxxxxxxxxxx>
Cc: Borislav Petkov <bp@xxxxxxxxx>
Cc: Frederic Weisbecker <fweisbec@xxxxxxxxx>
Cc: H. Peter Anvin <hpa@xxxxxxxxx>
Cc: Kees Cook <keescook@xxxxxxxxxxxx>
Cc: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
Cc: Oleg Nesterov <oleg@xxxxxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: Steven Rostedt <rostedt@xxxxxxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Cc: Will Drewry <wad@xxxxxxxxxxxx>
Link: http://lkml.kernel.org/r/1429889495-27850-1-git-send-email-dvlasenk@xxxxxxxxxx
Signed-off-by: Ingo Molnar <mingo@xxxxxxxxxx>
(cherry picked from commit 63332a8455d8310b77d38779c6c21a660a8d9feb)
Signed-off-by: Pavel Tatashin <pasha.tatashin@xxxxxxxxxx>
---
arch/x86/ia32/ia32entry.S | 2 +-
arch/x86/include/asm/thread_info.h | 8 +++++++-
arch/x86/kernel/entry_64.S | 4 ++--
arch/x86/xen/xen-asm_64.S | 5 +++--
4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 27e54946ef35..2ae22c951fa0 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -356,7 +356,7 @@ ENTRY(ia32_cstar_target)
SWAPGS_UNSAFE_STACK
movl %esp,%r8d
CFI_REGISTER rsp,r8
- movq PER_CPU_VAR(kernel_stack),%rsp
+ movq PER_CPU_VAR(cpu_tss + TSS_sp0),%rsp
ENABLE_INTERRUPTS(CLBR_NONE)

/* Zero-extending 32-bit regs, do not remove */
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index b4bdec3e9523..d656a363e1eb 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -198,9 +198,15 @@ static inline unsigned long current_stack_pointer(void)
#else /* !__ASSEMBLY__ */

/* Load thread_info address into "reg" */
+#ifdef CONFIG_X86_32
#define GET_THREAD_INFO(reg) \
- _ASM_MOV PER_CPU_VAR(kernel_stack),reg ; \
+ _ASM_MOV PER_CPU_VAR(cpu_current_top_of_stack),reg ; \
_ASM_SUB $(THREAD_SIZE),reg ;
+#else
+#define GET_THREAD_INFO(reg) \
+ _ASM_MOV PER_CPU_VAR(cpu_tss + TSS_sp0),reg ; \
+ _ASM_SUB $(THREAD_SIZE),reg ;
+#endif

/*
* ASM operand which evaluates to a 'thread_info' address of
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 6c9cb6073832..99f67a38f7f0 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -216,7 +216,7 @@ ENTRY(system_call)
GLOBAL(system_call_after_swapgs)

movq %rsp,PER_CPU_VAR(rsp_scratch)
- movq PER_CPU_VAR(kernel_stack),%rsp
+ movq PER_CPU_VAR(cpu_tss + TSS_sp0),%rsp

/* Construct struct pt_regs on stack */
pushq_cfi $__USER_DS /* pt_regs->ss */
@@ -1464,7 +1464,7 @@ ENTRY(nmi)
SWAPGS_UNSAFE_STACK
cld
movq %rsp, %rdx
- movq PER_CPU_VAR(kernel_stack), %rsp
+ movq PER_CPU_VAR(cpu_tss + TSS_sp0), %rsp
pushq 5*8(%rdx) /* pt_regs->ss */
pushq 4*8(%rdx) /* pt_regs->rsp */
pushq 3*8(%rdx) /* pt_regs->flags */
diff --git a/arch/x86/xen/xen-asm_64.S b/arch/x86/xen/xen-asm_64.S
index 985fc3ee0973..acc49e088ec5 100644
--- a/arch/x86/xen/xen-asm_64.S
+++ b/arch/x86/xen/xen-asm_64.S
@@ -15,6 +15,7 @@
#include <asm/percpu.h>
#include <asm/processor-flags.h>
#include <asm/segment.h>
+#include <asm/asm-offsets.h>

#include <xen/interface/xen.h>

@@ -69,7 +70,7 @@ ENTRY(xen_sysret64)
* still with the kernel gs, so we can easily switch back
*/
movq %rsp, PER_CPU_VAR(rsp_scratch)
- movq PER_CPU_VAR(kernel_stack), %rsp
+ movq PER_CPU_VAR(cpu_tss + TSS_sp0), %rsp

pushq $__USER_DS
pushq PER_CPU_VAR(rsp_scratch)
@@ -88,7 +89,7 @@ ENTRY(xen_sysret32)
* still with the kernel gs, so we can easily switch back
*/
movq %rsp, PER_CPU_VAR(rsp_scratch)
- movq PER_CPU_VAR(kernel_stack), %rsp
+ movq PER_CPU_VAR(cpu_tss + TSS_sp0), %rsp

pushq $__USER32_DS
pushq PER_CPU_VAR(rsp_scratch)
--
2.16.2