Re: [PATCH 1/4] x86: save r11 into pt_regs->flags on SYSCALL64 fastpath

From: Andy Lutomirski
Date: Mon Mar 09 2015 - 16:02:38 EST

On Mon, Mar 9, 2015 at 11:39 AM, Denys Vlasenko <dvlasenk@xxxxxxxxxx> wrote:
> Before this patch, r11 was saved in pt_regs->r11.
> Which looks natural, but requires messy shuffling to/from iret frame
> whenever ptrace or e.g. sys_iopl wants to modify flags - because
> that's how this register is used by SYSCALL/SYSRET.
>
> This patch saves r11 in pt_regs->flags,
> and uses that value for SYSRET64 insn. Shuffling is eliminated.
>
> FIXUP/RESTORE_TOP_OF_STACK are simplified.
>
> stub_iopl is no longer needed: pt_regs->flags needs no fixing up.
>
> Testing shows that syscall fast path is ~54.3 ns before
> and after the patch (on 2.7 GHz Sandy Bridge CPU).

Acked-by: Andy Lutomirski <luto@xxxxxxxxxxxxxx>

>
> Signed-off-by: Denys Vlasenko <dvlasenk@xxxxxxxxxx>
> CC: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
> CC: Steven Rostedt <rostedt@xxxxxxxxxxx>
> CC: Ingo Molnar <mingo@xxxxxxxxxx>
> CC: Borislav Petkov <bp@xxxxxxxxx>
> CC: "H. Peter Anvin" <hpa@xxxxxxxxx>
> CC: Andy Lutomirski <luto@xxxxxxxxxxxxxx>
> CC: Oleg Nesterov <oleg@xxxxxxxxxx>
> CC: Frederic Weisbecker <fweisbec@xxxxxxxxx>
> CC: Alexei Starovoitov <ast@xxxxxxxxxxxx>
> CC: Will Drewry <wad@xxxxxxxxxxxx>
> CC: Kees Cook <keescook@xxxxxxxxxxxx>
> CC: x86@xxxxxxxxxx
> CC: linux-kernel@xxxxxxxxxxxxxxx
> ---
> arch/x86/include/asm/calling.h | 20 ++++++++++++++------
> arch/x86/kernel/entry_64.S | 24 +++++++++++-------------
> 2 files changed, 25 insertions(+), 19 deletions(-)
>
> diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h
> index f1a962f..4b5f7bf 100644
> --- a/arch/x86/include/asm/calling.h
> +++ b/arch/x86/include/asm/calling.h
> @@ -95,9 +95,11 @@ For 32-bit we have the following conventions - kernel is built with
> CFI_ADJUST_CFA_OFFSET 15*8+\addskip
> .endm
>
> - .macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8plus=1
> - .if \r8plus
> + .macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8910=1 r11=1
> + .if \r11
> movq_cfi r11, 6*8+\offset
> + .endif
> + .if \r8910
> movq_cfi r10, 7*8+\offset
> movq_cfi r9, 8*8+\offset
> movq_cfi r8, 9*8+\offset
> @@ -113,16 +115,19 @@ For 32-bit we have the following conventions - kernel is built with
> movq_cfi rdi, 14*8+\offset
> .endm
> .macro SAVE_C_REGS offset=0
> - SAVE_C_REGS_HELPER \offset, 1, 1, 1
> + SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1
> .endm
> .macro SAVE_C_REGS_EXCEPT_RAX_RCX offset=0
> - SAVE_C_REGS_HELPER \offset, 0, 0, 1
> + SAVE_C_REGS_HELPER \offset, 0, 0, 1, 1
> .endm
> .macro SAVE_C_REGS_EXCEPT_R891011
> - SAVE_C_REGS_HELPER 0, 1, 1, 0
> + SAVE_C_REGS_HELPER 0, 1, 1, 0, 0
> .endm
> .macro SAVE_C_REGS_EXCEPT_RCX_R891011
> - SAVE_C_REGS_HELPER 0, 1, 0, 0
> + SAVE_C_REGS_HELPER 0, 1, 0, 0, 0
> + .endm
> + .macro SAVE_C_REGS_EXCEPT_RAX_RCX_R11
> + SAVE_C_REGS_HELPER 0, 0, 0, 1, 0
> .endm
>
> .macro SAVE_EXTRA_REGS offset=0
> @@ -179,6 +184,9 @@ For 32-bit we have the following conventions - kernel is built with
> .macro RESTORE_C_REGS_EXCEPT_R11
> RESTORE_C_REGS_HELPER 1,1,0,1,1
> .endm
> + .macro RESTORE_C_REGS_EXCEPT_RCX_R11
> + RESTORE_C_REGS_HELPER 1,0,0,1,1
> + .endm
> .macro RESTORE_RSI_RDI
> RESTORE_C_REGS_HELPER 0,0,0,0,0
> .endm
> diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
> index 5117a2b..324200a 100644
> --- a/arch/x86/kernel/entry_64.S
> +++ b/arch/x86/kernel/entry_64.S
> @@ -121,14 +121,12 @@ ENDPROC(native_usergs_sysret64)
> #endif
>
> /*
> - * C code is not supposed to know about undefined top of stack. Every time
> - * a C function with an pt_regs argument is called from the SYSCALL based
> - * fast path FIXUP_TOP_OF_STACK is needed.
> + * C code is not supposed to know that the iret frame is not populated.
> + * Every time a C function with an pt_regs argument is called from
> + * the SYSCALL based fast path FIXUP_TOP_OF_STACK is needed.
> * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
> * manipulation.
> */
> -
> - /* %rsp:at FRAMEEND */
> .macro FIXUP_TOP_OF_STACK tmp offset=0
> movq PER_CPU_VAR(old_rsp),\tmp
> movq \tmp,RSP+\offset(%rsp)
> @@ -136,15 +134,13 @@ ENDPROC(native_usergs_sysret64)
> movq $__USER_CS,CS+\offset(%rsp)
> movq RIP+\offset(%rsp),\tmp /* get rip */
> movq \tmp,RCX+\offset(%rsp) /* copy it to rcx as sysret would do */
> - movq R11+\offset(%rsp),\tmp /* get eflags */
> - movq \tmp,EFLAGS+\offset(%rsp)
> + movq EFLAGS+\offset(%rsp),\tmp /* ditto for rflags->r11 */
> + movq \tmp,R11+\offset(%rsp)
> .endm
>
> .macro RESTORE_TOP_OF_STACK tmp offset=0
> movq RSP+\offset(%rsp),\tmp
> movq \tmp,PER_CPU_VAR(old_rsp)
> - movq EFLAGS+\offset(%rsp),\tmp
> - movq \tmp,R11+\offset(%rsp)
> .endm
>
> /*
> @@ -257,9 +253,10 @@ GLOBAL(system_call_after_swapgs)
> */
> ENABLE_INTERRUPTS(CLBR_NONE)
> ALLOC_PT_GPREGS_ON_STACK 8 /* +8: space for orig_ax */
> - SAVE_C_REGS_EXCEPT_RAX_RCX
> + SAVE_C_REGS_EXCEPT_RAX_RCX_R11
> movq $-ENOSYS,RAX(%rsp)
> movq_cfi rax,ORIG_RAX
> + movq %r11,EFLAGS(%rsp)
> movq %rcx,RIP(%rsp)
> CFI_REL_OFFSET rip,RIP
> testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
> @@ -277,7 +274,7 @@ system_call_fastpath:
> movq %rax,RAX(%rsp)
> /*
> * Syscall return path ending with SYSRET (fast path)
> - * Has incomplete stack frame and undefined top of stack.
> + * Has incompletely filled pt_regs, iret frame is also incomplete.
> */
> ret_from_sys_call:
> testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
> @@ -291,9 +288,10 @@ ret_from_sys_call:
> * sysretq will re-enable interrupts:
> */
> TRACE_IRQS_ON
> - RESTORE_C_REGS_EXCEPT_RCX
> - movq RIP(%rsp),%rcx
> + RESTORE_C_REGS_EXCEPT_RCX_R11
> + movq RIP(%rsp),%rcx
> CFI_REGISTER rip,rcx
> + movq EFLAGS(%rsp),%r11
> /*CFI_REGISTER rflags,r11*/
> movq PER_CPU_VAR(old_rsp), %rsp
> /*
> --
> 1.8.1.4
>



--
Andy Lutomirski
AMA Capital Management, LLC
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/