[PATCH v2 03/12] x86/entry/32: Filter NT and speed up AC filtering in SYSENTER
From: Andy Lutomirski
Date: Wed Mar 09 2016 - 22:01:06 EST
This makes the 32-bit code work just like the 64-bit code. It should
speed up syscalls on 32-bit kernels on Skylake by something like 20
cycles (by analogy to the 64-bit compat case).
It also cleans up NT just like we do for the 64-bit case.
Signed-off-by: Andy Lutomirski <luto@xxxxxxxxxx>
arch/x86/entry/entry_32.S | 23 ++++++++++++++++++++++-
1 file changed, 22 insertions(+), 1 deletion(-)
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 684d2505b93a..86665f93ff5d 100644
@@ -294,7 +294,6 @@ sysenter_past_esp:
pushl $__USER_DS /* pt_regs->ss */
pushl %ebp /* pt_regs->sp (stashed in bp) */
pushfl /* pt_regs->flags (except IF = 0) */
- ASM_CLAC /* Clear AC after saving FLAGS */
orl $X86_EFLAGS_IF, (%esp) /* Fix IF */
pushl $__USER_CS /* pt_regs->cs */
pushl $0 /* pt_regs->ip = 0 (placeholder) */
@@ -302,6 +301,23 @@ sysenter_past_esp:
SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */
+ * SYSENTER doesn't filter flags, so we need to clear NT and AC
+ * ourselves. To save a few cycles, we can check whether
+ * either was set instead of doing an unconditional popfq.
+ * This needs to happen before enabling interrupts so that
+ * we don't get preempted with NT set.
+ * NB.: .Lsysenter_fix_flags is a label with the code under it moved
+ * out-of-line as an optimization: NT is unlikely to be set in the
+ * majority of the cases and instead of polluting the I$ unnecessarily,
+ * we're keeping that code behind a branch which will predict as
+ * not-taken and therefore its instructions won't be fetched.
+ testl $X86_EFLAGS_NT|X86_EFLAGS_AC, PT_EFLAGS(%esp)
+ jnz .Lsysenter_fix_flags
* User mode is traced as though IRQs are on, and SYSENTER
* turned them off.
@@ -339,6 +355,11 @@ sysenter_past_esp:
+ pushl $X86_EFLAGS_FIXED
+ jmp .Lsysenter_flags_fixed
# system call handler stub