[PATCH 1/6] x86_64/entry: Unconditionally use -fno-jump-table

[Peter Zijlstra]

Currently we force -fno-jump-table for RETPOLINE=y || X86_KERNEL_IBT=y, which
means that it is effectively disabled for most everyone these days.

Simply unconditionally disable jump tables for x86_64 to take away the edge
case.

Doing so, allows removing the array_index_nospec() barrier in the syscall
dispatch since it cannot be an array index anymore.

Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
---
 arch/x86/Kconfig            |    4 +---
 arch/x86/Makefile           |   27 ++++++++-------------------
 arch/x86/entry/syscall_32.c |    2 ++
 arch/x86/entry/syscall_64.c |    2 --
 4 files changed, 11 insertions(+), 24 deletions(-)

--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2366,10 +2366,8 @@ config CC_OPT_INDIRECT_BRANCH_CS_PREFIX
 config CC_OPT_CF_PROTECTION_NONE
 	def_string "$(cc-option-str,-fcf-protection=none)"
 
-# with jump tables off: the compilers use NOTRACK for them, which kernel IBT
-# does not allow
 config CC_OPT_CF_PROTECTION_BRANCH
-	def_string "$(cc-option-str,-fcf-protection=branch -fno-jump-tables)"
+	def_string "$(cc-option-str,-fcf-protection=branch)"
 
 config CC_OPT_FALIGN_JUMPS
 	def_string "$(cc-option-str,-falign-jumps=1)"
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -90,17 +90,8 @@ CC_FLAGS_FPU += -mhard-float
 endif
 
 ifeq ($(CONFIG_X86_KERNEL_IBT),y)
-#
-# Kernel IBT has S_CET.NOTRACK_EN=0, as such the compilers must not generate
-# NOTRACK prefixes. Current generation compilers unconditionally employ NOTRACK
-# for jump-tables, as such, disable jump-tables for now.
-#
-# (jump-tables are implicitly disabled by RETPOLINE)
-#
-#   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104816
-#
 KBUILD_CFLAGS += $(CONFIG_CC_OPT_CF_PROTECTION_BRANCH)
-KBUILD_RUSTFLAGS += -Zcf-protection=branch $(if $(call rustc-min-version,109300),-Cjump-tables=n,-Zno-jump-tables)
+KBUILD_RUSTFLAGS += -Zcf-protection=branch
 else
 KBUILD_CFLAGS += $(CONFIG_CC_OPT_CF_PROTECTION_NONE)
 endif
@@ -178,6 +169,13 @@ endif
         KBUILD_RUSTFLAGS += -Ccode-model=kernel
 
         percpu_seg := gs
+
+	# Due to retpolines and cf-protection=branch's implicit NOTRACK usage
+	# for jump-tables, blanked disable jump-tables for all x86_64 builds to
+	# get a consistent behaviour across configurations. This allows
+	# removing some array_index_nospec() usage.
+	KBUILD_CFLAGS += -fno-jump-tables
+	KBUILD_RUSTFLAGS += $(if $(call rustc-min-version,109300),-Cjump-tables=n,-Zno-jump-tables)
 endif
 
 ifeq ($(CONFIG_STACKPROTECTOR),y)
@@ -214,15 +212,6 @@ KBUILD_CFLAGS += -fno-asynchronous-unwin
 ifdef CONFIG_MITIGATION_RETPOLINE
   KBUILD_CFLAGS += $(RETPOLINE_CFLAGS)
   KBUILD_RUSTFLAGS += $(RETPOLINE_RUSTFLAGS)
-  # Additionally, avoid generating expensive indirect jumps which
-  # are subject to retpolines for small number of switch cases.
-  # LLVM turns off jump table generation by default when under
-  # retpoline builds, however, gcc does not for x86. This has
-  # only been fixed starting from gcc stable version 8.4.0 and
-  # onwards, but not for older ones. See gcc bug #86952.
-  ifndef CONFIG_CC_IS_CLANG
-    KBUILD_CFLAGS += -fno-jump-tables
-  endif
 endif
 
 ifdef CONFIG_MITIGATION_SLS
--- a/arch/x86/entry/syscall_32.c
+++ b/arch/x86/entry/syscall_32.c
@@ -75,7 +75,9 @@ early_param("ia32_emulation", ia32_emula
 static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, unsigned long nr)
 {
 	if (likely(nr < IA32_NR_syscalls)) {
+#ifdef CONFIG_X86_32
 		nr = array_index_nospec(nr, IA32_NR_syscalls);
+#endif
 		regs->ax = ia32_sys_call(regs, (unsigned int)nr);
 	}
 }
--- a/arch/x86/entry/syscall_64.c
+++ b/arch/x86/entry/syscall_64.c
@@ -57,7 +57,6 @@ static noinline long x32_sys_call(const
 static __always_inline bool do_syscall_x64(struct pt_regs *regs, unsigned long nr)
 {
 	if (likely(nr < NR_syscalls)) {
-		nr = array_index_nospec(nr, NR_syscalls);
 		regs->ax = x64_sys_call(regs, (unsigned int)nr);
 		return true;
 	}
@@ -70,7 +69,6 @@ static __always_inline void do_syscall_x
 	nr -= __X32_SYSCALL_BIT;
 
 	if (IS_ENABLED(CONFIG_X86_X32_ABI) && likely(nr < X32_NR_syscalls)) {
-		nr = array_index_nospec(nr, X32_NR_syscalls);
 		regs->ax = x32_sys_call(regs, (unsigned int)nr);
 	}
 }