[PATCH 6/6] x86_64/entry: Mark C noinstr __noendbr
From: Peter Zijlstra
Date: Mon Nov 02 2026 - 06:42:06 EST
Mark all the C noinstr entry functions __noendbr where possible. Nobody has any
business of doing indirect calls to this.
Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
---
arch/x86/entry/common.c | 2 +-
arch/x86/entry/entry_fred.c | 6 +++---
arch/x86/entry/syscall_32.c | 10 +++++-----
arch/x86/entry/syscall_64.c | 6 +++---
arch/x86/include/asm/entry-common.h | 2 +-
arch/x86/include/asm/fred.h | 6 +++---
arch/x86/include/asm/syscall.h | 10 +++++-----
7 files changed, 21 insertions(+), 21 deletions(-)
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -15,7 +15,7 @@
* purely a signal that an NMI/IRQ is pending, i.e. the event that triggered
* the VM-Exit is held pending until it's unblocked in the host.
*/
-noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
+noinstr __noendbr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
{
if (event_type == EVENT_TYPE_EXTINT) {
#ifdef CONFIG_X86_64
--- a/arch/x86/entry/entry_fred.c
+++ b/arch/x86/entry/entry_fred.c
@@ -234,7 +234,7 @@ static noinstr void fred_swexc(struct pt
}
}
-__visible noinstr void fred_entry_from_user(struct pt_regs *regs)
+__visible noinstr __noendbr void fred_entry_from_user(struct pt_regs *regs)
{
unsigned long error_code = regs->orig_ax;
@@ -266,7 +266,7 @@ __visible noinstr void fred_entry_from_u
return fred_bad_type(regs, error_code);
}
-__visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
+__visible noinstr __noendbr void fred_entry_from_kernel(struct pt_regs *regs)
{
unsigned long error_code = regs->orig_ax;
@@ -295,7 +295,7 @@ __visible noinstr void fred_entry_from_k
}
#if IS_ENABLED(CONFIG_KVM_INTEL)
-__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs)
+__visible noinstr __noendbr void __fred_entry_from_kvm(struct pt_regs *regs)
{
switch (regs->fred_ss.type) {
case EVENT_TYPE_EXTINT:
--- a/arch/x86/entry/syscall_32.c
+++ b/arch/x86/entry/syscall_32.c
@@ -124,7 +124,7 @@ static __always_inline bool int80_is_ext
* eax: system call number
* ebx, ecx, edx, esi, edi, ebp: arg1 - arg 6
*/
-__visible noinstr void do_int80_emulation(struct pt_regs *regs)
+__visible noinstr __noendbr void do_int80_emulation(struct pt_regs *regs)
{
long nr;
@@ -239,7 +239,7 @@ DEFINE_FREDENTRY_RAW(int80_emulation)
#else /* CONFIG_IA32_EMULATION */
/* Handles int $0x80 on a 32bit kernel */
-__visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
+__visible noinstr __noendbr void do_int80_syscall_32(struct pt_regs *regs)
{
long nr = syscall_32_enter(regs);
@@ -259,7 +259,7 @@ __visible noinstr void do_int80_syscall_
}
#endif /* !CONFIG_IA32_EMULATION */
-static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
+static noinstr __noendbr bool __do_fast_syscall_32(struct pt_regs *regs)
{
long nr = syscall_32_enter(regs);
int res;
@@ -291,7 +291,7 @@ static noinstr bool __do_fast_syscall_32
}
/* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
-__visible noinstr bool do_fast_syscall_32(struct pt_regs *regs)
+__visible noinstr __noendbr bool do_fast_syscall_32(struct pt_regs *regs)
{
/*
* Called using the internal vDSO SYSENTER/SYSCALL32 calling
@@ -338,7 +338,7 @@ __visible noinstr bool do_fast_syscall_3
}
/* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
-__visible noinstr bool do_SYSENTER_32(struct pt_regs *regs)
+__visible noinstr __noendbr bool do_SYSENTER_32(struct pt_regs *regs)
{
/* SYSENTER loses RSP, but the vDSO saved it in RBP. */
regs->sp = regs->bp;
--- a/arch/x86/entry/syscall_64.c
+++ b/arch/x86/entry/syscall_64.c
@@ -35,7 +35,7 @@ const sys_call_ptr_t sys_call_table[] =
#define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs);
/* The unsigned int @nr argument is intentional as it creates denser code */
-static noinstr long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
+static noinstr __noendbr long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
{
/*
* Because -fno-jump-tables, this compiles into a binary branch tree
@@ -51,7 +51,7 @@ static noinstr long x64_sys_call(const s
}
}
-static noinstr long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
+static noinstr __noendbr long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
{
#ifdef CONFIG_X86_X32_ABI
instrumentation_weak();
@@ -84,7 +84,7 @@ static __always_inline void do_syscall_x
}
/* Returns true to return using SYSRET, or false to use IRET */
-__visible noinstr bool do_syscall_64(struct pt_regs *regs, long nr)
+__visible noinstr __noendbr bool do_syscall_64(struct pt_regs *regs, long nr)
{
if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) {
instrumentation_begin();
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -96,6 +96,6 @@ static __always_inline void arch_exit_to
}
#define arch_exit_to_user_mode arch_exit_to_user_mode
-extern void x86_entry_from_kvm(unsigned int entry_type, unsigned int vector);
+extern __noendbr void x86_entry_from_kvm(unsigned int entry_type, unsigned int vector);
#endif
--- a/arch/x86/include/asm/fred.h
+++ b/arch/x86/include/asm/fred.h
@@ -67,9 +67,9 @@ void asm_fred_entrypoint_user(void);
void asm_fred_entrypoint_kernel(void);
void asm_fred_entry_from_kvm(struct fred_ss);
-__visible void fred_entry_from_user(struct pt_regs *regs);
-__visible void fred_entry_from_kernel(struct pt_regs *regs);
-__visible void __fred_entry_from_kvm(struct pt_regs *regs);
+__visible __noendbr void fred_entry_from_user(struct pt_regs *regs);
+__visible __noendbr void fred_entry_from_kernel(struct pt_regs *regs);
+__visible __noendbr void __fred_entry_from_kvm(struct pt_regs *regs);
/* Can be called from noinstr code, thus __always_inline */
static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector)
--- a/arch/x86/include/asm/syscall.h
+++ b/arch/x86/include/asm/syscall.h
@@ -164,13 +164,13 @@ static inline int syscall_get_arch(struc
? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64;
}
-bool do_syscall_64(struct pt_regs *regs, long nr);
-void do_int80_emulation(struct pt_regs *regs);
+__noendbr bool do_syscall_64(struct pt_regs *regs, long nr);
+__noendbr void do_int80_emulation(struct pt_regs *regs);
#endif /* CONFIG_X86_32 */
-void do_int80_syscall_32(struct pt_regs *regs);
-bool do_fast_syscall_32(struct pt_regs *regs);
-bool do_SYSENTER_32(struct pt_regs *regs);
+__noendbr void do_int80_syscall_32(struct pt_regs *regs);
+__noendbr bool do_fast_syscall_32(struct pt_regs *regs);
+__noendbr bool do_SYSENTER_32(struct pt_regs *regs);
#endif /* _ASM_X86_SYSCALL_H */