Re: [PATCH 6/6] s390: introduce execute-trampolines for branches

From: Pavel Machek
Date: Wed Feb 07 2018 - 05:07:35 EST

On Wed 2018-02-07 08:00:11, Martin Schwidefsky wrote:
> Add CONFIG_EXPOLINE to enable the use of the new -mindirect-branch= and
> -mfunction_return= compiler options to create a kernel fortified against
> the specte v2 attack.
>
> With CONFIG_EXPOLINE=y all indirect branches will be issued with an
> execute type instruction. For z10 or newer the EXRL instruction will
> be used, for older machines the EX instruction. The typical indirect
> call
>
> basr %r14,%r1
>
> is replaced with a PC relative call to a new thunk
>
> brasl %r14,__s390x_indirect_jump_r1
>
> The thunk contains the EXRL/EX instruction to the indirect branch
>
> __s390x_indirect_jump_r1:
> exrl 0,0f
> j .
> 0: br %r1
>
> The detour via the execute type instruction has a performance impact.
> To get rid of the detour the new kernel parameter "nospectre_v2" and
> "spectre_v2=[on,off,auto]" can be used. If the parameter is specified
> the kernel and module code will be patched at runtime.

This is really unfortunate naming of kernel option.

spectre_v2=off sounds like we are turning the "bug" off, but i somehow
suspect you are turning the bug _workaround_ off.

Pavel

> Signed-off-by: Martin Schwidefsky <schwidefsky@xxxxxxxxxx>
> ---
> arch/s390/Kconfig | 28 +++++++++
> arch/s390/Makefile | 12 ++++
> arch/s390/include/asm/lowcore.h | 6 +-
> arch/s390/include/asm/nospec-branch.h | 18 ++++++
> arch/s390/kernel/Makefile | 4 ++
> arch/s390/kernel/entry.S | 113 ++++++++++++++++++++++++++--------
> arch/s390/kernel/module.c | 62 ++++++++++++++++---
> arch/s390/kernel/nospec-branch.c | 100 ++++++++++++++++++++++++++++++
> arch/s390/kernel/setup.c | 4 ++
> arch/s390/kernel/smp.c | 1 +
> arch/s390/kernel/vmlinux.lds.S | 14 +++++
> drivers/s390/char/Makefile | 2 +
> 12 files changed, 329 insertions(+), 35 deletions(-)
> create mode 100644 arch/s390/include/asm/nospec-branch.h
> create mode 100644 arch/s390/kernel/nospec-branch.c
>
> diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
> index d514e25..d4a65bf 100644
> --- a/arch/s390/Kconfig
> +++ b/arch/s390/Kconfig
> @@ -557,6 +557,34 @@ config KERNEL_NOBP
>
> If unsure, say N.
>
> +config EXPOLINE
> + def_bool n
> + prompt "Avoid speculative indirect branches in the kernel"
> + help
> + Compile the kernel with the expoline compiler options to guard
> + against kernel-to-user data leaks by avoiding speculative indirect
> + branches.
> + Requires a compiler with -mindirect-branch=thunk support for full
> + protection. The kernel may run slower.
> +
> + If unsure, say N.
> +
> +choice
> + prompt "Expoline default"
> + depends on EXPOLINE
> + default EXPOLINE_FULL
> +
> +config EXPOLINE_OFF
> + bool "spectre_v2=off"
> +
> +config EXPOLINE_MEDIUM
> + bool "spectre_v2=auto"
> +
> +config EXPOLINE_FULL
> + bool "spectre_v2=on"
> +
> +endchoice
> +
> endmenu
>
> menu "Memory setup"
> diff --git a/arch/s390/Makefile b/arch/s390/Makefile
> index fd691c4..2f925ef 100644
> --- a/arch/s390/Makefile
> +++ b/arch/s390/Makefile
> @@ -78,6 +78,18 @@ ifeq ($(call cc-option-yn,-mwarn-dynamicstack),y)
> cflags-$(CONFIG_WARN_DYNAMIC_STACK) += -mwarn-dynamicstack
> endif
>
> +ifdef CONFIG_EXPOLINE
> + ifeq ($(call cc-option-yn,$(CC_FLAGS_MARCH) -mindirect-branch=thunk),y)
> + CC_FLAGS_EXPOLINE := -mindirect-branch=thunk
> + CC_FLAGS_EXPOLINE += -mfunction-return=thunk
> + CC_FLAGS_EXPOLINE += -mindirect-branch-table
> + export CC_FLAGS_EXPOLINE
> + cflags-y += $(CC_FLAGS_EXPOLINE)
> + else
> + $(warning "Your gcc lacks the -mindirect-branch= option")
> + endif
> +endif
> +
> ifdef CONFIG_FUNCTION_TRACER
> # make use of hotpatch feature if the compiler supports it
> cc_hotpatch := -mhotpatch=0,3
> diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
> index c63986a..5bc8888 100644
> --- a/arch/s390/include/asm/lowcore.h
> +++ b/arch/s390/include/asm/lowcore.h
> @@ -136,7 +136,11 @@ struct lowcore {
> __u64 vdso_per_cpu_data; /* 0x03b8 */
> __u64 machine_flags; /* 0x03c0 */
> __u64 gmap; /* 0x03c8 */
> - __u8 pad_0x03d0[0x0e00-0x03d0]; /* 0x03d0 */
> + __u8 pad_0x03d0[0x0400-0x03d0]; /* 0x03d0 */
> +
> + /* br %r1 trampoline */
> + __u16 br_r1_trampoline; /* 0x0400 */
> + __u8 pad_0x0402[0x0e00-0x0402]; /* 0x0402 */
>
> /*
> * 0xe00 contains the address of the IPL Parameter Information
> diff --git a/arch/s390/include/asm/nospec-branch.h b/arch/s390/include/asm/nospec-branch.h
> new file mode 100644
> index 0000000..7df48e5
> --- /dev/null
> +++ b/arch/s390/include/asm/nospec-branch.h
> @@ -0,0 +1,18 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef _ASM_S390_EXPOLINE_H
> +#define _ASM_S390_EXPOLINE_H
> +
> +#ifndef __ASSEMBLY__
> +
> +#include <linux/types.h>
> +
> +extern int nospec_call_disable;
> +extern int nospec_return_disable;
> +
> +void nospec_init_branches(void);
> +void nospec_call_revert(s32 *start, s32 *end);
> +void nospec_return_revert(s32 *start, s32 *end);
> +
> +#endif /* __ASSEMBLY__ */
> +
> +#endif /* _ASM_S390_EXPOLINE_H */
> diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile
> index 909bce6..7f27e3d 100644
> --- a/arch/s390/kernel/Makefile
> +++ b/arch/s390/kernel/Makefile
> @@ -29,6 +29,7 @@ UBSAN_SANITIZE_early.o := n
> #
> ifneq ($(CC_FLAGS_MARCH),-march=z900)
> CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH)
> +CFLAGS_REMOVE_als.o += $(CC_FLAGS_EXPOLINE)
> CFLAGS_als.o += -march=z900
> AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH)
> AFLAGS_head.o += -march=z900
> @@ -63,6 +64,9 @@ obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o
>
> extra-y += head.o head64.o vmlinux.lds
>
> +obj-$(CONFIG_EXPOLINE) += nospec-branch.o
> +CFLAGS_REMOVE_expoline.o += $(CC_FLAGS_EXPOLINE)
> +
> obj-$(CONFIG_MODULES) += module.o
> obj-$(CONFIG_SMP) += smp.o
> obj-$(CONFIG_SCHED_TOPOLOGY) += topology.o
> diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
> index 53145b5..13a133a 100644
> --- a/arch/s390/kernel/entry.S
> +++ b/arch/s390/kernel/entry.S
> @@ -222,6 +222,68 @@ _PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
> .popsection
> .endm
>
> +#ifdef CONFIG_EXPOLINE
> +
> + .macro GEN_BR_THUNK name,reg,tmp
> + .section .text.\name,"axG",@progbits,\name,comdat
> + .globl \name
> + .hidden \name
> + .type \name,@function
> +\name:
> + .cfi_startproc
> +#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES
> + exrl 0,0f
> +#else
> + larl \tmp,0f
> + ex 0,0(\tmp)
> +#endif
> + j .
> +0: br \reg
> + .cfi_endproc
> + .endm
> +
> + GEN_BR_THUNK __s390x_indirect_jump_r1use_r9,%r9,%r1
> + GEN_BR_THUNK __s390x_indirect_jump_r1use_r14,%r14,%r1
> + GEN_BR_THUNK __s390x_indirect_jump_r11use_r14,%r14,%r11
> +
> + .macro BASR_R14_R9
> +0: brasl %r14,__s390x_indirect_jump_r1use_r9
> + .pushsection .s390_indirect_branches,"a",@progbits
> + .long 0b-.
> + .popsection
> + .endm
> +
> + .macro BR_R1USE_R14
> +0: jg __s390x_indirect_jump_r1use_r14
> + .pushsection .s390_indirect_branches,"a",@progbits
> + .long 0b-.
> + .popsection
> + .endm
> +
> + .macro BR_R11USE_R14
> +0: jg __s390x_indirect_jump_r11use_r14
> + .pushsection .s390_indirect_branches,"a",@progbits
> + .long 0b-.
> + .popsection
> + .endm
> +
> +#else /* CONFIG_EXPOLINE */
> +
> + .macro BASR_R14_R9
> + basr %r14,%r9
> + .endm
> +
> + .macro BR_R1USE_R14
> + br %r14
> + .endm
> +
> + .macro BR_R11USE_R14
> + br %r14
> + .endm
> +
> +#endif /* CONFIG_EXPOLINE */
> +
> +
> .section .kprobes.text, "ax"
> .Ldummy:
> /*
> @@ -237,7 +299,7 @@ _PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
> ENTRY(__bpon)
> .globl __bpon
> BPON
> - br %r14
> + BR_R1USE_R14
>
> /*
> * Scheduler resume function, called by switch_to
> @@ -261,9 +323,9 @@ ENTRY(__switch_to)
> mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next
> lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
> TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_LPP
> - bzr %r14
> + jz 0f
> .insn s,0xb2800000,__LC_LPP # set program parameter
> - br %r14
> +0: BR_R1USE_R14
>
> .L__critical_start:
>
> @@ -330,7 +392,7 @@ sie_exit:
> xgr %r5,%r5
> lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
> lg %r2,__SF_EMPTY+16(%r15) # return exit reason code
> - br %r14
> + BR_R1USE_R14
> .Lsie_fault:
> lghi %r14,-EFAULT
> stg %r14,__SF_EMPTY+16(%r15) # set exit reason code
> @@ -389,7 +451,7 @@ ENTRY(system_call)
> lgf %r9,0(%r8,%r10) # get system call add.
> TSTMSK __TI_flags(%r12),_TIF_TRACE
> jnz .Lsysc_tracesys
> - basr %r14,%r9 # call sys_xxxx
> + BASR_R14_R9 # call sys_xxxx
> stg %r2,__PT_R2(%r11) # store return value
>
> .Lsysc_return:
> @@ -574,7 +636,7 @@ ENTRY(system_call)
> lmg %r3,%r7,__PT_R3(%r11)
> stg %r7,STACK_FRAME_OVERHEAD(%r15)
> lg %r2,__PT_ORIG_GPR2(%r11)
> - basr %r14,%r9 # call sys_xxx
> + BASR_R14_R9 # call sys_xxx
> stg %r2,__PT_R2(%r11) # store return value
> .Lsysc_tracenogo:
> TSTMSK __TI_flags(%r12),_TIF_TRACE
> @@ -598,7 +660,7 @@ ENTRY(ret_from_fork)
> lmg %r9,%r10,__PT_R9(%r11) # load gprs
> ENTRY(kernel_thread_starter)
> la %r2,0(%r10)
> - basr %r14,%r9
> + BASR_R14_R9
> j .Lsysc_tracenogo
>
> /*
> @@ -678,9 +740,9 @@ ENTRY(pgm_check_handler)
> nill %r10,0x007f
> sll %r10,2
> je .Lpgm_return
> - lgf %r1,0(%r10,%r1) # load address of handler routine
> + lgf %r9,0(%r10,%r1) # load address of handler routine
> lgr %r2,%r11 # pass pointer to pt_regs
> - basr %r14,%r1 # branch to interrupt-handler
> + BASR_R14_R9 # branch to interrupt-handler
> .Lpgm_return:
> LOCKDEP_SYS_EXIT
> tm __PT_PSW+1(%r11),0x01 # returning to user ?
> @@ -998,7 +1060,7 @@ ENTRY(psw_idle)
> stpt __TIMER_IDLE_ENTER(%r2)
> .Lpsw_idle_lpsw:
> lpswe __SF_EMPTY(%r15)
> - br %r14
> + BR_R1USE_R14
> .Lpsw_idle_end:
>
> /*
> @@ -1012,7 +1074,7 @@ ENTRY(save_fpu_regs)
> lg %r2,__LC_CURRENT
> aghi %r2,__TASK_thread
> TSTMSK __LC_CPU_FLAGS,_CIF_FPU
> - bor %r14
> + jo .Lsave_fpu_regs_exit
> stfpc __THREAD_FPU_fpc(%r2)
> lg %r3,__THREAD_FPU_regs(%r2)
> TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
> @@ -1039,7 +1101,8 @@ ENTRY(save_fpu_regs)
> std 15,120(%r3)
> .Lsave_fpu_regs_done:
> oi __LC_CPU_FLAGS+7,_CIF_FPU
> - br %r14
> +.Lsave_fpu_regs_exit:
> + BR_R1USE_R14
> .Lsave_fpu_regs_end:
> EXPORT_SYMBOL(save_fpu_regs)
>
> @@ -1057,7 +1120,7 @@ load_fpu_regs:
> lg %r4,__LC_CURRENT
> aghi %r4,__TASK_thread
> TSTMSK __LC_CPU_FLAGS,_CIF_FPU
> - bnor %r14
> + jno .Lload_fpu_regs_exit
> lfpc __THREAD_FPU_fpc(%r4)
> TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
> lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area
> @@ -1084,7 +1147,8 @@ load_fpu_regs:
> ld 15,120(%r4)
> .Lload_fpu_regs_done:
> ni __LC_CPU_FLAGS+7,255-_CIF_FPU
> - br %r14
> +.Lload_fpu_regs_exit:
> + BR_R1USE_R14
> .Lload_fpu_regs_end:
>
> .L__critical_end:
> @@ -1301,7 +1365,7 @@ cleanup_critical:
> jl 0f
> clg %r9,BASED(.Lcleanup_table+104) # .Lload_fpu_regs_end
> jl .Lcleanup_load_fpu_regs
> -0: br %r14
> +0: BR_R11USE_R14
>
> .align 8
> .Lcleanup_table:
> @@ -1337,7 +1401,7 @@ cleanup_critical:
> ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
> lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
> larl %r9,sie_exit # skip forward to sie_exit
> - br %r14
> + BR_R11USE_R14
> #endif
>
> .Lcleanup_system_call:
> @@ -1390,7 +1454,7 @@ cleanup_critical:
> stg %r15,56(%r11) # r15 stack pointer
> # set new psw address and exit
> larl %r9,.Lsysc_do_svc
> - br %r14
> + BR_R11USE_R14
> .Lcleanup_system_call_insn:
> .quad system_call
> .quad .Lsysc_stmg
> @@ -1402,7 +1466,7 @@ cleanup_critical:
>
> .Lcleanup_sysc_tif:
> larl %r9,.Lsysc_tif
> - br %r14
> + BR_R11USE_R14
>
> .Lcleanup_sysc_restore:
> # check if stpt has been executed
> @@ -1419,14 +1483,14 @@ cleanup_critical:
> mvc 0(64,%r11),__PT_R8(%r9)
> lmg %r0,%r7,__PT_R0(%r9)
> 1: lmg %r8,%r9,__LC_RETURN_PSW
> - br %r14
> + BR_R11USE_R14
> .Lcleanup_sysc_restore_insn:
> .quad .Lsysc_exit_timer
> .quad .Lsysc_done - 4
>
> .Lcleanup_io_tif:
> larl %r9,.Lio_tif
> - br %r14
> + BR_R11USE_R14
>
> .Lcleanup_io_restore:
> # check if stpt has been executed
> @@ -1440,7 +1504,7 @@ cleanup_critical:
> mvc 0(64,%r11),__PT_R8(%r9)
> lmg %r0,%r7,__PT_R0(%r9)
> 1: lmg %r8,%r9,__LC_RETURN_PSW
> - br %r14
> + BR_R11USE_R14
> .Lcleanup_io_restore_insn:
> .quad .Lio_exit_timer
> .quad .Lio_done - 4
> @@ -1493,17 +1557,17 @@ cleanup_critical:
> # prepare return psw
> nihh %r8,0xfcfd # clear irq & wait state bits
> lg %r9,48(%r11) # return from psw_idle
> - br %r14
> + BR_R11USE_R14
> .Lcleanup_idle_insn:
> .quad .Lpsw_idle_lpsw
>
> .Lcleanup_save_fpu_regs:
> larl %r9,save_fpu_regs
> - br %r14
> + BR_R11USE_R14
>
> .Lcleanup_load_fpu_regs:
> larl %r9,load_fpu_regs
> - br %r14
> + BR_R11USE_R14
>
> /*
> * Integer constants
> @@ -1523,7 +1587,6 @@ cleanup_critical:
> .Lsie_crit_mcck_length:
> .quad .Lsie_skip - .Lsie_entry
> #endif
> -
> .section .rodata, "a"
> #define SYSCALL(esame,emu) .long esame
> .globl sys_call_table
> diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c
> index b7abfad..1fc6d1f 100644
> --- a/arch/s390/kernel/module.c
> +++ b/arch/s390/kernel/module.c
> @@ -19,6 +19,8 @@
> #include <linux/moduleloader.h>
> #include <linux/bug.h>
> #include <asm/alternative.h>
> +#include <asm/nospec-branch.h>
> +#include <asm/facility.h>
>
> #if 0
> #define DEBUGP printk
> @@ -156,7 +158,11 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
> me->arch.got_offset = me->core_layout.size;
> me->core_layout.size += me->arch.got_size;
> me->arch.plt_offset = me->core_layout.size;
> - me->core_layout.size += me->arch.plt_size;
> + if (me->arch.plt_size) {
> + if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_call_disable)
> + me->arch.plt_size += PLT_ENTRY_SIZE;
> + me->core_layout.size += me->arch.plt_size;
> + }
> return 0;
> }
>
> @@ -310,9 +316,21 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab,
> unsigned int *ip;
> ip = me->core_layout.base + me->arch.plt_offset +
> info->plt_offset;
> - ip[0] = 0x0d10e310; /* basr 1,0; lg 1,10(1); br 1 */
> - ip[1] = 0x100a0004;
> - ip[2] = 0x07f10000;
> + ip[0] = 0x0d10e310; /* basr 1,0 */
> + ip[1] = 0x100a0004; /* lg 1,10(1) */
> + if (IS_ENABLED(CONFIG_EXPOLINE) &&
> + !nospec_call_disable) {
> + unsigned int *ij;
> + ij = me->core_layout.base +
> + me->arch.plt_offset +
> + me->arch.plt_size - PLT_ENTRY_SIZE;
> + ip[2] = 0xa7f40000 + /* j __jump_r1 */
> + (unsigned int)(u16)
> + (((unsigned long) ij - 8 -
> + (unsigned long) ip) / 2);
> + } else {
> + ip[2] = 0x07f10000; /* br %r1 */
> + }
> ip[3] = (unsigned int) (val >> 32);
> ip[4] = (unsigned int) val;
> info->plt_initialized = 1;
> @@ -418,16 +436,42 @@ int module_finalize(const Elf_Ehdr *hdr,
> struct module *me)
> {
> const Elf_Shdr *s;
> - char *secstrings;
> + char *secstrings, *secname;
> + void *aseg;
> +
> + if (IS_ENABLED(CONFIG_EXPOLINE) &&
> + !nospec_call_disable && me->arch.plt_size) {
> + unsigned int *ij;
> +
> + ij = me->core_layout.base + me->arch.plt_offset +
> + me->arch.plt_size - PLT_ENTRY_SIZE;
> + if (test_facility(35)) {
> + ij[0] = 0xc6000000; /* exrl %r0,.+10 */
> + ij[1] = 0x0005a7f4; /* j . */
> + ij[2] = 0x000007f1; /* br %r1 */
> + } else {
> + ij[0] = 0x44000000 | (unsigned int)
> + offsetof(struct lowcore, br_r1_trampoline);
> + ij[1] = 0xa7f40000; /* j . */
> + }
> + }
>
> secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
> for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
> - if (!strcmp(".altinstructions", secstrings + s->sh_name)) {
> - /* patch .altinstructions */
> - void *aseg = (void *)s->sh_addr;
> + aseg = (void *) s->sh_addr;
> + secname = secstrings + s->sh_name;
>
> + if (!strcmp(".altinstructions", secname))
> + /* patch .altinstructions */
> apply_alternatives(aseg, aseg + s->sh_size);
> - }
> +
> + if (IS_ENABLED(CONFIG_EXPOLINE) &&
> + (!strcmp(".nospec_call_table", secname)))
> + nospec_call_revert(aseg, aseg + s->sh_size);
> +
> + if (IS_ENABLED(CONFIG_EXPOLINE) &&
> + (!strcmp(".nospec_return_table", secname)))
> + nospec_return_revert(aseg, aseg + s->sh_size);
> }
>
> jump_label_apply_nops(me);
> diff --git a/arch/s390/kernel/nospec-branch.c b/arch/s390/kernel/nospec-branch.c
> new file mode 100644
> index 0000000..69d7fcf
> --- /dev/null
> +++ b/arch/s390/kernel/nospec-branch.c
> @@ -0,0 +1,100 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#include <linux/module.h>
> +#include <asm/nospec-branch.h>
> +
> +int nospec_call_disable = IS_ENABLED(EXPOLINE_OFF);
> +int nospec_return_disable = !IS_ENABLED(EXPOLINE_FULL);
> +
> +static int __init nospectre_v2_setup_early(char *str)
> +{
> + nospec_call_disable = 1;
> + nospec_return_disable = 1;
> + return 0;
> +}
> +early_param("nospectre_v2", nospectre_v2_setup_early);
> +
> +static int __init spectre_v2_setup_early(char *str)
> +{
> + if (str && !strncmp(str, "on", 2)) {
> + nospec_call_disable = 0;
> + nospec_return_disable = 0;
> + }
> + if (str && !strncmp(str, "off", 3)) {
> + nospec_call_disable = 1;
> + nospec_return_disable = 1;
> + }
> + if (str && !strncmp(str, "auto", 4)) {
> + nospec_call_disable = 0;
> + nospec_return_disable = 1;
> + }
> + return 0;
> +}
> +early_param("spectre_v2", spectre_v2_setup_early);
> +
> +static void __init_or_module __nospec_revert(s32 *start, s32 *end)
> +{
> + enum { BRCL_EXPOLINE, BRASL_EXPOLINE } type;
> + u8 *instr, *thunk, *br;
> + u8 insnbuf[6];
> + s32 *epo;
> +
> + /* Second part of the instruction replace is always a nop */
> + memcpy(insnbuf + 2, (char[]) { 0x47, 0x00, 0x00, 0x00 }, 4);
> + for (epo = start; epo < end; epo++) {
> + instr = (u8 *) epo + *epo;
> + if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x04)
> + type = BRCL_EXPOLINE; /* brcl instruction */
> + else if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x05)
> + type = BRASL_EXPOLINE; /* brasl instruction */
> + else
> + continue;
> + thunk = instr + (*(int *)(instr + 2)) * 2;
> + if (thunk[0] == 0xc6 && thunk[1] == 0x00)
> + /* exrl %r0,<target-br> */
> + br = thunk + (*(int *)(thunk + 2)) * 2;
> + else if (thunk[0] == 0xc0 && (thunk[1] & 0x0f) == 0x00 &&
> + thunk[6] == 0x44 && thunk[7] == 0x00 &&
> + (thunk[8] & 0x0f) == 0x00 && thunk[9] == 0x00 &&
> + (thunk[1] & 0xf0) == (thunk[8] & 0xf0))
> + /* larl %rx,<target br> + ex %r0,0(%rx) */
> + br = thunk + (*(int *)(thunk + 2)) * 2;
> + else
> + continue;
> + if (br[0] != 0x07 || (br[1] & 0xf0) != 0xf0)
> + continue;
> + switch (type) {
> + case BRCL_EXPOLINE:
> + /* brcl to thunk, replace with br + nop */
> + insnbuf[0] = br[0];
> + insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f);
> + break;
> + case BRASL_EXPOLINE:
> + /* brasl to thunk, replace with basr + nop */
> + insnbuf[0] = 0x0d;
> + insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f);
> + break;
> + }
> +
> + s390_kernel_write(instr, insnbuf, 6);
> + }
> +}
> +
> +void __init_or_module nospec_call_revert(s32 *start, s32 *end)
> +{
> + if (nospec_call_disable)
> + __nospec_revert(start, end);
> +}
> +
> +void __init_or_module nospec_return_revert(s32 *start, s32 *end)
> +{
> + if (nospec_return_disable)
> + __nospec_revert(start, end);
> +}
> +
> +extern s32 __nospec_call_start[], __nospec_call_end[];
> +extern s32 __nospec_return_start[], __nospec_return_end[];
> +void __init nospec_init_branches(void)
> +{
> + nospec_call_revert(__nospec_call_start, __nospec_call_end);
> + nospec_return_revert(__nospec_return_start, __nospec_return_end);
> +}
> diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
> index bcd2a4a..a6a91f0 100644
> --- a/arch/s390/kernel/setup.c
> +++ b/arch/s390/kernel/setup.c
> @@ -68,6 +68,7 @@
> #include <asm/sysinfo.h>
> #include <asm/numa.h>
> #include <asm/alternative.h>
> +#include <asm/nospec-branch.h>
> #include "entry.h"
>
> /*
> @@ -379,6 +380,7 @@ static void __init setup_lowcore(void)
> lc->spinlock_index = 0;
> arch_spin_lock_setup(0);
> #endif
> + lc->br_r1_trampoline = 0x07f1; /* br %r1 */
>
> set_prefix((u32)(unsigned long) lc);
> lowcore_ptr[0] = lc;
> @@ -954,6 +956,8 @@ void __init setup_arch(char **cmdline_p)
> set_preferred_console();
>
> apply_alternative_instructions();
> + if (IS_ENABLED(CONFIG_EXPOLINE))
> + nospec_init_branches();
>
> /* Setup zfcpdump support */
> setup_zfcpdump();
> diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
> index 2fd7d60..a4a9fe1 100644
> --- a/arch/s390/kernel/smp.c
> +++ b/arch/s390/kernel/smp.c
> @@ -214,6 +214,7 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
> lc->cpu_nr = cpu;
> lc->spinlock_lockval = arch_spin_lockval(cpu);
> lc->spinlock_index = 0;
> + lc->br_r1_trampoline = 0x07f1; /* br %r1 */
> if (nmi_alloc_per_cpu(lc))
> goto out;
> if (vdso_alloc_per_cpu(lc))
> diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S
> index 608cf29..08d12cf 100644
> --- a/arch/s390/kernel/vmlinux.lds.S
> +++ b/arch/s390/kernel/vmlinux.lds.S
> @@ -123,6 +123,20 @@ SECTIONS
> *(.altinstr_replacement)
> }
>
> + /*
> + * Table with the patch locations to undo expolines
> + */
> + .nospec_call_table : {
> + __nospec_call_start = . ;
> + *(.s390_indirect*)
> + __nospec_call_end = . ;
> + }
> + .nospec_return_table : {
> + __nospec_return_start = . ;
> + *(.s390_return*)
> + __nospec_return_end = . ;
> + }
> +
> /* early.c uses stsi, which requires page aligned data. */
> . = ALIGN(PAGE_SIZE);
> INIT_DATA_SECTION(0x100)
> diff --git a/drivers/s390/char/Makefile b/drivers/s390/char/Makefile
> index 614b44e..a2b33a2 100644
> --- a/drivers/s390/char/Makefile
> +++ b/drivers/s390/char/Makefile
> @@ -19,6 +19,8 @@ endif
>
> CFLAGS_sclp_early_core.o += -D__NO_FORTIFY
>
> +CFLAGS_REMOVE_sclp_early_core.o += $(CC_FLAGS_EXPOLINE)
> +
> obj-y += ctrlchar.o keyboard.o defkeymap.o sclp.o sclp_rw.o sclp_quiesce.o \
> sclp_cmd.o sclp_config.o sclp_cpi_sys.o sclp_ocf.o sclp_ctl.o \
> sclp_early.o sclp_early_core.o

--
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html

Attachment: signature.asc
Description: Digital signature