Re: [tip:x86/trampoline] x86, trampoline: Common infrastructure forlow memory trampolines

[Brian Gerst]

On Fri, Feb 18, 2011 at 12:19 AM, tip-bot for H. Peter Anvin
<hpa@xxxxxxxxxxxxxxx> wrote:
> Commit-ID: Â4822b7fc6d4870685a9feadfc348d48f5e47460a
> Gitweb: Â Â http://git.kernel.org/tip/4822b7fc6d4870685a9feadfc348d48f5e47460a
> Author: Â Â H. Peter Anvin <hpa@xxxxxxxxxxxxxxx>
> AuthorDate: Mon, 14 Feb 2011 15:34:57 -0800
> Committer: ÂH. Peter Anvin <hpa@xxxxxxxxxxxxxxx>
> CommitDate: Thu, 17 Feb 2011 21:02:43 -0800
>
> x86, trampoline: Common infrastructure for low memory trampolines
>
> Common infrastructure for low memory trampolines. ÂThis code installs
> the trampolines permanently in low memory very early. ÂIt also permits
> multiple pieces of code to be used for this purpose.
>
> This code also introduces a standard infrastructure for computing
> symbol addresses in the trampoline code.
>
> The only change to the actual SMP trampolines themselves is that the
> 64-bit trampoline has been made reusable -- the previous version would
> overwrite the code with a status variable; this moves the status
> variable to a separate location.
>
> Signed-off-by: H. Peter Anvin <hpa@xxxxxxxxxxxxxxx>
> LKML-Reference: <4D5DFBE4.7090104@xxxxxxxxx>
> Cc: Rafael J. Wysocki <rjw@xxxxxxx>
> Cc: Matthieu Castet <castet.matthieu@xxxxxxx>
> Cc: Stephen Rothwell <sfr@xxxxxxxxxxxxxxxx>
> ---
> Âarch/x86/Kconfig        Â|  Â4 ---
> Âarch/x86/kernel/Makefile    Â|  Â3 +-
> Âarch/x86/kernel/head32.c    Â|  Â9 --------
> Âarch/x86/kernel/head_64.S Â Â Â | Â Â3 +-
> Âarch/x86/kernel/setup.c     |  Â2 +-
> Âarch/x86/kernel/smpboot.c    |  10 +++++---
> Âarch/x86/kernel/trampoline.c  Â|  42 ++++++++++++++++++++------------------
> Âarch/x86/kernel/trampoline_32.S | Â 15 ++++++++++---
> Âarch/x86/kernel/trampoline_64.S | Â 30 ++++++++++++++++-----------
> Âarch/x86/kernel/vmlinux.lds.S Â | Â 13 ++++++++++++
> Â10 files changed, 73 insertions(+), 58 deletions(-)
>
> diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
> index d5ed94d..1359bc9 100644
> --- a/arch/x86/Kconfig
> +++ b/arch/x86/Kconfig
> @@ -217,10 +217,6 @@ config X86_HT
> Â Â Â Âdef_bool y
> Â Â Â Âdepends on SMP
>
> -config X86_TRAMPOLINE
> - Â Â Â def_bool y
> - Â Â Â depends on SMP || (64BIT && ACPI_SLEEP)
> -
> Âconfig X86_32_LAZY_GS
> Â Â Â Âdef_bool y
> Â Â Â Âdepends on X86_32 && !CC_STACKPROTECTOR
> diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
> index 34244b2..2e8ce0d 100644
> --- a/arch/x86/kernel/Makefile
> +++ b/arch/x86/kernel/Makefile
> @@ -47,7 +47,7 @@ obj-y         += tsc.o io_delay.o rtc.o
> Âobj-y         Â+= pci-iommu_table.o
> Âobj-y         Â+= resource.o
>
> -obj-$(CONFIG_X86_TRAMPOLINE) Â += trampoline.o
> +obj-y             Â+= trampoline.o trampoline_$(BITS).o
> Âobj-y             Â+= process.o
> Âobj-y             Â+= i387.o xsave.o
> Âobj-y             Â+= ptrace.o
> @@ -69,7 +69,6 @@ obj-$(CONFIG_SMP) Â Â Â Â Â Â += smp.o
> Âobj-$(CONFIG_SMP) Â Â Â Â Â Â Â+= smpboot.o tsc_sync.o
> Âobj-$(CONFIG_SMP) Â Â Â Â Â Â Â+= setup_percpu.o
> Âobj-$(CONFIG_X86_64_SMP) Â Â Â += tsc_sync.o
> -obj-$(CONFIG_X86_TRAMPOLINE) Â += trampoline_$(BITS).o
> Âobj-$(CONFIG_X86_MPPARSE) Â Â Â+= mpparse.o
> Âobj-y             Â+= apic/
> Âobj-$(CONFIG_X86_REBOOTFIXUPS) += reboot_fixups_32.o
> diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c
> index 7f138b3..d6d6bb3 100644
> --- a/arch/x86/kernel/head32.c
> +++ b/arch/x86/kernel/head32.c
> @@ -34,15 +34,6 @@ void __init i386_start_kernel(void)
> Â{
> Â Â Â Âmemblock_init();
>
> -#ifdef CONFIG_X86_TRAMPOLINE
> - Â Â Â /*
> - Â Â Â Â* But first pinch a few for the stack/trampoline stuff
> - Â Â Â Â* FIXME: Don't need the extra page at 4K, but need to fix
> - Â Â Â Â* trampoline before removing it. (see the GDT stuff)
> - Â Â Â Â*/
> - Â Â Â memblock_x86_reserve_range(PAGE_SIZE, PAGE_SIZE + PAGE_SIZE, "EX TRAMPOLINE");
> -#endif
> -
> Â Â Â Âmemblock_x86_reserve_range(__pa_symbol(&_text), __pa_symbol(&__bss_stop), "TEXT DATA BSS");
>
> Â#ifdef CONFIG_BLK_DEV_INITRD
> diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
> index 239046b..e11e394 100644
> --- a/arch/x86/kernel/head_64.S
> +++ b/arch/x86/kernel/head_64.S
> @@ -136,10 +136,9 @@ ident_complete:
> Â Â Â Â/* Fixup phys_base */
>    Âaddq  Â%rbp, phys_base(%rip)
>
> -#ifdef CONFIG_X86_TRAMPOLINE
> + Â Â Â /* Fixup trampoline */
>    Âaddq  Â%rbp, trampoline_level4_pgt + 0(%rip)
>    Âaddq  Â%rbp, trampoline_level4_pgt + (511*8)(%rip)
> -#endif
>
> Â Â Â Â/* Due to ENTRY(), sometimes the empty space gets filled with
> Â Â Â Â * zeros. Better take a jmp than relying on empty space being
> diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
> index d3cfe26..994ea20 100644
> --- a/arch/x86/kernel/setup.c
> +++ b/arch/x86/kernel/setup.c
> @@ -935,7 +935,7 @@ void __init setup_arch(char **cmdline_p)
> Â Â Â Âprintk(KERN_DEBUG "initial memory mapped : 0 - %08lx\n",
> Â Â Â Â Â Â Â Â Â Â Â Âmax_pfn_mapped<<PAGE_SHIFT);
>
> - Â Â Â reserve_trampoline_memory();
> + Â Â Â setup_trampolines();
>
> Â#ifdef CONFIG_ACPI_SLEEP
> Â Â Â Â/*
> diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
> index 08776a9..5452733 100644
> --- a/arch/x86/kernel/smpboot.c
> +++ b/arch/x86/kernel/smpboot.c
> @@ -788,7 +788,7 @@ do_rest:
> Â Â Â Âstack_start Â= c_idle.idle->thread.sp;
>
> Â Â Â Â/* start_ip had better be page-aligned! */
> - Â Â Â start_ip = setup_trampoline();
> + Â Â Â start_ip = trampoline_address();
>
> Â Â Â Â/* So we see what's up */
> Â Â Â Âannounce_cpu(cpu, apicid);
> @@ -798,6 +798,8 @@ do_rest:
> Â Â Â Â * the targeted processor.
> Â Â Â Â */
>
> + Â Â Â printk(KERN_DEBUG "smpboot cpu %d: start_ip = %lx\n", cpu, start_ip);
> +
> Â Â Â Âatomic_set(&init_deasserted, 0);
>
> Â Â Â Âif (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
> @@ -851,8 +853,8 @@ do_rest:
> Â Â Â Â Â Â Â Â Â Â Â Âpr_debug("CPU%d: has booted.\n", cpu);
> Â Â Â Â Â Â Â Âelse {
> Â Â Â Â Â Â Â Â Â Â Â Âboot_error = 1;
> - Â Â Â Â Â Â Â Â Â Â Â if (*((volatile unsigned char *)trampoline_base)
> - Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â == 0xA5)
> + Â Â Â Â Â Â Â Â Â Â Â if (*(volatile u32 *)TRAMPOLINE_SYM(trampoline_status)
> + Â Â Â Â Â Â Â Â Â Â Â Â Â == 0xA5A5A5A5)
> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â/* trampoline started but...? */
> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Âpr_err("CPU%d: Stuck ??\n", cpu);
> Â Â Â Â Â Â Â Â Â Â Â Âelse
> @@ -878,7 +880,7 @@ do_rest:
> Â Â Â Â}
>
> Â Â Â Â/* mark "stuck" area as not stuck */
> - Â Â Â *((volatile unsigned long *)trampoline_base) = 0;
> + Â Â Â *(volatile u32 *)TRAMPOLINE_SYM(trampoline_status) = 0;
>
> Â Â Â Âif (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
> Â Â Â Â Â Â Â Â/*
> diff --git a/arch/x86/kernel/trampoline.c b/arch/x86/kernel/trampoline.c
> index a375616..a91ae77 100644
> --- a/arch/x86/kernel/trampoline.c
> +++ b/arch/x86/kernel/trampoline.c
> @@ -2,39 +2,41 @@
> Â#include <linux/memblock.h>
>
> Â#include <asm/trampoline.h>
> +#include <asm/cacheflush.h>
> Â#include <asm/pgtable.h>
>
> -#if defined(CONFIG_X86_64) && defined(CONFIG_ACPI_SLEEP)
> -#define __trampinit
> -#define __trampinitdata
> -#else
> -#define __trampinit __cpuinit
> -#define __trampinitdata __cpuinitdata
> -#endif
> +unsigned char *x86_trampoline_base;
>
> -/* ready for x86_64 and x86 */
> -unsigned char *__trampinitdata trampoline_base;
> -
> -void __init reserve_trampoline_memory(void)
> +void __init setup_trampolines(void)
> Â{
> Â Â Â Âphys_addr_t mem;
> + Â Â Â size_t size = PAGE_ALIGN(x86_trampoline_end - x86_trampoline_start);
>
> Â Â Â Â/* Has to be in very low memory so we can execute real-mode AP code. */
> - Â Â Â mem = memblock_find_in_range(0, 1<<20, TRAMPOLINE_SIZE, PAGE_SIZE);
> + Â Â Â mem = memblock_find_in_range(0, 1<<20, size, PAGE_SIZE);
> Â Â Â Âif (mem == MEMBLOCK_ERROR)
> Â Â Â Â Â Â Â Âpanic("Cannot allocate trampoline\n");
>
> - Â Â Â trampoline_base = __va(mem);
> - Â Â Â memblock_x86_reserve_range(mem, mem + TRAMPOLINE_SIZE, "TRAMPOLINE");
> + Â Â Â x86_trampoline_base = __va(mem);
> + Â Â Â memblock_x86_reserve_range(mem, mem + size, "TRAMPOLINE");
> +
> + Â Â Â printk(KERN_DEBUG "Base memory trampoline at [%p] %llx size %zu\n",
> + Â Â Â Â Â Â Âx86_trampoline_base, (unsigned long long)mem, size);
> +
> + Â Â Â memcpy(x86_trampoline_base, x86_trampoline_start, size);
> Â}
>
> Â/*
> - * Currently trivial. Write the real->protected mode
> - * bootstrap into the page concerned. The caller
> - * has made sure it's suitably aligned.
> + * setup_trampolines() gets called very early, to guarantee the
> + * availability of low memory. ÂThis is before the proper kernel page
> + * tables are set up, so we cannot set page permissions in that
> + * function. ÂThus, we use an arch_initcall instead.
> Â*/
> -unsigned long __trampinit setup_trampoline(void)
> +static int __init configure_trampolines(void)
> Â{
> - Â Â Â memcpy(trampoline_base, trampoline_data, TRAMPOLINE_SIZE);
> - Â Â Â return virt_to_phys(trampoline_base);
> + Â Â Â size_t size = PAGE_ALIGN(x86_trampoline_end - x86_trampoline_start);
> +
> + Â Â Â set_memory_x((unsigned long)x86_trampoline_base, size >> PAGE_SHIFT);
> + Â Â Â return 0;
> Â}
> +arch_initcall(configure_trampolines);
> diff --git a/arch/x86/kernel/trampoline_32.S b/arch/x86/kernel/trampoline_32.S
> index 8508237..451c0a7 100644
> --- a/arch/x86/kernel/trampoline_32.S
> +++ b/arch/x86/kernel/trampoline_32.S
> @@ -32,9 +32,11 @@
> Â#include <asm/segment.h>
> Â#include <asm/page_types.h>
>
> -/* We can free up trampoline after bootup if cpu hotplug is not supported. */
> -__CPUINITRODATA
> -.code16
> +#ifdef CONFIG_SMP
> +
> + Â Â Â .section ".x86_trampoline","a"
> + Â Â Â .balign PAGE_SIZE
> + Â Â Â .code16
>
> ÂENTRY(trampoline_data)
> Âr_base = .
> @@ -44,7 +46,7 @@ r_base = .
>
>    Âcli           # We should be safe anyway
>
> -    movl  Â$0xA5A5A5A5, trampoline_data - r_base
> +    movl  Â$0xA5A5A5A5, trampoline_status - r_base
> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â# write marker for master knows we're running
>
> Â Â Â Â/* GDT tables in non default location kernel can be beyond 16MB and
> @@ -72,5 +74,10 @@ boot_idt_descr:
>    Â.word  0                # idt limit = 0
>    Â.long  0                # idt base = 0L
>
> +ENTRY(trampoline_status)
> +    .long  0
> +
> Â.globl trampoline_end
> Âtrampoline_end:
> +
> +#endif /* CONFIG_SMP */
> diff --git a/arch/x86/kernel/trampoline_64.S b/arch/x86/kernel/trampoline_64.S
> index 075d130..49c77a6 100644
> --- a/arch/x86/kernel/trampoline_64.S
> +++ b/arch/x86/kernel/trampoline_64.S
> @@ -32,13 +32,9 @@
> Â#include <asm/segment.h>
> Â#include <asm/processor-flags.h>
>
> -#ifdef CONFIG_ACPI_SLEEP
> -.section .rodata, "a", @progbits
> -#else
> -/* We can free up the trampoline after bootup if cpu hotplug is not supported. */
> -__CPUINITRODATA
> -#endif
> -.code16
> + Â Â Â .section ".x86_trampoline","a"
> + Â Â Â .balign PAGE_SIZE
> + Â Â Â .code16
>
> ÂENTRY(trampoline_data)
> Âr_base = .
> @@ -50,7 +46,7 @@ r_base = .
>    Âmov   %ax, %ss
>
>
> -    movl  Â$0xA5A5A5A5, trampoline_data - r_base
> +    movl  Â$0xA5A5A5A5, trampoline_status - r_base
> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â# write marker for master knows we're running
>
> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â# Setup stack
> @@ -64,10 +60,13 @@ r_base = .
>    Âmovzx  %ax, %esi        # Find the 32bit trampoline location
>    Âshll  Â$4, %esi
>
> - Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â # Fixup the vectors
> -    addl  Â%esi, startup_32_vector - r_base
> -    addl  Â%esi, startup_64_vector - r_base
> -    addl  Â%esi, tgdt + 2 - r_base # Fixup the gdt pointer
> + Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â # Fixup the absolute vectors
> +    leal  Â(startup_32 - r_base)(%esi), %eax
> +    movl  Â%eax, startup_32_vector - r_base
> +    leal  Â(startup_64 - r_base)(%esi), %eax
> +    movl  Â%eax, startup_64_vector - r_base
> +    leal  Â(tgdt - r_base)(%esi), %eax
> +    movl  Â%eax, (tgdt + 2 - r_base)
>
> Â Â Â Â/*
> Â Â Â Â * GDT tables in non default location kernel can be beyond 16MB and
> @@ -129,6 +128,7 @@ no_longmode:
> Â Â Â Âjmp no_longmode
> Â#include "verify_cpu.S"
>
> + Â Â Â .balign 4
> Â Â Â Â# Careful these need to be in the same 64K segment as the above;
> Âtidt:
>    Â.word  0            # idt limit = 0
> @@ -156,6 +156,12 @@ startup_64_vector:
>    Â.long  startup_64 - r_base
>    Â.word  __KERNEL_CS, 0
>
> + Â Â Â .balign 4
> +fixup_base:
> +    .long  0

fixup_base looks unused.

> +ENTRY(trampoline_status)
> +    .long  0
> +
> Âtrampoline_stack:
> Â Â Â Â.org 0x1000
> Âtrampoline_stack_end:
> diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
> index bf47007..cb2c506 100644
> --- a/arch/x86/kernel/vmlinux.lds.S
> +++ b/arch/x86/kernel/vmlinux.lds.S
> @@ -240,6 +240,18 @@ SECTIONS
>
> Â Â Â ÂINIT_DATA_SECTION(16)
>
> + Â Â Â /*
> + Â Â Â Â* Code and data for a variety of lowlevel trampolines, to be
> + Â Â Â Â* copied into base memory (< 1 MiB) during initialization.
> + Â Â Â Â* Since it is copied early, the main copy can be discarded
> + Â Â Â Â* afterwards.
> + Â Â Â Â*/
> + Â Â Â Â.x86_trampoline : AT(ADDR(.x86_trampoline) - LOAD_OFFSET) {
> + Â Â Â Â Â Â Â x86_trampoline_start = .;
> + Â Â Â Â Â Â Â *(.x86_trampoline)
> + Â Â Â Â Â Â Â x86_trampoline_end = .;
> + Â Â Â }
> +
> Â Â Â Â.x86_cpu_dev.init : AT(ADDR(.x86_cpu_dev.init) - LOAD_OFFSET) {
> Â Â Â Â Â Â Â Â__x86_cpu_dev_start = .;
> Â Â Â Â Â Â Â Â*(.x86_cpu_dev.init)
> @@ -291,6 +303,7 @@ SECTIONS
> Â Â Â Â Â Â Â Â*(.iommu_table)
> Â Â Â Â Â Â Â Â__iommu_table_end = .;
> Â Â Â Â}
> +
> Â Â Â Â. = ALIGN(8);
> Â Â Â Â/*
> Â Â Â Â * .exit.text is discard at runtime, not link time, to deal with
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@xxxxxxxxxxxxxxx
> More majordomo info at Âhttp://vger.kernel.org/majordomo-info.html
> Please read the FAQ at Âhttp://www.tux.org/lkml/
>

--
Brian Gerst
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/