Re: [PATCH RESEND v4] reboot: support offline CPUs before reboot

From: Guenter Roeck
Date: Tue Jan 14 2020 - 08:54:16 EST

On Tue, Jan 14, 2020 at 3:07 AM Hsin-Yi Wang <hsinyi@xxxxxxxxxxxx> wrote:
>
> Currently system reboots uses architecture specific codes (smp_send_stop)
> to offline non reboot CPUs. Most architecture's implementation is looping
> through all non reboot online CPUs and call ipi function to each of them. Some
> architecture like arm64, arm, and x86... would set offline masks to cpu without
> really offline them. This causes some race condition and kernel warning comes
> out sometimes when system reboots.
>
> This patch adds a config ARCH_OFFLINE_CPUS_ON_REBOOT, which would offline cpus in
> migrate_to_reboot_cpu(). If non reboot cpus are all offlined here, the loop for
> checking online cpus would be an empty loop. If architecture don't enable this
> config, or some cpus somehow fails to offline, it would fallback to ipi
> function.
>
> Opt in this config for architectures that support CONFIG_HOTPLUG_CPU.
>
> Signed-off-by: Hsin-Yi Wang <hsinyi@xxxxxxxxxxxx>
> ---
> Resend v4:
> * Cc more people and mailing lists. Also fix a few nits from v4.
>
> Change from v3:
> * Opt in config for architectures that support CONFIG_HOTPLUG_CPU
> * Merge function offline_secondary_cpus() and freeze_secondary_cpus()
> with an additional flag.
>
> Change from v2:
> * Add another config instead of configed by CONFIG_HOTPLUG_CPU
>
> Previous related discussion on list:
> https://lore.kernel.org/lkml/20190727164450.GA11726@xxxxxxxxxxxx/
> https://lore.kernel.org/patchwork/patch/1117201/
> ---
> arch/Kconfig | 5 +++++
> arch/arm/Kconfig | 1 +
> arch/arm64/Kconfig | 1 +
> arch/arm64/kernel/hibernate.c | 2 +-
> arch/csky/Kconfig | 1 +
> arch/ia64/Kconfig | 1 +
> arch/mips/Kconfig | 1 +
> arch/parisc/Kconfig | 1 +
> arch/powerpc/Kconfig | 1 +
> arch/s390/Kconfig | 1 +
> arch/sh/Kconfig | 1 +
> arch/sparc/Kconfig | 1 +
> arch/x86/Kconfig | 1 +
> arch/xtensa/Kconfig | 1 +
> drivers/power/reset/sc27xx-poweroff.c | 2 +-
> include/linux/cpu.h | 9 ++++++---
> kernel/cpu.c | 12 ++++++++----
> kernel/reboot.c | 8 ++++++++
> 18 files changed, 41 insertions(+), 9 deletions(-)
>
> diff --git a/arch/Kconfig b/arch/Kconfig
> index 48b5e103bdb0..210095ce2d92 100644
> --- a/arch/Kconfig
> +++ b/arch/Kconfig
> @@ -255,6 +255,11 @@ config ARCH_HAS_UNCACHED_SEGMENT
> select ARCH_HAS_DMA_PREP_COHERENT
> bool
>
> +# Select to do a full offline on secondary CPUs before reboot.
> +config ARCH_OFFLINE_CPUS_ON_REBOOT
> + bool "Support for offline CPUs before reboot"

Doesn't the text make it show up in menuconfig ? Is that really what
you want ? If so, I think you might need a help text as well.

Guenter

> + depends on HOTPLUG_CPU
> +
> # Select if arch init_task must go in the __init_task_data section
> config ARCH_TASK_STRUCT_ON_STACK
> bool
> diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
> index 69950fb5be64..d53cc8cb47e3 100644
> --- a/arch/arm/Kconfig
> +++ b/arch/arm/Kconfig
> @@ -28,6 +28,7 @@ config ARM
> select ARCH_KEEP_MEMBLOCK if HAVE_ARCH_PFN_VALID || KEXEC
> select ARCH_MIGHT_HAVE_PC_PARPORT
> select ARCH_NO_SG_CHAIN if !ARM_HAS_SG_CHAIN
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX
> select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7
> select ARCH_SUPPORTS_ATOMIC_RMW
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 9af26ac75d19..9f913bc5c1f6 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -61,6 +61,7 @@ config ARM64
> select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPTION
> select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION
> select ARCH_KEEP_MEMBLOCK
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ARCH_USE_CMPXCHG_LOCKREF
> select ARCH_USE_QUEUED_RWLOCKS
> select ARCH_USE_QUEUED_SPINLOCKS
> diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
> index 590963c9c609..f7245dfa09d9 100644
> --- a/arch/arm64/kernel/hibernate.c
> +++ b/arch/arm64/kernel/hibernate.c
> @@ -581,5 +581,5 @@ int hibernate_resume_nonboot_cpu_disable(void)
> return -ENODEV;
> }
>
> - return freeze_secondary_cpus(sleep_cpu);
> + return freeze_secondary_cpus(sleep_cpu, false);
> }
> diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig
> index 4acef4088de7..0f03e5c3f2fc 100644
> --- a/arch/csky/Kconfig
> +++ b/arch/csky/Kconfig
> @@ -5,6 +5,7 @@ config CSKY
> select ARCH_HAS_DMA_PREP_COHERENT
> select ARCH_HAS_SYNC_DMA_FOR_CPU
> select ARCH_HAS_SYNC_DMA_FOR_DEVICE
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ARCH_USE_BUILTIN_BSWAP
> select ARCH_USE_QUEUED_RWLOCKS if NR_CPUS>2
> select COMMON_CLK
> diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
> index bab7cd878464..f12b4b11ee98 100644
> --- a/arch/ia64/Kconfig
> +++ b/arch/ia64/Kconfig
> @@ -10,6 +10,7 @@ config IA64
> bool
> select ARCH_MIGHT_HAVE_PC_PARPORT
> select ARCH_MIGHT_HAVE_PC_SERIO
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ACPI
> select ACPI_NUMA if NUMA
> select ARCH_SUPPORTS_ACPI
> diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
> index b6b5f83af169..9bb2556d21fc 100644
> --- a/arch/mips/Kconfig
> +++ b/arch/mips/Kconfig
> @@ -8,6 +8,7 @@ config MIPS
> select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
> select ARCH_HAS_UBSAN_SANITIZE_ALL
> select ARCH_HAS_FORTIFY_SOURCE
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ARCH_SUPPORTS_UPROBES
> select ARCH_USE_BUILTIN_BSWAP
> select ARCH_USE_CMPXCHG_LOCKREF if 64BIT
> diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
> index 71034b54d74e..41609f00b057 100644
> --- a/arch/parisc/Kconfig
> +++ b/arch/parisc/Kconfig
> @@ -13,6 +13,7 @@ config PARISC
> select ARCH_HAS_STRICT_KERNEL_RWX
> select ARCH_HAS_UBSAN_SANITIZE_ALL
> select ARCH_NO_SG_CHAIN
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ARCH_SUPPORTS_MEMORY_FAILURE
> select RTC_CLASS
> select RTC_DRV_GENERIC
> diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
> index 658e0324d256..a6b76dd82a2d 100644
> --- a/arch/powerpc/Kconfig
> +++ b/arch/powerpc/Kconfig
> @@ -142,6 +142,7 @@ config PPC
> select ARCH_KEEP_MEMBLOCK
> select ARCH_MIGHT_HAVE_PC_PARPORT
> select ARCH_MIGHT_HAVE_PC_SERIO
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX
> select ARCH_SUPPORTS_ATOMIC_RMW
> select ARCH_USE_BUILTIN_BSWAP
> diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
> index 287714d51b47..19eec37b1682 100644
> --- a/arch/s390/Kconfig
> +++ b/arch/s390/Kconfig
> @@ -102,6 +102,7 @@ config S390
> select ARCH_INLINE_WRITE_UNLOCK_IRQ
> select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE
> select ARCH_KEEP_MEMBLOCK
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ARCH_SAVE_PAGE_KEYS if HIBERNATION
> select ARCH_STACKWALK
> select ARCH_SUPPORTS_ATOMIC_RMW
> diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
> index 9ece111b0254..4ed1e0ca83a2 100644
> --- a/arch/sh/Kconfig
> +++ b/arch/sh/Kconfig
> @@ -18,6 +18,7 @@ config SUPERH
> select ARCH_HAVE_CUSTOM_GPIO_H
> select ARCH_HAVE_NMI_SAFE_CMPXCHG if (GUSA_RB || CPU_SH4A)
> select ARCH_HAS_GCOV_PROFILE_ALL
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select PERF_USE_VMALLOC
> select HAVE_DEBUG_KMEMLEAK
> select HAVE_KERNEL_GZIP
> diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
> index e8c3ea01c12f..f31700309621 100644
> --- a/arch/sparc/Kconfig
> +++ b/arch/sparc/Kconfig
> @@ -30,6 +30,7 @@ config SPARC
> select RTC_SYSTOHC
> select HAVE_ARCH_JUMP_LABEL if SPARC64
> select GENERIC_IRQ_SHOW
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ARCH_WANT_IPC_PARSE_VERSION
> select GENERIC_PCI_IOMAP
> select HAVE_NMI_WATCHDOG if SPARC64
> diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
> index b595ecb21a0f..e8edab974f67 100644
> --- a/arch/x86/Kconfig
> +++ b/arch/x86/Kconfig
> @@ -85,6 +85,7 @@ config X86
> select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI
> select ARCH_MIGHT_HAVE_PC_PARPORT
> select ARCH_MIGHT_HAVE_PC_SERIO
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ARCH_STACKWALK
> select ARCH_SUPPORTS_ACPI
> select ARCH_SUPPORTS_ATOMIC_RMW
> diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig
> index 1c645172b4b5..c862dfa69ed9 100644
> --- a/arch/xtensa/Kconfig
> +++ b/arch/xtensa/Kconfig
> @@ -7,6 +7,7 @@ config XTENSA
> select ARCH_HAS_SYNC_DMA_FOR_CPU if MMU
> select ARCH_HAS_SYNC_DMA_FOR_DEVICE if MMU
> select ARCH_HAS_UNCACHED_SEGMENT if MMU
> + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU
> select ARCH_USE_QUEUED_RWLOCKS
> select ARCH_USE_QUEUED_SPINLOCKS
> select ARCH_WANT_FRAME_POINTERS
> diff --git a/drivers/power/reset/sc27xx-poweroff.c b/drivers/power/reset/sc27xx-poweroff.c
> index 29fb08b8faa0..d6cdf837235c 100644
> --- a/drivers/power/reset/sc27xx-poweroff.c
> +++ b/drivers/power/reset/sc27xx-poweroff.c
> @@ -30,7 +30,7 @@ static void sc27xx_poweroff_shutdown(void)
> #ifdef CONFIG_PM_SLEEP_SMP
> int cpu = smp_processor_id();
>
> - freeze_secondary_cpus(cpu);
> + freeze_secondary_cpus(cpu, false);
> #endif
> }
>
> diff --git a/include/linux/cpu.h b/include/linux/cpu.h
> index 1ca2baf817ed..9c62274a4db9 100644
> --- a/include/linux/cpu.h
> +++ b/include/linux/cpu.h
> @@ -137,11 +137,14 @@ static inline void cpu_hotplug_done(void) { cpus_write_unlock(); }
> static inline void get_online_cpus(void) { cpus_read_lock(); }
> static inline void put_online_cpus(void) { cpus_read_unlock(); }
>
> +#if defined(CONFIG_PM_SLEEP_SMP) || defined(CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT)
> +extern int freeze_secondary_cpus(int primary, bool reboot);
> +#endif
> +
> #ifdef CONFIG_PM_SLEEP_SMP
> -extern int freeze_secondary_cpus(int primary);
> static inline int disable_nonboot_cpus(void)
> {
> - return freeze_secondary_cpus(0);
> + return freeze_secondary_cpus(0, false);
> }
> extern void enable_nonboot_cpus(void);
>
> @@ -152,7 +155,7 @@ static inline int suspend_disable_secondary_cpus(void)
> if (IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU))
> cpu = -1;
>
> - return freeze_secondary_cpus(cpu);
> + return freeze_secondary_cpus(cpu, false);
> }
> static inline void suspend_enable_secondary_cpus(void)
> {
> diff --git a/kernel/cpu.c b/kernel/cpu.c
> index 9c706af713fb..52d04e4e1aab 100644
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -1209,10 +1209,10 @@ int cpu_up(unsigned int cpu)
> }
> EXPORT_SYMBOL_GPL(cpu_up);
>
> -#ifdef CONFIG_PM_SLEEP_SMP
> +#if defined(CONFIG_PM_SLEEP_SMP) || defined(CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT)
> static cpumask_var_t frozen_cpus;
>
> -int freeze_secondary_cpus(int primary)
> +int freeze_secondary_cpus(int primary, bool reboot)
> {
> int cpu, error = 0;
>
> @@ -1237,11 +1237,13 @@ int freeze_secondary_cpus(int primary)
> if (cpu == primary)
> continue;
>
> - if (pm_wakeup_pending()) {
> +#ifdef CONFIG_PM_SLEEP
> + if (!reboot && pm_wakeup_pending()) {
> pr_info("Wakeup pending. Abort CPU freeze\n");
> error = -EBUSY;
> break;
> }
> +#endif
>
> trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
> error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
> @@ -1250,7 +1252,9 @@ int freeze_secondary_cpus(int primary)
> cpumask_set_cpu(cpu, frozen_cpus);
> else {
> pr_err("Error taking CPU%d down: %d\n", cpu, error);
> - break;
> + /* When rebooting, offline as many CPUs as possible. */
> + if (!reboot)
> + break;
> }
> }
>
> diff --git a/kernel/reboot.c b/kernel/reboot.c
> index c4d472b7f1b4..12f643b66e57 100644
> --- a/kernel/reboot.c
> +++ b/kernel/reboot.c
> @@ -7,6 +7,7 @@
>
> #define pr_fmt(fmt) "reboot: " fmt
>
> +#include <linux/cpu.h>
> #include <linux/ctype.h>
> #include <linux/export.h>
> #include <linux/kexec.h>
> @@ -220,7 +221,9 @@ void migrate_to_reboot_cpu(void)
> /* The boot cpu is always logical cpu 0 */
> int cpu = reboot_cpu;
>
> +#if !IS_ENABLED(CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT)
> cpu_hotplug_disable();
> +#endif
>
> /* Make certain the cpu I'm about to reboot on is online */
> if (!cpu_online(cpu))
> @@ -231,6 +234,11 @@ void migrate_to_reboot_cpu(void)
>
> /* Make certain I only run on the appropriate processor */
> set_cpus_allowed_ptr(current, cpumask_of(cpu));
> +
> +#if IS_ENABLED(CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT)
> + /* Offline other cpus if possible */
> + freeze_secondary_cpus(cpu, true);
> +#endif
> }
>
> /**
> --
> 2.25.0.rc1.283.g88dfdc4193-goog
>