Re: [PATCH RFC v3 1/3] reboot: support hotplug CPUs before reboot

[Vitaly Kuznetsov]

Hsin-Yi Wang <hsinyi@xxxxxxxxxxxx> writes:

> Currently system reboots uses architecture specific codes (smp_send_stop)
> to offline non reboot CPUs. Most architecture's implementation is looping
> through all non reboot online CPUs and call ipi function to each of them. Some
> architecture like arm64, arm, and x86... would set offline masks to cpu without
> really offline them. This causes some race condition and kernel warning comes
> out sometimes when system reboots.
>
> This patch adds a config REBOOT_HOTPLUG_CPU, which would hotplug cpus in
> migrate_to_reboot_cpu(). If non reboot cpus are all offlined here, the loop for
> checking online cpus would be an empty loop. If architecture don't enable this
> config, or some cpus somehow fails to offline, it would fallback to ipi
> function.
>
> Signed-off-by: Hsin-Yi Wang <hsinyi@xxxxxxxxxxxx>
> ---
>  arch/Kconfig        |  6 ++++++
>  include/linux/cpu.h |  3 +++
>  kernel/cpu.c        | 19 +++++++++++++++++++
>  kernel/reboot.c     |  8 ++++++++
>  4 files changed, 36 insertions(+)
>
> diff --git a/arch/Kconfig b/arch/Kconfig
> index 48b5e103bdb0..a043b9be1499 100644
> --- a/arch/Kconfig
> +++ b/arch/Kconfig
> @@ -52,6 +52,12 @@ config OPROFILE_EVENT_MULTIPLEX
>  
>  	  If unsure, say N.
>  
> +config REBOOT_HOTPLUG_CPU
> +	bool "Support for hotplug CPUs before reboot"
> +	depends on HOTPLUG_CPU
> +	help
> +	  Say Y to do a full hotplug on secondary CPUs before reboot.

I'm not sure this should be a configurable option, e.g. in case this is
a good approach in general, why not just use CONFIG_HOTPLUG_CPU in the
code? 

> +
>  config HAVE_OPROFILE
>  	bool
>  
> diff --git a/include/linux/cpu.h b/include/linux/cpu.h
> index 1ca2baf817ed..3bf5ab289954 100644
> --- a/include/linux/cpu.h
> +++ b/include/linux/cpu.h
> @@ -118,6 +118,9 @@ extern void cpu_hotplug_disable(void);
>  extern void cpu_hotplug_enable(void);
>  void clear_tasks_mm_cpumask(int cpu);
>  int cpu_down(unsigned int cpu);
> +#if IS_ENABLED(CONFIG_REBOOT_HOTPLUG_CPU)
> +extern void offline_secondary_cpus(int primary);
> +#endif
>  
>  #else /* CONFIG_HOTPLUG_CPU */
>  
> diff --git a/kernel/cpu.c b/kernel/cpu.c
> index 9c706af713fb..52afc47dd56a 100644
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -1057,6 +1057,25 @@ int cpu_down(unsigned int cpu)
>  }
>  EXPORT_SYMBOL(cpu_down);
>  
> +#if IS_ENABLED(CONFIG_REBOOT_HOTPLUG_CPU)
> +void offline_secondary_cpus(int primary)
> +{
> +	int i, err;
> +
> +	cpu_maps_update_begin();
> +
> +	for_each_online_cpu(i) {
> +		if (i == primary)
> +			continue;
> +		err = _cpu_down(i, 0, CPUHP_OFFLINE);
> +		if (err)
> +			pr_warn("Failed to offline cpu %d\n", i);
> +	}
> +	cpu_hotplug_disabled++;
> +
> +	cpu_maps_update_done();
> +}
> +#endif

This looks like a simplified version of freeze_secondary_cpus(), can
they be merged?


>  #else
>  #define takedown_cpu		NULL
>  #endif /*CONFIG_HOTPLUG_CPU*/
> diff --git a/kernel/reboot.c b/kernel/reboot.c
> index c4d472b7f1b4..fda84794ce46 100644
> --- a/kernel/reboot.c
> +++ b/kernel/reboot.c
> @@ -7,6 +7,7 @@
>  
>  #define pr_fmt(fmt)	"reboot: " fmt
>  
> +#include <linux/cpu.h>
>  #include <linux/ctype.h>
>  #include <linux/export.h>
>  #include <linux/kexec.h>
> @@ -220,7 +221,9 @@ void migrate_to_reboot_cpu(void)
>  	/* The boot cpu is always logical cpu 0 */
>  	int cpu = reboot_cpu;
>  
> +#if !IS_ENABLED(CONFIG_REBOOT_HOTPLUG_CPU)
>  	cpu_hotplug_disable();
> +#endif
>  
>  	/* Make certain the cpu I'm about to reboot on is online */
>  	if (!cpu_online(cpu))
> @@ -231,6 +234,11 @@ void migrate_to_reboot_cpu(void)
>  
>  	/* Make certain I only run on the appropriate processor */
>  	set_cpus_allowed_ptr(current, cpumask_of(cpu));
> +
> +	/* Hotplug other cpus if possible */
> +#if IS_ENABLED(CONFIG_REBOOT_HOTPLUG_CPU)
> +	offline_secondary_cpus(cpu);
> +#endif
>  }
>  
>  /**

-- 
Vitaly