Re: [PATCH v5] KVM: RISC-V: Add CPU PM notifier for non-retention idle states

[Anup Patel]

On Mon, Aug 10, 2026 at 3:49 PM Yong-Xuan Wang <yongxuan.wang@xxxxxxxxxx> wrote:
>
> Register a CPU_PM notifier to restore hypervisor CSR content during CPU
> non-retention idle states. When a CPU enters a deep idle state that
> powers off the CPU domain, hypervisor CSRs and VS CSRs lose their state
> and must be saved before entry and restored after exit.
>
> This completes KVM's power management coverage for RISC-V:
> - CPU hotplug: handled by kvm_online_cpu/kvm_offline_cpu (cpuhp callbacks)
> - System suspend: handled by kvm_suspend/kvm_resume (syscore ops)
> - CPU idle (retention): no action needed, CSRs are retained
> - CPU idle (non-retention): handled by this CPU_PM notifier
>
> Signed-off-by: Yong-Xuan Wang <yongxuan.wang@xxxxxxxxxx>

LGTM. I have renamed "saved_value" to "saved_hgeie" at the
time of merging.

Reviewed-by: Anup Patel <anup@xxxxxxxxxxxxxx>

Queued this patch for Linux-7.3

Thanks,
Anup

> ---
> Changes in v5:
> - Remove patch1 as it was merged.
> - Add per-CPU virtualization state tracking (sashiko)
> - Rename CSR helper functions (Anup)
> - Introduce dedicated PM functions for AIA (Anup)
> - Fix module exit sequence (Anup)
> - Link to v4: https://patch.msgid.link/20260721-kvm-cpu-pm-v4-0-146bf942547d@xxxxxxxxxx
>
> Changes in v4:
> - Remove the system_state checking (sashiko)
> - Link to v3: https://patch.msgid.link/20260626-kvm-cpu-pm-v3-0-be051aafe9ba@xxxxxxxxxx
>
> Changes in v3:
> - Remove the error kvm_riscv_nacl_disable() call in kvm_riscv_csr_disable()
> (sashiko)
> - Unregister CPU PM notifier first in riscv_kvm_exit() (sashiko)
> - Link to v2: https://patch.msgid.link/20260626-kvm-cpu-pm-v2-0-478e5ef8dc9b@xxxxxxxxxx
>
> Changes in v2:
> - Add patch 1 force restore VCPU after power state transitions (sashiko)
> - Added CPU_PM_ENTER_FAILED event handling (sashiko)
> - Optimized HGEIE save/restore with saved_value field (sashiko)
> - Fixed HIE CSR restoration in lightweight idle resume path (sashiko)
> - Link to v1: https://patch.msgid.link/20260624-kvm-cpu-pm-v1-1-52088e127a55@xxxxxxxxxx
> ---
> To: Anup Patel <anup@xxxxxxxxxxxxxx>
> To: Atish Patra <atish.patra@xxxxxxxxx>
> To: Paul Walmsley <pjw@xxxxxxxxxx>
> To: Palmer Dabbelt <palmer@xxxxxxxxxxx>
> To: Albert Ou <aou@xxxxxxxxxxxxxxxxx>
> To: Alexandre Ghiti <alex@xxxxxxxx>
> To: Radim Krčmář <radim.krcmar@xxxxxxxxxxxxxxxx>
> To: Andrew Jones <andrew.jones@xxxxxxxxxxxxxxxx>
> To: Nutty Liu <nutty.liu@xxxxxxxxxxx>
> To: Jinyu Tang <tjytimi@xxxxxxx>
> Cc: greentime.hu@xxxxxxxxxx
> Cc: vincent.chen@xxxxxxxxxx
> Cc: zong.li@xxxxxxxxxx
> Cc: kvm@xxxxxxxxxxxxxxx
> Cc: kvm-riscv@xxxxxxxxxxxxxxxxxxx
> Cc: linux-riscv@xxxxxxxxxxxxxxxxxxx
> Cc: linux-kernel@xxxxxxxxxxxxxxx
> ---
>  arch/riscv/include/asm/kvm_aia.h |   3 ++
>  arch/riscv/kvm/aia.c             |  42 ++++++++++++++++
>  arch/riscv/kvm/main.c            | 106 ++++++++++++++++++++++++++++++++-------
>  3 files changed, 133 insertions(+), 18 deletions(-)
>
> diff --git a/arch/riscv/include/asm/kvm_aia.h b/arch/riscv/include/asm/kvm_aia.h
> index c67ec5ac0a14..ab03f853e507 100644
> --- a/arch/riscv/include/asm/kvm_aia.h
> +++ b/arch/riscv/include/asm/kvm_aia.h
> @@ -165,6 +165,9 @@ int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner,
>                              void __iomem **hgei_va, phys_addr_t *hgei_pa);
>  void kvm_riscv_aia_free_hgei(int cpu, int hgei);
>
> +void kvm_riscv_aia_pm_exit(void);
> +void kvm_riscv_aia_pm_enter(void);
> +
>  void kvm_riscv_aia_enable(void);
>  void kvm_riscv_aia_disable(void);
>  int kvm_riscv_aia_init(void);
> diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c
> index 9a653b4ad40a..0de9b8852ed2 100644
> --- a/arch/riscv/kvm/aia.c
> +++ b/arch/riscv/kvm/aia.c
> @@ -25,6 +25,7 @@ struct aia_hgei_control {
>         unsigned long free_bitmap;
>         struct kvm_vcpu *owners[BITS_PER_LONG];
>         unsigned int nr_hgei;
> +       unsigned long saved_value;
>  };
>  static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei);
>  static int hgei_parent_irq;
> @@ -553,6 +554,47 @@ static void aia_hgei_exit(void)
>         free_percpu_irq(hgei_parent_irq, &aia_hgei);
>  }
>
> +void kvm_riscv_aia_pm_exit(void)
> +{
> +       struct aia_hgei_control *hgctrl;
> +
> +       if (!kvm_riscv_aia_available())
> +               return;
> +
> +       hgctrl = this_cpu_ptr(&aia_hgei);
> +       csr_write(CSR_HGEIE, hgctrl->saved_value);
> +
> +       csr_write(CSR_HVICTL, aia_hvictl_value(false));
> +       csr_write(CSR_HVIPRIO1, 0x0);
> +       csr_write(CSR_HVIPRIO2, 0x0);
> +#ifdef CONFIG_32BIT
> +       csr_write(CSR_HVIPH, 0x0);
> +       csr_write(CSR_HIDELEGH, 0x0);
> +       csr_write(CSR_HVIPRIO1H, 0x0);
> +       csr_write(CSR_HVIPRIO2H, 0x0);
> +#endif
> +       csr_set(CSR_HIE, BIT(IRQ_S_GEXT));
> +       /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */
> +       if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF))
> +               csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF));
> +}
> +
> +void kvm_riscv_aia_pm_enter(void)
> +{
> +       struct aia_hgei_control *hgctrl;
> +
> +       if (!kvm_riscv_aia_available())
> +               return;
> +
> +       if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF))
> +               csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF));
> +
> +       csr_write(CSR_HVICTL, aia_hvictl_value(false));
> +
> +       hgctrl = this_cpu_ptr(&aia_hgei);
> +       hgctrl->saved_value = csr_read(CSR_HGEIE);
> +}
> +
>  void kvm_riscv_aia_enable(void)
>  {
>         const struct imsic_global_config *gc;
> diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c
> index 350e4f097d6e..20b516e84e60 100644
> --- a/arch/riscv/kvm/main.c
> +++ b/arch/riscv/kvm/main.c
> @@ -10,11 +10,14 @@
>  #include <linux/err.h>
>  #include <linux/module.h>
>  #include <linux/kvm_host.h>
> +#include <linux/cpu_pm.h>
>  #include <asm/cpufeature.h>
>  #include <asm/kvm_mmu.h>
>  #include <asm/kvm_nacl.h>
>  #include <asm/sbi.h>
>
> +static DEFINE_PER_CPU(bool, kvm_riscv_virtualization_enabled);
> +
>  DEFINE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa);
>
>  static void kvm_riscv_setup_vendor_features(void)
> @@ -33,14 +36,9 @@ long kvm_arch_dev_ioctl(struct file *filp,
>         return -EINVAL;
>  }
>
> -int kvm_arch_enable_virtualization_cpu(void)
> +/* Initialize hypervisor CSRs - called during CPU online and non-retention idle resume */
> +static void kvm_riscv_csr_init(void)
>  {
> -       int rc;
> -
> -       rc = kvm_riscv_nacl_enable();
> -       if (rc)
> -               return rc;
> -
>         csr_write(CSR_HEDELEG, 0);
>         csr_write(CSR_HIDELEG, 0);
>
> @@ -48,16 +46,11 @@ int kvm_arch_enable_virtualization_cpu(void)
>         csr_write(CSR_HCOUNTEREN, 0x02);
>
>         csr_write(CSR_HVIP, 0);
> -
> -       kvm_riscv_aia_enable();
> -
> -       return 0;
>  }
>
> -void kvm_arch_disable_virtualization_cpu(void)
> +/* Clear hypervisor CSRs - called during CPU offline and non-retention idle entry */
> +static void kvm_riscv_csr_cleanup(void)
>  {
> -       kvm_riscv_aia_disable();
> -
>         /*
>          * After clearing the hideleg CSR, the host kernel will receive
>          * spurious interrupts if hvip CSR has pending interrupts and the
> @@ -70,10 +63,69 @@ void kvm_arch_disable_virtualization_cpu(void)
>         csr_write(CSR_HIDELEG, 0);
>
>         kvm_riscv_clear_former_vcpu();
> +}
> +
> +int kvm_arch_enable_virtualization_cpu(void)
> +{
> +       int rc;
> +
> +       rc = kvm_riscv_nacl_enable();
> +       if (rc)
> +               return rc;
> +
> +       kvm_riscv_csr_init();
> +       kvm_riscv_aia_enable();
> +
> +       __this_cpu_write(kvm_riscv_virtualization_enabled, true);
>
> +       return 0;
> +}
> +
> +void kvm_arch_disable_virtualization_cpu(void)
> +{
> +       kvm_riscv_aia_disable();
> +       kvm_riscv_csr_cleanup();
>         kvm_riscv_nacl_disable();
> +
> +       __this_cpu_write(kvm_riscv_virtualization_enabled, false);
> +}
> +
> +static int kvm_riscv_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v)
> +{
> +       switch (cmd) {
> +       case CPU_PM_EXIT:
> +       case CPU_PM_ENTER_FAILED:
> +               /*
> +                * Only restore hypervisor state if KVM virtualization is
> +                * enabled on this CPU. This prevents unintentional re-enabling
> +                * of virtualization after it has been explicitly disabled.
> +                */
> +               if (__this_cpu_read(kvm_riscv_virtualization_enabled)) {
> +                       kvm_riscv_csr_init();
> +                       kvm_riscv_aia_pm_exit();
> +               }
> +               return NOTIFY_OK;
> +       case CPU_PM_ENTER:
> +               /*
> +                * Only save and clear hypervisor state if KVM virtualization
> +                * is enabled on this CPU.
> +                */
> +               if (__this_cpu_read(kvm_riscv_virtualization_enabled)) {
> +                       kvm_riscv_aia_pm_enter();
> +                       kvm_riscv_csr_cleanup();
> +               }
> +               return NOTIFY_OK;
> +       default:
> +               break;
> +       }
> +
> +       return NOTIFY_DONE;
>  }
>
> +static struct notifier_block kvm_riscv_cpu_pm_nb = {
> +       .notifier_call = kvm_riscv_cpu_pm_notifier,
> +};
> +
>  static void kvm_riscv_teardown(void)
>  {
>         kvm_riscv_aia_exit();
> @@ -174,17 +226,31 @@ static int __init riscv_kvm_init(void)
>
>         kvm_register_perf_callbacks();
>
> -       rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
> -       if (rc) {
> -               kvm_riscv_teardown();
> -               return rc;
> +       /* Register CPU PM notifier for CPU idle non-retention states */
> +       if (IS_ENABLED(CONFIG_CPU_PM)) {
> +               rc = cpu_pm_register_notifier(&kvm_riscv_cpu_pm_nb);
> +               if (rc) {
> +                       kvm_err("Failed to register CPU PM notifier: %d\n", rc);
> +                       goto err_teardown;
> +               }
>         }
>
> +       rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
> +       if (rc)
> +               goto err_unregister_cpu_pm;
> +
>         if (kvm_riscv_aia_available())
>                 kvm_info("AIA available with %d guest external interrupts\n",
>                          atomic_read(&kvm_riscv_aia_nr_hgei));
>
>         return 0;
> +
> +err_unregister_cpu_pm:
> +       if (IS_ENABLED(CONFIG_CPU_PM))
> +               cpu_pm_unregister_notifier(&kvm_riscv_cpu_pm_nb);
> +err_teardown:
> +       kvm_riscv_teardown();
> +       return rc;
>  }
>  module_init(riscv_kvm_init);
>
> @@ -192,6 +258,10 @@ static void __exit riscv_kvm_exit(void)
>  {
>         kvm_exit();
>
> +       /* Unregister CPU PM notifier */
> +       if (IS_ENABLED(CONFIG_CPU_PM))
> +               cpu_pm_unregister_notifier(&kvm_riscv_cpu_pm_nb);
> +
>         kvm_riscv_teardown();
>  }
>  module_exit(riscv_kvm_exit);
>
> ---
> base-commit: dfdf1374fdeccb5b7e3d35186228e01ec5ea5f01
> change-id: 20260624-kvm-cpu-pm-94141aecd5fa
>