[RFC PATCH 2/3] RISC-V: KVM: Preserve guest Vector state across vCPU put/load
From: slack up
Date: Tue Oct 27 2026 - 09:53:06 EST
From: Xiaochun Yu <yxcnew@xxxxxxxxxxx>
kvm_arch_vcpu_put() saves the guest Vector state and restores the host
register file. A later load on the same hart saves the host again and
requests a full guest restore, even when the host has not used Vector in
between.
Track a per-hart owner with an opaque vCPU identity, an allocation cookie
and a state generation. After saving a dirty guest to memory, leave its
data registers resident when the host has no Vector restore obligation. A
matching load restores only the Vector control CSRs. All mismatches and
generic Vector users fall back to a full restore.
Keep guest memory authoritative by continuing to save at every dirty put.
Advance the generation on saves, resets and SET_ONE_REG. Do so before
copy_from_user() because a fault can leave a modified prefix in the
destination even though the ioctl fails.
Register the generic invalidation hook only when the read-only
vector_owner_fastpath module parameter is enabled. The parameter defaults
to false so the existing full-restore behavior remains the default. Add
tracepoints for restore mode, owner decisions and invalidation reasons.
Development note: OpenAI Codex assisted with source analysis,
implementation drafting, test orchestration and changelog preparation.
Signed-off-by: Xiaochun Yu <yxcnew@xxxxxxxxxxx>
---
arch/riscv/include/asm/kvm_host.h | 4 +
arch/riscv/include/asm/kvm_vcpu_vector.h | 54 ++++-
arch/riscv/include/asm/processor.h | 1 +
arch/riscv/kvm/trace.h | 98 +++++++++
arch/riscv/kvm/vcpu.c | 54 ++++-
arch/riscv/kvm/vcpu_vector.c | 252 ++++++++++++++++++++++-
6 files changed, 439 insertions(+), 24 deletions(-)
diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h
index e28f058a94d7..a6e2b3097644 100644
--- a/arch/riscv/include/asm/kvm_host.h
+++ b/arch/riscv/include/asm/kvm_host.h
@@ -204,6 +204,10 @@ struct kvm_vcpu_arch {
/* CPU context of Guest VCPU */
struct kvm_cpu_context guest_context;
+ /* Identity and version used by Vector owner tracking. */
+ u64 vector_owner_cookie;
+ u64 vector_state_generation;
+
/* CPU CSR context of Guest VCPU */
struct kvm_vcpu_csr guest_csr;
diff --git a/arch/riscv/include/asm/kvm_vcpu_vector.h b/arch/riscv/include/asm/kvm_vcpu_vector.h
index ebc619ed67dd..7f3eecf4217b 100644
--- a/arch/riscv/include/asm/kvm_vcpu_vector.h
+++ b/arch/riscv/include/asm/kvm_vcpu_vector.h
@@ -26,29 +26,47 @@ static __always_inline void __kvm_riscv_vector_restore(struct kvm_cpu_context *c
__riscv_v_vstate_restore(&context->vector, context->vector.datap);
}
+static __always_inline void __kvm_riscv_vector_restore_csrs(struct kvm_cpu_context *context)
+{
+ riscv_v_enable();
+ __vstate_csr_restore(&context->vector);
+ riscv_v_disable();
+}
+
void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu);
void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx,
unsigned long *isa);
void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx,
- unsigned long *isa);
+ unsigned long *isa, bool csr_only);
void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx);
-void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx);
+void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx,
+ bool preserve_guest);
int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu);
void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu);
void kvm_riscv_register_vctx_callback(void (*func)(void));
void kvm_riscv_unregister_vctx_callback(void);
+bool kvm_riscv_vcpu_vector_owner_enabled(void);
+bool kvm_riscv_vcpu_vector_owner_load(struct kvm_vcpu *vcpu, int cpu);
+bool kvm_riscv_vcpu_vector_owner_put(struct kvm_vcpu *vcpu);
+void kvm_riscv_vcpu_vector_owner_invalidate(enum riscv_v_invalidate_reason reason);
void kvm_riscv_vcpu_flush_vector(void);
static inline void kvm_riscv_v_init(void)
{
- if (has_vector())
+ if (has_vector()) {
kvm_riscv_register_vctx_callback(&kvm_riscv_vcpu_flush_vector);
+ if (kvm_riscv_vcpu_vector_owner_enabled())
+ kvm_riscv_register_vctx_invalidate(&kvm_riscv_vcpu_vector_owner_invalidate);
+ }
}
static inline void kvm_riscv_v_exit(void)
{
- if (has_vector())
+ if (has_vector()) {
+ if (kvm_riscv_vcpu_vector_owner_enabled())
+ kvm_riscv_unregister_vctx_invalidate();
kvm_riscv_unregister_vctx_callback();
+ }
}
#else
@@ -59,6 +77,10 @@ static inline void __kvm_riscv_vector_restore(struct kvm_cpu_context *context)
{
}
+static inline void __kvm_riscv_vector_restore_csrs(struct kvm_cpu_context *context)
+{
+}
+
static inline void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu)
{
}
@@ -69,7 +91,7 @@ static inline void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx
}
static inline void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx,
- unsigned long *isa)
+ unsigned long *isa, bool csr_only)
{
}
@@ -77,7 +99,8 @@ static inline void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx)
{
}
-static inline void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx)
+static inline void
+kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx, bool preserve_guest)
{
}
@@ -90,6 +113,25 @@ static inline void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu)
{
}
+static inline bool kvm_riscv_vcpu_vector_owner_enabled(void)
+{
+ return false;
+}
+
+static inline bool kvm_riscv_vcpu_vector_owner_load(struct kvm_vcpu *vcpu, int cpu)
+{
+ return false;
+}
+
+static inline bool kvm_riscv_vcpu_vector_owner_put(struct kvm_vcpu *vcpu)
+{
+ return false;
+}
+
+static inline void kvm_riscv_vcpu_vector_owner_invalidate(enum riscv_v_invalidate_reason reason)
+{
+}
+
static inline void kvm_riscv_v_init(void)
{
}
diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index 815715c67f94..abda84ca9548 100644
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -105,6 +105,7 @@ struct pt_regs;
#define RISCV_KERNEL_MODE_V 0x00000001
#define RISCV_V_VCPU_NEED_RESTORE 0x00000002
#define RISCV_V_VCPU_CTX 0x00000004
+#define RISCV_V_VCPU_CSR_RESTORE 0x00000008
#define RISCV_PREEMPT_V 0x00000100
#define RISCV_PREEMPT_V_DIRTY 0x80000000
#define RISCV_PREEMPT_V_NEED_RESTORE 0x40000000
diff --git a/arch/riscv/kvm/trace.h b/arch/riscv/kvm/trace.h
index 23a5fd1b686e..c805606c704d 100644
--- a/arch/riscv/kvm/trace.h
+++ b/arch/riscv/kvm/trace.h
@@ -133,6 +133,104 @@ TRACE_EVENT(kvm_vcpu_irq,
__entry->vcpu_id, __entry->irq, __entry->level)
);
+TRACE_EVENT(kvm_riscv_vector_restore_request,
+ TP_PROTO(unsigned long vcpu_id, int cpu, unsigned int source,
+ unsigned long host_vs),
+ TP_ARGS(vcpu_id, cpu, source, host_vs),
+
+ TP_STRUCT__entry(
+ __field(unsigned long, vcpu_id)
+ __field(int, cpu)
+ __field(unsigned int, source)
+ __field(unsigned long, host_vs)
+ ),
+
+ TP_fast_assign(
+ __entry->vcpu_id = vcpu_id;
+ __entry->cpu = cpu;
+ __entry->source = source;
+ __entry->host_vs = host_vs;
+ ),
+
+ TP_printk("VCPU: %lu, CPU: %d, source: %s, host VS: 0x%lx",
+ __entry->vcpu_id, __entry->cpu,
+ __print_symbolic(__entry->source,
+ { 0, "load" }, { 1, "kernel_vector" }),
+ __entry->host_vs)
+);
+
+TRACE_EVENT(kvm_riscv_vector_restore,
+ TP_PROTO(unsigned long vcpu_id, int cpu, bool csr_only),
+ TP_ARGS(vcpu_id, cpu, csr_only),
+
+ TP_STRUCT__entry(
+ __field(unsigned long, vcpu_id)
+ __field(int, cpu)
+ __field(bool, csr_only)
+ ),
+
+ TP_fast_assign(
+ __entry->vcpu_id = vcpu_id;
+ __entry->cpu = cpu;
+ __entry->csr_only = csr_only;
+ ),
+
+ TP_printk("VCPU: %lu, CPU: %d, mode: %s",
+ __entry->vcpu_id, __entry->cpu,
+ __entry->csr_only ? "csr" : "full")
+);
+
+TRACE_EVENT(kvm_riscv_vector_owner,
+ TP_PROTO(unsigned long vcpu_id, int cpu, unsigned int action,
+ bool eligible, unsigned int reason, unsigned long host_vs,
+ unsigned long guest_vs, u64 generation),
+ TP_ARGS(vcpu_id, cpu, action, eligible, reason, host_vs, guest_vs,
+ generation),
+
+ TP_STRUCT__entry(
+ __field(unsigned long, vcpu_id)
+ __field(int, cpu)
+ __field(unsigned int, action)
+ __field(bool, eligible)
+ __field(unsigned int, reason)
+ __field(unsigned long, host_vs)
+ __field(unsigned long, guest_vs)
+ __field(u64, generation)
+ ),
+
+ TP_fast_assign(
+ __entry->vcpu_id = vcpu_id;
+ __entry->cpu = cpu;
+ __entry->action = action;
+ __entry->eligible = eligible;
+ __entry->reason = reason;
+ __entry->host_vs = host_vs;
+ __entry->guest_vs = guest_vs;
+ __entry->generation = generation;
+ ),
+
+ TP_printk("VCPU: %lu, CPU: %d, action: %s, eligible: %d, reason: %s, host VS: 0x%lx, guest VS: 0x%lx, generation: %llu",
+ __entry->vcpu_id, __entry->cpu,
+ __print_symbolic(__entry->action,
+ { 0, "load" }, { 1, "put" },
+ { 2, "invalidate" }),
+ __entry->eligible,
+ __print_symbolic(__entry->reason,
+ { 0, "hit" }, { 1, "empty" },
+ { 2, "other_vcpu" }, { 3, "cookie" },
+ { 4, "generation" }, { 5, "guest_off" },
+ { 6, "host_on" }, { 7, "migration" },
+ { 8, "put_candidate" },
+ { 9, "task_save" },
+ { 10, "task_restore" },
+ { 11, "task_discard" },
+ { 12, "kernel_vector" },
+ { 13, "cpu_state" },
+ { 14, "not_resident" }),
+ __entry->host_vs, __entry->guest_vs,
+ __entry->generation)
+);
+
#endif /* _TRACE_KVM_H */
#undef TRACE_INCLUDE_PATH
diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index 7f1636c44d41..a59cac26ca7c 100644
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -35,6 +35,7 @@ void kvm_riscv_clear_former_vcpu(void)
* via the fast-path optimization.
*/
__this_cpu_write(kvm_former_vcpu, NULL);
+ kvm_riscv_vcpu_vector_owner_invalidate(RISCV_V_INVALIDATE_CPU_STATE);
}
const struct kvm_stats_desc kvm_vcpu_stats_desc[] = {
@@ -589,6 +590,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
void *nsh;
struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
+ bool vector_owner_hit;
/*
* If VCPU is being reloaded on the same physical CPU and no
@@ -647,9 +649,11 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context,
vcpu->arch.isa);
get_cpu_vector_context();
- kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context);
+ vector_owner_hit = kvm_riscv_vcpu_vector_owner_load(vcpu, cpu);
+ if (!vector_owner_hit)
+ kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context);
kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context,
- vcpu->arch.isa);
+ vcpu->arch.isa, vector_owner_hit);
put_cpu_vector_context();
kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
@@ -661,6 +665,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
void *nsh;
struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
+ bool preserve_guest;
vcpu->cpu = -1;
@@ -674,7 +679,9 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
get_cpu_vector_context();
kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context,
vcpu->arch.isa);
- kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context);
+ preserve_guest = kvm_riscv_vcpu_vector_owner_put(vcpu);
+ kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context,
+ preserve_guest);
put_cpu_vector_context();
if (kvm_riscv_nacl_available()) {
@@ -793,6 +800,12 @@ static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *v
zicficsr->ssp = csr_swap(CSR_SSP, 0);
}
+enum kvm_riscv_vector_restore_type {
+ KVM_RISCV_VECTOR_RESTORE_NONE,
+ KVM_RISCV_VECTOR_RESTORE_FULL,
+ KVM_RISCV_VECTOR_RESTORE_CSR,
+};
+
/*
* Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
* the vCPU is running.
@@ -800,12 +813,14 @@ static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *v
* This must be noinstr as instrumentation may make use of RCU, and this is not
* safe during the EQS.
*/
-static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu,
- struct kvm_cpu_trap *trap)
+static int noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu,
+ struct kvm_cpu_trap *trap)
{
void *nsh;
struct kvm_cpu_context *gcntx = &vcpu->arch.guest_context;
struct kvm_cpu_context *hcntx = &vcpu->arch.host_context;
+ int vector_restore = KVM_RISCV_VECTOR_RESTORE_NONE;
+ u32 vector_flags;
/*
* We save trap CSRs (such as SEPC, SCAUSE, STVAL, HTVAL, and
@@ -818,11 +833,21 @@ static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu,
kvm_riscv_vcpu_swap_in_guest_state(vcpu);
guest_state_enter_irqoff();
- /* sstatus.VS != SR_VS_OFF is guaranteed when NEED_RESTORE is set */
- if (current->thread.riscv_v_flags & RISCV_V_VCPU_NEED_RESTORE) {
- current->thread.riscv_v_flags &= ~RISCV_V_VCPU_NEED_RESTORE;
+ vector_flags = current->thread.riscv_v_flags;
+ /* sstatus.VS != SR_VS_OFF is guaranteed for either restore request. */
+ if (vector_flags & (RISCV_V_VCPU_NEED_RESTORE |
+ RISCV_V_VCPU_CSR_RESTORE)) {
+ current->thread.riscv_v_flags &=
+ ~(RISCV_V_VCPU_NEED_RESTORE |
+ RISCV_V_VCPU_CSR_RESTORE);
current->thread.riscv_v_flags |= RISCV_V_VCPU_CTX;
- __kvm_riscv_vector_restore(gcntx);
+ if (vector_flags & RISCV_V_VCPU_NEED_RESTORE) {
+ __kvm_riscv_vector_restore(gcntx);
+ vector_restore = KVM_RISCV_VECTOR_RESTORE_FULL;
+ } else {
+ __kvm_riscv_vector_restore_csrs(gcntx);
+ vector_restore = KVM_RISCV_VECTOR_RESTORE_CSR;
+ }
gcntx->sstatus = (gcntx->sstatus & ~SR_VS) | SR_VS_CLEAN;
}
@@ -886,6 +911,8 @@ static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu,
vcpu->arch.last_exit_cpu = vcpu->cpu;
guest_state_exit_irqoff();
kvm_riscv_vcpu_swap_in_host_state(vcpu);
+
+ return vector_restore;
}
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
@@ -893,6 +920,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
int ret;
struct kvm_cpu_trap trap;
struct kvm_run *run = vcpu->run;
+ int vector_restore;
if (!vcpu->arch.ran_atleast_once)
kvm_riscv_vcpu_config_ran_once(vcpu);
@@ -1001,7 +1029,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
guest_timing_enter_irqoff();
- kvm_riscv_vcpu_enter_exit(vcpu, &trap);
+ vector_restore = kvm_riscv_vcpu_enter_exit(vcpu, &trap);
vcpu->mode = OUTSIDE_GUEST_MODE;
vcpu->stat.exits++;
@@ -1026,6 +1054,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
local_irq_enable();
+ if (vector_restore == KVM_RISCV_VECTOR_RESTORE_FULL)
+ trace_kvm_riscv_vector_restore(vcpu->vcpu_id,
+ smp_processor_id(), false);
+ else if (vector_restore == KVM_RISCV_VECTOR_RESTORE_CSR)
+ trace_kvm_riscv_vector_restore(vcpu->vcpu_id,
+ smp_processor_id(), true);
trace_kvm_exit(&trap);
preempt_enable();
diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c
index fefc2857f753..c197d3d7b539 100644
--- a/arch/riscv/kvm/vcpu_vector.c
+++ b/arch/riscv/kvm/vcpu_vector.c
@@ -7,9 +7,11 @@
* Greentime Hu <greentime.hu@xxxxxxxxxx>
*/
+#include <linux/atomic.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/kvm_host.h>
+#include <linux/moduleparam.h>
#include <linux/nospec.h>
#include <linux/uaccess.h>
#include <asm/cpufeature.h>
@@ -17,7 +19,50 @@
#include <asm/kvm_vcpu_vector.h>
#include <asm/vector.h>
+#include "trace.h"
+
#ifdef CONFIG_RISCV_ISA_V
+enum kvm_riscv_vector_owner_reason {
+ KVM_RISCV_VECTOR_OWNER_HIT,
+ KVM_RISCV_VECTOR_OWNER_EMPTY,
+ KVM_RISCV_VECTOR_OWNER_OTHER_VCPU,
+ KVM_RISCV_VECTOR_OWNER_COOKIE,
+ KVM_RISCV_VECTOR_OWNER_GENERATION,
+ KVM_RISCV_VECTOR_OWNER_GUEST_OFF,
+ KVM_RISCV_VECTOR_OWNER_HOST_ON,
+ KVM_RISCV_VECTOR_OWNER_MIGRATION,
+ KVM_RISCV_VECTOR_OWNER_PUT_CANDIDATE,
+ KVM_RISCV_VECTOR_OWNER_TASK_SAVE,
+ KVM_RISCV_VECTOR_OWNER_TASK_RESTORE,
+ KVM_RISCV_VECTOR_OWNER_TASK_DISCARD,
+ KVM_RISCV_VECTOR_OWNER_KERNEL_VECTOR,
+ KVM_RISCV_VECTOR_OWNER_CPU_STATE,
+ KVM_RISCV_VECTOR_OWNER_NOT_RESIDENT,
+};
+
+struct kvm_riscv_vector_owner {
+ /* Opaque comparison token; cookie rejects address reuse after destroy. */
+ unsigned long identity;
+ u64 cookie;
+ u64 generation;
+ unsigned long vcpu_id;
+ unsigned long guest_vs;
+};
+
+static DEFINE_PER_CPU(struct kvm_riscv_vector_owner,
+ kvm_riscv_vector_owner);
+static atomic64_t kvm_riscv_vector_owner_cookie = ATOMIC64_INIT(0);
+static bool kvm_riscv_vector_owner_fastpath;
+module_param_named(vector_owner_fastpath, kvm_riscv_vector_owner_fastpath,
+ bool, 0444);
+MODULE_PARM_DESC(vector_owner_fastpath,
+ "Preserve matched guest Vector registers across vCPU put/load");
+
+bool kvm_riscv_vcpu_vector_owner_enabled(void)
+{
+ return READ_ONCE(kvm_riscv_vector_owner_fastpath);
+}
+
void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu)
{
unsigned long *isa = vcpu->arch.isa;
@@ -34,6 +79,8 @@ void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu)
} else {
cntx->sstatus |= SR_VS_OFF;
}
+
+ vcpu->arch.vector_state_generation++;
}
static void kvm_riscv_vcpu_vector_clean(struct kvm_cpu_context *cntx)
@@ -46,18 +93,35 @@ void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx,
unsigned long *isa)
{
if ((cntx->sstatus & SR_VS) == SR_VS_DIRTY) {
- if (riscv_isa_extension_available(isa, V))
+ if (riscv_isa_extension_available(isa, V)) {
+ struct kvm_vcpu *vcpu = container_of(cntx, struct kvm_vcpu,
+ arch.guest_context);
+
__kvm_riscv_vector_save(cntx);
+ vcpu->arch.vector_state_generation++;
+ }
kvm_riscv_vcpu_vector_clean(cntx);
}
}
void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx,
- unsigned long *isa)
+ unsigned long *isa, bool csr_only)
{
if ((cntx->sstatus & SR_VS) != SR_VS_OFF) {
- if (riscv_isa_extension_available(isa, V))
- riscv_v_flags_set(riscv_v_flags() | RISCV_V_VCPU_NEED_RESTORE);
+ if (riscv_isa_extension_available(isa, V)) {
+ struct kvm_vcpu *vcpu = container_of(cntx, struct kvm_vcpu,
+ arch.guest_context);
+ unsigned long host_vs = task_pt_regs(current)->status & SR_VS;
+ u32 flags = riscv_v_flags();
+
+ flags &= ~(RISCV_V_VCPU_NEED_RESTORE |
+ RISCV_V_VCPU_CSR_RESTORE);
+ flags |= csr_only ? RISCV_V_VCPU_CSR_RESTORE :
+ RISCV_V_VCPU_NEED_RESTORE;
+ riscv_v_flags_set(flags);
+ trace_kvm_riscv_vector_restore_request(vcpu->vcpu_id,
+ smp_processor_id(), 0, host_vs);
+ }
}
}
@@ -68,11 +132,167 @@ void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx)
__kvm_riscv_vector_save(cntx);
}
-void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx)
+void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx,
+ bool preserve_guest)
{
- if (!kvm_riscv_isa_check_host(V))
+ if (!preserve_guest && !kvm_riscv_isa_check_host(V))
__kvm_riscv_vector_restore(cntx);
- riscv_v_flags_set(riscv_v_flags() & ~(RISCV_V_VCPU_CTX | RISCV_V_VCPU_NEED_RESTORE));
+ riscv_v_flags_set(riscv_v_flags() &
+ ~(RISCV_V_VCPU_CTX | RISCV_V_VCPU_NEED_RESTORE |
+ RISCV_V_VCPU_CSR_RESTORE));
+}
+
+bool kvm_riscv_vcpu_vector_owner_load(struct kvm_vcpu *vcpu, int cpu)
+{
+ struct kvm_riscv_vector_owner *owner;
+ unsigned long guest_vs = vcpu->arch.guest_context.sstatus & SR_VS;
+ unsigned long host_vs = task_pt_regs(current)->status & SR_VS;
+ enum kvm_riscv_vector_owner_reason reason;
+ bool eligible = false;
+
+ if (!kvm_riscv_vcpu_vector_owner_enabled())
+ return false;
+
+ owner = this_cpu_ptr(&kvm_riscv_vector_owner);
+ if (guest_vs == SR_VS_OFF) {
+ reason = KVM_RISCV_VECTOR_OWNER_GUEST_OFF;
+ } else if (host_vs != SR_VS_OFF) {
+ reason = KVM_RISCV_VECTOR_OWNER_HOST_ON;
+ } else if (!owner->identity) {
+ reason = vcpu->arch.last_exit_cpu == cpu ?
+ KVM_RISCV_VECTOR_OWNER_EMPTY :
+ KVM_RISCV_VECTOR_OWNER_MIGRATION;
+ } else if (owner->identity != (unsigned long)vcpu) {
+ reason = KVM_RISCV_VECTOR_OWNER_OTHER_VCPU;
+ } else if (owner->cookie != vcpu->arch.vector_owner_cookie) {
+ reason = KVM_RISCV_VECTOR_OWNER_COOKIE;
+ } else if (owner->generation != vcpu->arch.vector_state_generation) {
+ reason = KVM_RISCV_VECTOR_OWNER_GENERATION;
+ } else {
+ reason = KVM_RISCV_VECTOR_OWNER_HIT;
+ eligible = true;
+ }
+
+ trace_kvm_riscv_vector_owner(vcpu->vcpu_id, cpu, 0, eligible,
+ reason, host_vs, guest_vs,
+ vcpu->arch.vector_state_generation);
+
+ /* A load consumes the cached owner; the next put publishes it. */
+ owner->identity = 0;
+ owner->cookie = 0;
+ owner->generation = 0;
+ owner->vcpu_id = 0;
+ owner->guest_vs = 0;
+
+ return eligible;
+}
+
+bool kvm_riscv_vcpu_vector_owner_put(struct kvm_vcpu *vcpu)
+{
+ struct kvm_riscv_vector_owner *owner;
+ unsigned long guest_vs = vcpu->arch.guest_context.sstatus & SR_VS;
+ unsigned long host_vs = task_pt_regs(current)->status & SR_VS;
+ u32 flags = riscv_v_flags();
+ enum kvm_riscv_vector_owner_reason reason;
+ bool eligible;
+
+ if (!kvm_riscv_vcpu_vector_owner_enabled())
+ return false;
+
+ owner = this_cpu_ptr(&kvm_riscv_vector_owner);
+ eligible = guest_vs != SR_VS_OFF && host_vs == SR_VS_OFF &&
+ (flags & RISCV_V_VCPU_CTX) &&
+ !(flags & (RISCV_V_VCPU_NEED_RESTORE |
+ RISCV_V_VCPU_CSR_RESTORE));
+ if (eligible) {
+ owner->identity = (unsigned long)vcpu;
+ owner->cookie = vcpu->arch.vector_owner_cookie;
+ owner->generation = vcpu->arch.vector_state_generation;
+ owner->vcpu_id = vcpu->vcpu_id;
+ owner->guest_vs = guest_vs;
+ reason = KVM_RISCV_VECTOR_OWNER_PUT_CANDIDATE;
+ } else {
+ owner->identity = 0;
+ owner->cookie = 0;
+ owner->generation = 0;
+ owner->vcpu_id = 0;
+ owner->guest_vs = 0;
+ if (guest_vs == SR_VS_OFF)
+ reason = KVM_RISCV_VECTOR_OWNER_GUEST_OFF;
+ else if (host_vs != SR_VS_OFF)
+ reason = KVM_RISCV_VECTOR_OWNER_HOST_ON;
+ else
+ reason = KVM_RISCV_VECTOR_OWNER_NOT_RESIDENT;
+ }
+
+ trace_kvm_riscv_vector_owner(vcpu->vcpu_id, smp_processor_id(), 1,
+ eligible, reason, host_vs, guest_vs,
+ vcpu->arch.vector_state_generation);
+
+ return eligible;
+}
+
+void kvm_riscv_vcpu_vector_owner_invalidate(enum riscv_v_invalidate_reason reason)
+{
+ struct kvm_riscv_vector_owner *owner;
+ struct kvm_vcpu *vcpu;
+ enum kvm_riscv_vector_owner_reason trace_reason;
+ unsigned long host_vs;
+ u32 flags;
+
+ switch (reason) {
+ case RISCV_V_INVALIDATE_TASK_SAVE:
+ trace_reason = KVM_RISCV_VECTOR_OWNER_TASK_SAVE;
+ break;
+ case RISCV_V_INVALIDATE_TASK_RESTORE:
+ trace_reason = KVM_RISCV_VECTOR_OWNER_TASK_RESTORE;
+ break;
+ case RISCV_V_INVALIDATE_TASK_DISCARD:
+ trace_reason = KVM_RISCV_VECTOR_OWNER_TASK_DISCARD;
+ break;
+ case RISCV_V_INVALIDATE_KERNEL_VECTOR:
+ trace_reason = KVM_RISCV_VECTOR_OWNER_KERNEL_VECTOR;
+ break;
+ case RISCV_V_INVALIDATE_CPU_STATE:
+ trace_reason = KVM_RISCV_VECTOR_OWNER_CPU_STATE;
+ break;
+ default:
+ return;
+ }
+
+ preempt_disable();
+ flags = riscv_v_flags();
+ if (flags & RISCV_V_VCPU_CSR_RESTORE) {
+ vcpu = *this_cpu_ptr(kvm_get_running_vcpus());
+ if (!WARN_ON_ONCE(!vcpu)) {
+ flags &= ~RISCV_V_VCPU_CSR_RESTORE;
+ flags |= RISCV_V_VCPU_NEED_RESTORE;
+ riscv_v_flags_set(flags);
+ host_vs = task_pt_regs(current)->status & SR_VS;
+ trace_kvm_riscv_vector_owner(vcpu->vcpu_id,
+ smp_processor_id(), 2, false,
+ trace_reason, host_vs,
+ vcpu->arch.guest_context.sstatus & SR_VS,
+ vcpu->arch.vector_state_generation);
+ }
+ }
+
+ owner = this_cpu_ptr(&kvm_riscv_vector_owner);
+ if (!owner->identity)
+ goto out;
+
+ host_vs = task_pt_regs(current)->status & SR_VS;
+ trace_kvm_riscv_vector_owner(owner->vcpu_id, smp_processor_id(), 2,
+ false, trace_reason, host_vs,
+ owner->guest_vs, owner->generation);
+ owner->identity = 0;
+ owner->cookie = 0;
+ owner->generation = 0;
+ owner->vcpu_id = 0;
+ owner->guest_vs = 0;
+
+out:
+ preempt_enable();
}
int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu)
@@ -88,6 +308,10 @@ int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu)
return -ENOMEM;
}
+ vcpu->arch.vector_owner_cookie =
+ atomic64_inc_return(&kvm_riscv_vector_owner_cookie);
+ vcpu->arch.vector_state_generation = 0;
+
return 0;
}
@@ -110,8 +334,13 @@ void kvm_riscv_vcpu_flush_vector(void)
kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, vcpu->arch.isa);
- if ((vcpu->arch.guest_context.sstatus & SR_VS) != SR_VS_OFF)
+ if ((vcpu->arch.guest_context.sstatus & SR_VS) != SR_VS_OFF) {
+ unsigned long host_vs = task_pt_regs(current)->status & SR_VS;
+
riscv_v_flags_set(riscv_v_flags() | RISCV_V_VCPU_NEED_RESTORE);
+ trace_kvm_riscv_vector_restore_request(vcpu->vcpu_id,
+ smp_processor_id(), 1, host_vs);
+ }
}
#endif
@@ -229,6 +458,13 @@ int kvm_riscv_vcpu_set_reg_vector(struct kvm_vcpu *vcpu,
if (rc)
return rc;
+ /*
+ * copy_from_user() can modify a prefix of the destination before it
+ * reports a fault. Invalidate any cached hardware copy first so a
+ * failed SET_ONE_REG cannot leave newer memory hidden behind a stale
+ * per-hart owner.
+ */
+ vcpu->arch.vector_state_generation++;
if (copy_from_user(reg_addr, uaddr, reg_size))
return -EFAULT;
--
2.43.0