[RFC PATCH 43/46] KVM: arm64: Add Caretaker EL2 exception vectors and guest entry/exit assembly
From: Pasha Tatashin
Date: Mon Sep 21 2026 - 17:05:31 EST
Add arch/arm64/kvm/caretaker_vmenter.S for standalone VHE EL2 guest
entry/exit world switch and exception vector dispatch on preserved cores.
Signed-off-by: Pasha Tatashin <pasha.tatashin@xxxxxxxxxx>
---
arch/arm64/kvm/caretaker_vmenter.S | 248 +++++++++++++++++++++++++++++
1 file changed, 248 insertions(+)
create mode 100644 arch/arm64/kvm/caretaker_vmenter.S
diff --git a/arch/arm64/kvm/caretaker_vmenter.S b/arch/arm64/kvm/caretaker_vmenter.S
new file mode 100644
index 000000000000..6dff9fbe931f
--- /dev/null
+++ b/arch/arm64/kvm/caretaker_vmenter.S
@@ -0,0 +1,248 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Caretaker low-level guest entry/exit and vector table for ARM64 VHE.
+ *
+ * Copyright (c) 2026, Google LLC.
+ * Pasha Tatashin <pasha.tatashin@xxxxxxxxxx>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+#include <asm/kvm_arm.h>
+#include <asm/kvm_asm.h>
+#include <asm/sysreg.h>
+#include "caretaker.h"
+
+ .pushsection ".text.cpu_preserved", "ax"
+
+.macro cventry target
+ .align 7
+ stp x0, x1, [sp, #-16]!
+ b \target
+.endm
+
+ .align 11
+SYM_CODE_START(caretaker_hyp_vector)
+ /* Current EL with SP0 */
+ cventry caretaker_invalid
+ cventry caretaker_el2h_irq
+ cventry caretaker_el2h_irq
+ cventry caretaker_invalid
+
+ /* Current EL with SPx (EL2h) */
+ cventry caretaker_invalid
+ cventry caretaker_el2h_irq
+ cventry caretaker_el2h_irq
+ cventry caretaker_invalid
+
+ /* Lower EL using AArch64 (Guest EL1/EL0) */
+ cventry caretaker_el1_sync
+ cventry caretaker_el1_irq
+ cventry caretaker_el1_fiq
+ cventry caretaker_el1_error
+
+ /* Lower EL using AArch32 */
+ cventry caretaker_el1_sync
+ cventry caretaker_el1_irq
+ cventry caretaker_el1_fiq
+ cventry caretaker_el1_error
+SYM_CODE_END(caretaker_hyp_vector)
+
+SYM_CODE_START_LOCAL(caretaker_invalid)
+ mrs x0, elr_el2
+ mrs x1, esr_el2
+ mrs x2, far_el2
+ b arm64_caretaker_handle_invalid
+SYM_CODE_END(caretaker_invalid)
+
+SYM_CODE_START_LOCAL(caretaker_el2h_irq)
+ stp x2, x3, [sp, #-16]!
+ stp x4, x5, [sp, #-16]!
+ stp x6, x7, [sp, #-16]!
+ stp x8, x9, [sp, #-16]!
+ stp x10, x11, [sp, #-16]!
+ stp x12, x13, [sp, #-16]!
+ stp x14, x15, [sp, #-16]!
+ stp x16, x17, [sp, #-16]!
+ stp x29, x30, [sp, #-16]!
+
+ mrs_s x0, SYS_ICC_IAR1_EL1
+ cmp x0, #GIC_SPECIAL_INTID_START
+ b.ge 1f
+ msr_s SYS_ICC_EOIR1_EL1, x0
+ msr_s SYS_ICC_DIR_EL1, x0
+1:
+ /*
+ * Silence the EL2 physical timer before returning.
+ *
+ * It is the only EL2 interrupt source the Caretaker arms
+ * (arm64_caretaker_op_arm_timer() writes CNTHP_CTL_EL2.ENABLE to
+ * drive the on-core quantum), and it is level-triggered off
+ * CNTHP_CTL_EL2.ISTATUS. Neither EOIR1 nor DIR clears ISTATUS, so
+ * without disabling ENABLE here the line re-asserts the instant we
+ * eret and the core live-locks taking the same interrupt forever --
+ * it never reaches the run loop, which means it can never observe a
+ * detach request and can never be reclaimed.
+ *
+ * This is done on the special-INTID path too: a spurious read of
+ * IAR1 does not mean the timer has stopped asserting. The run loop
+ * re-arms the timer for each new quantum, so dropping it here costs
+ * at most the remainder of the current one.
+ */
+ msr_s SYS_CNTHP_CTL_EL2, xzr
+ isb
+
+ ldp x29, x30, [sp], #16
+ ldp x16, x17, [sp], #16
+ ldp x14, x15, [sp], #16
+ ldp x12, x13, [sp], #16
+ ldp x10, x11, [sp], #16
+ ldp x8, x9, [sp], #16
+ ldp x6, x7, [sp], #16
+ ldp x4, x5, [sp], #16
+ ldp x2, x3, [sp], #16
+ ldp x0, x1, [sp], #16
+ eret
+ sb
+SYM_CODE_END(caretaker_el2h_irq)
+
+SYM_CODE_START_LOCAL(caretaker_el1_sync)
+ mov x0, #ARM_EXCEPTION_TRAP
+ b caretaker_guest_exit
+SYM_CODE_END(caretaker_el1_sync)
+
+SYM_CODE_START_LOCAL(caretaker_el1_irq)
+SYM_CODE_START_LOCAL(caretaker_el1_fiq)
+ mov x0, #ARM_EXCEPTION_IRQ
+ b caretaker_guest_exit
+SYM_CODE_END(caretaker_el1_fiq)
+SYM_CODE_END(caretaker_el1_irq)
+
+SYM_CODE_START_LOCAL(caretaker_el1_error)
+ mov x0, #ARM_EXCEPTION_EL1_SERROR
+ b caretaker_guest_exit
+SYM_CODE_END(caretaker_el1_error)
+
+/*
+ * u64 caretaker_guest_enter(struct kvm_vcpu *vcpu);
+ * x0: vcpu pointer
+ * Returns exit code in x0.
+ */
+SYM_FUNC_START(caretaker_guest_enter)
+ // Save host callee-saved registers on stack
+ stp x19, x20, [sp, #-112]!
+ stp x21, x22, [sp, #16]
+ stp x23, x24, [sp, #32]
+ stp x25, x26, [sp, #48]
+ stp x27, x28, [sp, #64]
+ stp x29, x30, [sp, #80]
+ str x0, [sp, #96] // Save vcpu pointer on host stack
+ mrs x1, sp_el0
+ str x1, [sp, #104] // Save host sp_el0 (current task)
+
+ // Install caretaker vectors
+ adr_l x1, caretaker_hyp_vector
+ msr vbar_el1, x1
+ msr vbar_el2, x1
+ isb
+
+ // Restore guest context from ctx->ctxt
+ add x29, x0, #CAP_CTXT_OFFSET
+
+ ldr x1, [x29, #CPU_XREG_OFFSET(32)] // regs.pc
+ msr elr_el2, x1
+ ldr x1, [x29, #CPU_XREG_OFFSET(33)] // regs.pstate
+ msr spsr_el2, x1
+ ldr x1, [x29, #CPU_XREG_OFFSET(31)] // regs.sp
+ msr sp_el0, x1
+
+ // Restore guest GPRs x0-x28, lr (x30)
+ ldp x0, x1, [x29, #CPU_XREG_OFFSET(0)]
+ ldp x2, x3, [x29, #CPU_XREG_OFFSET(2)]
+ ldp x4, x5, [x29, #CPU_XREG_OFFSET(4)]
+ ldp x6, x7, [x29, #CPU_XREG_OFFSET(6)]
+ ldp x8, x9, [x29, #CPU_XREG_OFFSET(8)]
+ ldp x10, x11, [x29, #CPU_XREG_OFFSET(10)]
+ ldp x12, x13, [x29, #CPU_XREG_OFFSET(12)]
+ ldp x14, x15, [x29, #CPU_XREG_OFFSET(14)]
+ ldp x16, x17, [x29, #CPU_XREG_OFFSET(16)]
+ ldp x18, x19, [x29, #CPU_XREG_OFFSET(18)]
+ ldp x20, x21, [x29, #CPU_XREG_OFFSET(20)]
+ ldp x22, x23, [x29, #CPU_XREG_OFFSET(22)]
+ ldp x24, x25, [x29, #CPU_XREG_OFFSET(24)]
+ ldp x26, x27, [x29, #CPU_XREG_OFFSET(26)]
+ ldr x30, [x29, #CPU_XREG_OFFSET(30)]
+ ldp x28, x29, [x29, #CPU_XREG_OFFSET(28)]
+
+ eret
+ sb
+SYM_FUNC_END(caretaker_guest_enter)
+
+SYM_CODE_START_LOCAL(caretaker_guest_exit)
+ // Save exception code in x0 temporarily
+ str x0, [sp, #-16]!
+
+ // Retrieve ctx pointer from host stack
+ ldr x0, [sp, #128]
+ add x1, x0, #CAP_CTXT_OFFSET
+
+ // Save guest GPRs x2-x29, lr (x30)
+ stp x2, x3, [x1, #CPU_XREG_OFFSET(2)]
+ stp x4, x5, [x1, #CPU_XREG_OFFSET(4)]
+ stp x6, x7, [x1, #CPU_XREG_OFFSET(6)]
+ stp x8, x9, [x1, #CPU_XREG_OFFSET(8)]
+ stp x10, x11, [x1, #CPU_XREG_OFFSET(10)]
+ stp x12, x13, [x1, #CPU_XREG_OFFSET(12)]
+ stp x14, x15, [x1, #CPU_XREG_OFFSET(14)]
+ stp x16, x17, [x1, #CPU_XREG_OFFSET(16)]
+ stp x18, x19, [x1, #CPU_XREG_OFFSET(18)]
+ stp x20, x21, [x1, #CPU_XREG_OFFSET(20)]
+ stp x22, x23, [x1, #CPU_XREG_OFFSET(22)]
+ stp x24, x25, [x1, #CPU_XREG_OFFSET(24)]
+ stp x26, x27, [x1, #CPU_XREG_OFFSET(26)]
+ stp x28, x29, [x1, #CPU_XREG_OFFSET(28)]
+ str x30, [x1, #CPU_XREG_OFFSET(30)]
+
+ // Clear Branch History Buffer (Spectre-BHB mitigation) across EL1->EL2 exit
+ mov x2, #32
+1: b 2f
+2: subs x2, x2, #1
+ b.ne 1b
+ isb
+
+ // Retrieve and save guest x0, x1 (from vector entry on stack)
+ ldp x2, x3, [sp, #16]
+ stp x2, x3, [x1, #CPU_XREG_OFFSET(0)]
+
+ // Save guest system registers
+ mrs x2, elr_el2
+ str x2, [x1, #CPU_XREG_OFFSET(32)] // regs.pc
+ mrs x2, spsr_el2
+ str x2, [x1, #CPU_XREG_OFFSET(33)] // regs.pstate
+ mrs x2, sp_el0
+ str x2, [x1, #CPU_XREG_OFFSET(31)] // regs.sp
+ mrs x2, esr_el2
+ str x2, [x0, #CAP_FAULT_ESR]
+ mrs x2, far_el2
+ str x2, [x0, #CAP_FAULT_FAR]
+ mrs x2, hpfar_el2
+ str x2, [x0, #CAP_FAULT_HPFAR]
+
+ // Restore return value (exception code) in x0
+ ldr x0, [sp], #32 // pops exception code + saved x0,x1
+
+ // Restore host callee-saved registers
+ ldp x19, x20, [sp, #0]
+ ldp x21, x22, [sp, #16]
+ ldp x23, x24, [sp, #32]
+ ldp x25, x26, [sp, #48]
+ ldp x27, x28, [sp, #64]
+ ldp x29, x30, [sp, #80]
+ ldr x1, [sp, #104]
+ msr sp_el0, x1
+ add sp, sp, #112
+
+ ret
+SYM_CODE_END(caretaker_guest_exit)
+
+ .popsection
--
2.55.0.1082.g2b9226bbc0-goog