[RFC PATCH v2 22/31] KVM: selftests: Test a plane suspending with an undelivered interrupt

[David Woodhouse]

From: David Woodhouse <dwmw@xxxxxxxxxxxx>

Pin the forced-VTL-entry re-arm: a plane-1 twin self-IPIs with IF
clear, runs on (consuming its single-shot hv_apic_event_pending
marker), and is then suspended by KVM_SET_PLANE_RUN_STATE exactly like
a userspace VTL switch.  Running plane 0 must report the suspended
twin's deliverable interrupt as a KVM_EXIT_HYPERV_VTL_INTERRUPT exit,
and the re-entered twin must receive the vector once it sets IF.  On a
kernel without the re-arm, plane 0 reaches its guest code and the
twin's interrupt is stranded in its IRR, which is the Windows VSM
hibernate deadlock in miniature.  The VM enables
KVM_HYPERV_HCALL_VTL_CALLS, which VTL interrupt exits require.

The default selftest VM owns a fully in-kernel irqchip, which planes
refuse; add a kvm_force_split_irqchip override to the library (as in
the public planes tree) so a planes test can use the stock VM setup.

Signed-off-by: David Woodhouse <dwmw@xxxxxxxxxxxx>
---
 tools/testing/selftests/kvm/Makefile.kvm      |   1 +
 .../testing/selftests/kvm/include/kvm_util.h  |   1 +
 tools/testing/selftests/kvm/lib/kvm_util.c    |  11 ++
 .../selftests/kvm/x86/planes_vtl_entry_test.c | 186 ++++++++++++++++++
 4 files changed, 199 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/x86/planes_vtl_entry_test.c

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index 8e1c0ffe81f6..250b324d0881 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -114,6 +114,7 @@ TEST_GEN_PROGS_x86 += x86/nested_x2apic_test
 TEST_GEN_PROGS_x86 += x86/platform_info_test
 TEST_GEN_PROGS_x86 += x86/plane_x86_test
 TEST_GEN_PROGS_x86 += x86/planes_test
+TEST_GEN_PROGS_x86 += x86/planes_vtl_entry_test
 TEST_GEN_PROGS_x86 += x86/pmu_counters_test
 TEST_GEN_PROGS_x86 += x86/pmu_event_filter_test
 TEST_GEN_PROGS_x86 += x86/private_mem_conversions_test
diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h
index 4818e3f5454c..01f7fe111188 100644
--- a/tools/testing/selftests/kvm/include/kvm_util.h
+++ b/tools/testing/selftests/kvm/include/kvm_util.h
@@ -853,6 +853,7 @@ static inline bool is_smt_on(void)
 	return false;
 }
 
+extern bool kvm_force_split_irqchip;
 void vm_create_irqchip(struct kvm_vm *vm);
 
 /*
diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c
index 7e90f3b619ba..8ba1d0c011e9 100644
--- a/tools/testing/selftests/kvm/lib/kvm_util.c
+++ b/tools/testing/selftests/kvm/lib/kvm_util.c
@@ -1802,11 +1802,22 @@ int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset,
 	return region->region.guest_memfd;
 }
 
+/*
+ * Tests that need a split IRQ chip (x86 only), e.g. for VM planes, set this
+ * around VM creation to override the fully in-kernel default.
+ */
+bool kvm_force_split_irqchip;
+
 /* Create an interrupt controller chip for the specified VM. */
 void vm_create_irqchip(struct kvm_vm *vm)
 {
 	int r;
 
+	if (kvm_force_split_irqchip && kvm_has_cap(KVM_CAP_SPLIT_IRQCHIP)) {
+		vm_enable_split_irqchip(vm);
+		return;
+	}
+
 	/*
 	 * Allocate a fully in-kernel IRQ chip by default, but fall back to a
 	 * split model (x86 only) if that fails (KVM x86 allows compiling out
diff --git a/tools/testing/selftests/kvm/x86/planes_vtl_entry_test.c b/tools/testing/selftests/kvm/x86/planes_vtl_entry_test.c
new file mode 100644
index 000000000000..b0565b6c442b
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86/planes_vtl_entry_test.c
@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Test that an interrupt posted to a higher plane survives the plane
+ * suspending without having delivered it.
+ *
+ * A higher plane's pending interrupt forces entry into that plane via a
+ * single-shot marker (hv_apic_event_pending) that the plane consumes when
+ * it runs.  A plane that runs with IF clear until it suspends consumes
+ * the marker without delivering the vector, which used to strand the
+ * vector in the plane's IRR forever: with the marker gone, the lower
+ * plane never again exits with KVM_EXIT_HYPERV_VTL_INTERRUPT, and a
+ * guest waiting on the higher plane deadlocks.
+ *
+ * The test models the failing sequence: the plane-1 twin self-IPIs with
+ * IF clear, keeps running (consuming the marker), then is suspended by
+ * the run-state ioctl exactly like a userspace VTL switch.  Running
+ * plane 0 must then report a pending VTL interrupt for plane 1, and the
+ * re-entered twin must receive the vector once it sets IF.
+ */
+#include <errno.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "test_util.h"
+#include "kvm_util.h"
+#include "processor.h"
+#include "apic.h"
+#include "kselftest.h"
+
+#ifndef KVM_EXIT_HYPERV_VTL_INTERRUPT
+#define KVM_EXIT_HYPERV_VTL_INTERRUPT  4
+#endif
+
+#define TEST_VECTOR		0xe0
+#define TWIN_SYNC_PORT		0x7e
+
+static volatile bool twin_got_ipi;
+
+static void twin_sync(uint8_t stage)
+{
+	asm volatile("outb %0, %1" :: "a"(stage), "Nd"((uint16_t)TWIN_SYNC_PORT));
+}
+
+static void twin_ipi_handler(struct ex_regs *regs)
+{
+	twin_got_ipi = true;
+	x2apic_write_reg(APIC_EOI, 0);
+}
+
+/* Runs with IF clear: the initial RFLAGS has IF unset and nothing sets it. */
+static void twin_guest_code(void)
+{
+	x2apic_enable();
+	x2apic_write_reg(APIC_SELF_IPI, TEST_VECTOR);
+	twin_sync(1);
+	twin_sync(2);
+
+	/* Re-entered after suspension: open the window, take the vector. */
+	sti();
+	while (!twin_got_ipi)
+		cpu_relax();
+	twin_sync(3);
+	for (;;)
+		twin_sync(4);
+}
+
+static void plane0_guest_code(void)
+{
+	GUEST_SYNC(0);
+	GUEST_DONE();
+}
+
+static void set_run_states(struct kvm_vcpu *vcpu0, u64 state0,
+			   struct kvm_plane_vcpu *twin, u64 state1)
+{
+	plane_vcpu_ioctl(twin, KVM_SET_PLANE_RUN_STATE, &state1);
+	vcpu_ioctl(vcpu0, KVM_SET_PLANE_RUN_STATE, &state0);
+}
+
+/*
+ * The selected plane runs through plane 0's fd; vcpu_run() retries
+ * EINTR.  A twin exit is reported in the twin's own kvm_run.
+ */
+static void run_twin_expect_sync(struct kvm_vcpu *vcpu0,
+				 struct kvm_run *twin_run, uint8_t stage)
+{
+	vcpu_run(vcpu0);
+	TEST_ASSERT_EQ(twin_run->exit_reason, KVM_EXIT_IO);
+	TEST_ASSERT_EQ(twin_run->io.port, TWIN_SYNC_PORT);
+	TEST_ASSERT_EQ(*((uint8_t *)twin_run + twin_run->io.data_offset), stage);
+}
+
+int main(int argc, char *argv[])
+{
+	struct kvm_plane_vcpu *twin;
+	struct kvm_plane *plane;
+	struct kvm_sregs sregs;
+	struct kvm_regs regs;
+	struct kvm_run *twin_run;
+	struct kvm_vcpu *vcpu0;
+	struct kvm_vm *vm;
+	int mmap_sz;
+
+	kvm_force_split_irqchip = true;
+	vm = __vm_create(VM_SHAPE_DEFAULT, 1, 0);
+	kvm_force_split_irqchip = false;
+
+	TEST_REQUIRE(vm_check_cap(vm, KVM_CAP_PLANES) >= 1);
+	TEST_REQUIRE(__vm_enable_cap(vm, KVM_CAP_PLANES, 2) == 0);
+	TEST_REQUIRE(kvm_check_cap(KVM_CAP_HYPERV_FORWARD_HCALL) &
+		     KVM_HYPERV_HCALL_VTL_CALLS);
+	/* VTL interrupt exits belong to the userspace-switch VTL contract. */
+	vm_enable_cap(vm, KVM_CAP_HYPERV_FORWARD_HCALL,
+		      KVM_HYPERV_HCALL_VTL_CALLS);
+
+	ksft_print_header();
+	ksft_set_plan(1);
+
+	vcpu0 = vm_vcpu_add(vm, 0, plane0_guest_code);
+	plane = vm_plane_add(vm, 1);
+	twin = __vm_plane_vcpu_add(vcpu0, plane);
+
+	mmap_sz = __kvm_ioctl(vm->kvm_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
+	TEST_ASSERT(mmap_sz >= (int)sizeof(*twin_run),
+		    "KVM_GET_VCPU_MMAP_SIZE failed");
+	twin_run = kvm_mmap(mmap_sz, PROT_READ | PROT_WRITE, MAP_SHARED,
+			    twin->fd);
+
+	vm_install_exception_handler(vm, TEST_VECTOR, twin_ipi_handler);
+
+	/*
+	 * The twin shares the mode and page tables with plane 0: reuse
+	 * plane 0's segment state and point it at its own code and stack.
+	 * RFLAGS keeps IF clear.
+	 */
+	vcpu_sregs_get(vcpu0, &sregs);
+	plane_vcpu_ioctl(twin, KVM_SET_SREGS, &sregs);
+
+	plane_vcpu_ioctl(twin, KVM_GET_REGS, &regs);
+	regs.rip = (unsigned long)twin_guest_code;
+	regs.rsp = vm_alloc_stack(vm, 4);
+	regs.rflags = 0x2;
+	plane_vcpu_ioctl(twin, KVM_SET_REGS, &regs);
+
+	/*
+	 * Run the twin as a userspace VTL switch would: twin runnable,
+	 * plane 0 held.  The twin self-IPIs with IF clear, so the
+	 * vector lands in its IRR without being delivered, and the
+	 * second sync guarantees a fresh guest entry consumed the
+	 * forced-entry marker.
+	 */
+	set_run_states(vcpu0, KVM_PLANE_RUN_STATE_USER_HOLD,
+		       twin, 0);
+	run_twin_expect_sync(vcpu0, twin_run, 1);
+	run_twin_expect_sync(vcpu0, twin_run, 2);
+
+	/* Suspend the twin with the vector still latched in its IRR. */
+	set_run_states(vcpu0, 0,
+		       twin, KVM_PLANE_RUN_STATE_USER_HOLD);
+
+	/*
+	 * Plane 0 must not run the guest: the suspended twin still has a
+	 * deliverable interrupt, which must surface as a VTL interrupt
+	 * exit.  On a broken kernel the marker was consumed and plane 0
+	 * reaches its guest code.
+	 */
+	vcpu_run(vcpu0);
+	TEST_ASSERT(vcpu0->run->exit_reason == KVM_EXIT_HYPERV,
+		    "Expected KVM_EXIT_HYPERV, got %u (%s): the twin's pending interrupt was lost when it suspended",
+		    vcpu0->run->exit_reason,
+		    exit_reason_str(vcpu0->run->exit_reason));
+	TEST_ASSERT_EQ(vcpu0->run->hyperv.type, KVM_EXIT_HYPERV_VTL_INTERRUPT);
+	TEST_ASSERT_EQ(vcpu0->run->hyperv.plane, 1);
+
+	/* Enter the twin as userspace would, and let it take the vector. */
+	set_run_states(vcpu0, KVM_PLANE_RUN_STATE_USER_HOLD,
+		       twin, 0);
+	run_twin_expect_sync(vcpu0, twin_run, 3);
+	ksft_test_result_pass("suspended plane keeps its undelivered interrupt\n");
+
+	kvm_munmap(twin_run, mmap_sz);
+	kvm_vm_free(vm);
+	ksft_finished();
+}
-- 
2.50.1