[RFC PATCH v2 04/31] KVM: x86: hyperv: Flush every plane twin on TLB flush hypercalls

[David Woodhouse]

From: David Woodhouse <dwmw@xxxxxxxxxxxx>

The Hyper-V TLB flush hypercalls resolve their target vCPUs through
the plane-0 array, so a flush only ever reached the plane-0 vCPUs, and
the caller's own plane and every other plane kept stale translations.
Two Windows VSM failures come from this. The secure kernel repopulated
a PFN-database page table entry, flushed, then took a fatal page fault
on the very address it had just mapped, because VTL1's TLB still held
the old read-only zero-page translation. And an SMP boot bugchecks 0x1A
(MEMORY_MANAGEMENT, 0x6102) minutes in, when VTL0 reads stale PTEs that
the secure kernel has already edited in mappings both VTLs use.

The TLFS lets a flush cover more than the caller asked for, and does
not return from the hypercall until the targets have left guest mode.
Enqueue the flush entries for every twin of each target VP, and raise
KVM_REQ_HV_TLB_FLUSH through the waiting request fan-out, so the
hypercall waits for every plane, whichever VTL issued it. Make the
request per plane: it lives in the request word the twins share, and a
plane-0 twin consuming it with an empty fifo would otherwise drop a
flush queued for another plane.

The walk reaches twins whose mutex the caller does not hold, so it
leaves the Hyper-V context check to hv_tlb_flush_enqueue(), which reads
it with to_hv_vcpu_safe() and skips a twin that has none yet.

Signed-off-by: David Woodhouse <dwmw@xxxxxxxxxxxx>
---
 arch/x86/include/asm/kvm_host.h |  3 ++-
 arch/x86/kvm/hyperv.c           | 41 ++++++++++++++++++++++++++++++---
 2 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 1ce0b81833b9..f46a3494aacc 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -129,7 +129,8 @@
 	KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP | \
 			   KVM_REQUEST_PER_PLANE)
 #define KVM_REQ_HV_TLB_FLUSH \
-	KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
+	KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP | \
+			   KVM_REQUEST_PER_PLANE)
 #define KVM_REQ_VMSA_PAGE_RELOAD \
 	KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP | \
 			       KVM_REQUEST_PER_PLANE)
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index c76236711208..65c39b30dae7 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -2385,6 +2385,32 @@ static int kvm_hv_hypercall_check_params(struct kvm_vcpu *vcpu,
 	return 0;
 }
 
+/*
+ * The Windows secure kernel edits mappings that the normal kernel also
+ * uses, and expects a flush issued from either VTL to cover both.  Planes
+ * have separate roots and TLB tags, so enqueue the flush for every twin
+ * of the target VP; the TLFS lets a flush cover more than the caller
+ * asked for.
+ */
+static void hv_tlb_flush_enqueue_all_planes(struct kvm_vcpu *v,
+					    u64 *entries, int count)
+{
+	struct kvm_vcpu *twin;
+	int p;
+
+	for (p = 0; p < KVM_MAX_PLANES; p++) {
+		twin = READ_ONCE(v->common->vcpus[p]);
+		if (!twin)
+			continue;
+		/*
+		 * The caller does not hold the twin's mutex; the enqueue
+		 * looks up the twin's Hyper-V context with to_hv_vcpu_safe()
+		 * and skips a twin that has none yet.
+		 */
+		hv_tlb_flush_enqueue(twin, entries, count, false);
+	}
+}
+
 static s64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
 {
 	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
@@ -2521,8 +2547,15 @@ static s64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
 	 */
 	if (all_cpus && !is_guest_mode(vcpu)) {
 		kvm_for_each_vcpu(i, v, kvm)
-			hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, false);
+			hv_tlb_flush_enqueue_all_planes(v, tlb_flush_entries,
+							hc->rep_cnt);
 
+		/*
+		 * KVM_REQ_HV_TLB_FLUSH is per-plane and fans out to every
+		 * twin; the WAIT in the fan-out is the barrier the TLFS
+		 * requires before the hypercall returns, for callers in
+		 * either VTL.
+		 */
 		kvm_make_all_cpus_request(kvm, KVM_REQ_HV_TLB_FLUSH);
 	} else if (!is_guest_mode(vcpu)) {
 		sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask, vcpu_mask);
@@ -2531,10 +2564,12 @@ static s64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
 			v = kvm_get_vcpu(kvm, i);
 			if (!v)
 				continue;
-			hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, false);
+			hv_tlb_flush_enqueue_all_planes(v, tlb_flush_entries,
+							hc->rep_cnt);
 		}
 
-		kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask);
+		kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH,
+					    vcpu_mask);
 	} else {
 		struct kvm_vcpu_hv *hv_v;
 
-- 
2.50.1