[PATCH 8/8] KVM: x86/xen: Use 32-bit locked ops in kvm_xen_inject_pending_events()

[David Woodhouse]

From: David Woodhouse <dwmw@xxxxxxxxxxxx>

The 64-bit path in kvm_xen_inject_pending_events() uses 'lock orq' and
'lock andq' on vcpu_info->evtchn_pending_sel. If the vcpu_info was
registered with only 4-byte alignment (valid for a 32-bit guest that
later switches to 64-bit mode), this 8-byte locked operation can cause
split-lock #AC exceptions on hosts with split_lock_detect=fatal.

Use the original 64-bit atomics when the vcpu_info is 8-byte aligned
(the common case). Fall back to a 32-bit loop for the rare case where
vcpu_info was registered at only 4-byte alignment. For compat guests
(32-bit evtchn_pending_sel) the loop executes once. For native guests
it executes a second iteration only if the high half has bits to
deliver.

Fixes: 14243b387137 ("KVM: x86/xen: Add KVM_IRQ_ROUTING_XEN_EVTCHN and event channel delivery")
Reported-by: sashiko-bot@xxxxxxxxxx
Assisted-by: Kiro:claude-opus-4.6-1m
Signed-off-by: David Woodhouse <dwmw@xxxxxxxxxxxx>
---
 arch/x86/kvm/xen.c | 63 ++++++++++++++++++++++++++++++++--------------
 1 file changed, 44 insertions(+), 19 deletions(-)

diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c
index 24e939ef5d64..e7b0263d5143 100644
--- a/arch/x86/kvm/xen.c
+++ b/arch/x86/kvm/xen.c
@@ -638,11 +638,17 @@ void kvm_xen_inject_vcpu_vector(struct kvm_vcpu *v)
  */
 void kvm_xen_inject_pending_events(struct kvm_vcpu *v)
 {
-	unsigned long evtchn_pending_sel = READ_ONCE(v->arch.xen.evtchn_pending_sel);
 	struct gfn_to_pfn_cache *gpc = &v->arch.xen.vcpu_info_cache;
+	bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(v->kvm);
+	union evtchn_pending_sel {
+		u64 sel64;
+		u32 sel32[2];
+	} pending, *sel_addr;
+	struct vcpu_info *vi;
 	unsigned long flags;
 
-	if (!evtchn_pending_sel)
+	pending.sel64 = READ_ONCE(v->arch.xen.evtchn_pending_sel);
+	if (!pending.sel64)
 		return;
 
 	/*
@@ -661,31 +667,50 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v)
 	}
 
 	/* Now gpc->khva is a valid kernel address for the vcpu_info */
-	if (kvm_xen_has_64bit_shinfo(v->kvm)) {
-		struct vcpu_info *vi = gpc->khva;
+	vi = gpc->khva;
+	sel_addr = gpc->khva + (has_64bit_shinfo ?
+				offsetof(struct vcpu_info, evtchn_pending_sel) :
+				offsetof(struct compat_vcpu_info, evtchn_pending_sel));
 
+	if (has_64bit_shinfo && IS_ALIGNED((unsigned long)sel_addr, sizeof(u64))) {
+		/*
+		 * 64-bit shinfo with 8-byte aligned vcpu_info (the common
+		 * case): use a single 64-bit atomic.
+		 */
 		asm volatile(LOCK_PREFIX "orq %0, %1\n"
 			     "notq %0\n"
 			     LOCK_PREFIX "andq %0, %2\n"
-			     : "=r" (evtchn_pending_sel),
-			       "+m" (vi->evtchn_pending_sel),
+			     : "=r" (pending.sel64),
+			       "+m" (sel_addr->sel64),
 			       "+m" (v->arch.xen.evtchn_pending_sel)
-			     : "0" (evtchn_pending_sel));
-		WRITE_ONCE(vi->evtchn_upcall_pending, 1);
+			     : "0" (pending.sel64));
 	} else {
-		u32 evtchn_pending_sel32 = evtchn_pending_sel;
-		struct compat_vcpu_info *vi = gpc->khva;
-
-		asm volatile(LOCK_PREFIX "orl %0, %1\n"
-			     "notl %0\n"
-			     LOCK_PREFIX "andl %0, %2\n"
-			     : "=r" (evtchn_pending_sel32),
-			       "+m" (vi->evtchn_pending_sel),
-			       "+m" (v->arch.xen.evtchn_pending_sel)
-			     : "0" (evtchn_pending_sel32));
-		WRITE_ONCE(vi->evtchn_upcall_pending, 1);
+		/*
+		 * Use 32-bit operations to avoid splitlock on a vcpu_info
+		 * that is only 4-byte aligned (registered in 32-bit mode).
+		 * The loop copes with the extremely rare case that the
+		 * vcpu_info was registered in 32-bit mode and only enforced
+		 * 4-byte alignment, and then the VM was latched to 64-bit
+		 * mode afterwards. Which Xen tolerates, so so should KVM.
+		 */
+		int i = 0;
+		do {
+			asm volatile(LOCK_PREFIX "orl %0, %1\n"
+				     "notl %0\n"
+				     LOCK_PREFIX "andl %0, %2\n"
+				     : "=r" (pending.sel32[i]),
+				       "+m" (sel_addr->sel32[i]),
+				       "+m" (((u32 *)&v->arch.xen.evtchn_pending_sel)[i])
+				     : "0" (pending.sel32[i]));
+			i++;
+		} while (has_64bit_shinfo && i < 2 && pending.sel32[i]);
 	}
 
+	/* Assert that there is no need for compat for evtchn_upcall_pending */
+	BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) !=
+		     offsetof(struct compat_vcpu_info, evtchn_upcall_pending));
+	WRITE_ONCE(vi->evtchn_upcall_pending, 1);
+
 	kvm_gpc_mark_dirty_in_slot(gpc);
 	read_unlock_irqrestore(&gpc->lock, flags);
 
-- 
2.54.0