[PATCH] KVM: pfncache: track all MMU notifier invalidations

[David Lee]

From: Kyle Zeng <kylebot@xxxxxxxxxx>

There is a race condition in KVM's gfn-to-pfn cache refresh and MMU
notifier handling. An HVA-backed cache can publish a stale PFN and
kernel virtual address after the corresponding userspace mapping has
been invalidated. The Xen shared-info HVA interface immediately reads
and writes through that stale address, resulting in a host-kernel
use-after-free.

The cache refresh path in virt/kvm/pfncache.c drops gpc->lock while
resolving and mapping an HVA. It uses mn_active_invalidate_count and
mmu_invalidate_seq to detect an MMU notifier interval that overlaps
this unlocked window. However, mmu_invalidate_seq is advanced only
when the invalidated HVA overlaps a KVM memslot. HVA-backed caches are
explicitly allowed to refer to memory outside all memslots. If such an
invalidation starts and finishes while gpc->valid is false, the active
count returns to zero without a sequence change and the refresh accepts
a stale PFN.

An unprivileged process with access to /dev/kvm can reach this path with
KVM_XEN_ATTR_TYPE_SHARED_INFO_HVA. KASAN-detected use-after-free in
kvm_xen_shared_info_init(). The affected function reads and writes Xen
wall-clock fields through the stale mapping, so the issue can cause a
host-kernel crash and memory corruption.

The attached KASAN output confirms:

  BUG: KASAN: use-after-free in kvm_xen_shared_info_init+0x344/0x3d0 [kvm]
  Read of size 4 at addr ffff888046000900 by task poc/1266

Add a notifier-specific sequence that advances for every completed
invalidate interval before mn_active_invalidate_count is decremented,
and use that sequence for pfncache retry. The existing barrier pairing
then guarantees refresh observes either an active invalidation or a
sequence change.

Fixes: 721f5b0dda78 ("KVM: pfncache: allow a cache to be activated with a fixed (userspace) HVA")
Cc: stable@xxxxxxxxxxxxxxx # 6.9+
Assisted-by: Codex:gpt-5.6-sol Codex:gpt-5.5-cyber
Signed-off-by: Kyle Zeng <kylebot@xxxxxxxxxx>
Co-developed-by: David Lee <david.lee@xxxxxxxxxxxxxxx>
Signed-off-by: David Lee <david.lee@xxxxxxxxxxxxxxx>
---
Bug found and triaged by OpenAI Security Research and
validated by Trail of Bits.

Trail of Bits has a reproducer for this bug that triggers a
KASAN use-after-free and can share if needed.

 include/linux/kvm_host.h |  1 +
 virt/kvm/kvm_main.c      |  9 ++++++++-
 virt/kvm/pfncache.c      | 18 +++++++++---------
 3 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index ab8cfaec8..0ac382cd9 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -800,6 +800,7 @@ struct kvm {
 	/* Used to wait for completion of MMU notifiers.  */
 	spinlock_t mn_invalidate_lock;
 	unsigned long mn_active_invalidate_count;
+	unsigned long mn_invalidate_seq;
 	struct rcuwait mn_memslots_update_rcuwait;
 
 	/* For management / invalidation of gfn_to_pfn_caches */
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 45e784462..5e43dd63c 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -812,8 +812,15 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 
 	/* Pairs with the increment in range_start(). */
 	spin_lock(&kvm->mn_invalidate_lock);
-	if (!WARN_ON_ONCE(!kvm->mn_active_invalidate_count))
+	if (!WARN_ON_ONCE(!kvm->mn_active_invalidate_count)) {
+		kvm->mn_invalidate_seq++;
+		/*
+		 * Publish the sequence update before dropping the active count
+		 * so that pfncache refreshes observe one or the other.
+		 */
+		smp_wmb();
 		--kvm->mn_active_invalidate_count;
+	}
 	wake = !kvm->mn_active_invalidate_count;
 	spin_unlock(&kvm->mn_invalidate_lock);
 
diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c
index 728d2c1b4..d360f1eda 100644
--- a/virt/kvm/pfncache.c
+++ b/virt/kvm/pfncache.c
@@ -124,7 +124,7 @@ static void gpc_unmap(kvm_pfn_t pfn, void *khva)
 #endif
 }
 
-static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_seq)
+static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mn_seq)
 {
 	/*
 	 * mn_active_invalidate_count acts for all intents and purposes
@@ -136,20 +136,20 @@ static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_s
 	 * Note, it does not matter that mn_active_invalidate_count
 	 * is not protected by gpc->lock.  It is guaranteed to
 	 * be elevated before the mmu_notifier acquires gpc->lock, and
-	 * isn't dropped until after mmu_invalidate_seq is updated.
+	 * isn't dropped until after mn_invalidate_seq is updated.
 	 */
-	if (kvm->mn_active_invalidate_count)
+	if (READ_ONCE(kvm->mn_active_invalidate_count))
 		return true;
 
 	/*
 	 * Ensure mn_active_invalidate_count is read before
-	 * mmu_invalidate_seq.  This pairs with the smp_wmb() in
+	 * mn_invalidate_seq.  This pairs with the smp_wmb() in
 	 * mmu_notifier_invalidate_range_end() to guarantee either the
 	 * old (non-zero) value of mn_active_invalidate_count or the
-	 * new (incremented) value of mmu_invalidate_seq is observed.
+	 * new (incremented) value of mn_invalidate_seq is observed.
 	 */
 	smp_rmb();
-	return kvm->mmu_invalidate_seq != mmu_seq;
+	return READ_ONCE(kvm->mn_invalidate_seq) != mn_seq;
 }
 
 static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc)
@@ -158,7 +158,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc)
 	void *old_khva = (void *)PAGE_ALIGN_DOWN((uintptr_t)gpc->khva);
 	kvm_pfn_t new_pfn = KVM_PFN_ERR_FAULT;
 	void *new_khva = NULL;
-	unsigned long mmu_seq;
+	unsigned long mn_seq;
 	struct page *page;
 
 	struct kvm_follow_pfn kfp = {
@@ -181,7 +181,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc)
 	gpc->valid = false;
 
 	do {
-		mmu_seq = gpc->kvm->mmu_invalidate_seq;
+		mn_seq = READ_ONCE(gpc->kvm->mn_invalidate_seq);
 		smp_rmb();
 
 		write_unlock_irq(&gpc->lock);
@@ -232,7 +232,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc)
 		 * attempting to refresh.
 		 */
 		WARN_ON_ONCE(gpc->valid);
-	} while (mmu_notifier_retry_cache(gpc->kvm, mmu_seq));
+	} while (mmu_notifier_retry_cache(gpc->kvm, mn_seq));
 
 	gpc->valid = true;
 	gpc->pfn = new_pfn;
-- 
2.53.0