[RFC PATCH v3 2/5] KVM: guest_memfd: Add ioctl KVM_LINK_GUEST_MEMFD

[Tao Gong]

From: Ackerley Tng <ackerleytng@xxxxxxxxxx>

A guest_memfd file is bound to the KVM instance that created it and
cannot be bound to another VM's memslots. Add KVM_LINK_GUEST_MEMFD to
link an existing guest_memfd's underlying inode to a new file (and fd)
bound to a target VM, allowing multiple VMs on the same host to use the
same underlying guest_memfd memory without copying (e.g. for intra-host
migration).

Only allow linking between distinct, compatible VMs (same VM type on x86)
so that private memory of a confidential VM cannot be mapped into an
incompatible VM, and serialize additions to the inode's gmem_file_list
with the mapping's invalidate_lock.

Signed-off-by: Ackerley Tng <ackerleytng@xxxxxxxxxx>
Co-developed-by: Ryan Afranji <afranji@xxxxxxxxxx>
Signed-off-by: Ryan Afranji <afranji@xxxxxxxxxx>
Co-developed-by: Tao Gong <gongtao@xxxxxxxxxx>
Signed-off-by: Tao Gong <gongtao@xxxxxxxxxx>
---
 Documentation/virt/kvm/api.rst | 42 +++++++++++++++++
 arch/x86/kvm/x86.c             | 10 ++++
 include/linux/kvm_host.h       |  1 +
 include/uapi/linux/kvm.h       |  8 ++++
 virt/kvm/guest_memfd.c         | 83 ++++++++++++++++++++++++++++++++++
 virt/kvm/guest_memfd.h         |  1 +
 virt/kvm/kvm_main.c            | 10 ++++
 7 files changed, 155 insertions(+)

diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst
index 3d6a4bf01c700..7c527a1e7c487 100644
--- a/Documentation/virt/kvm/api.rst
+++ b/Documentation/virt/kvm/api.rst
@@ -6759,6 +6759,48 @@ if the guest_memfd memory was pinned in IOMMU page tables.
 
 See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES`.
 
+4.147 KVM_LINK_GUEST_MEMFD
+--------------------------
+
+:Capability: KVM_CAP_GUEST_MEMFD_LINK
+:Architectures: all
+:Type: vm ioctl
+:Parameters: struct kvm_link_guest_memfd (in)
+:Returns: A file descriptor on success, <0 on error
+
+Errors:
+
+  ========== ===============================================================
+  EINVAL     ``fd`` is not an open guest_memfd file descriptor, ``flags`` or
+             ``reserved`` are not zero, the target VM is not compatible with
+             the VM ``fd`` is associated with (e.g. on x86, the VMs are of
+             different VM types), or ``fd`` is already associated with the
+             target VM.
+  EFAULT     The parameter address was invalid.
+  ENOMEM     Ran out of memory.
+  ========== ===============================================================
+
+::
+
+  struct kvm_link_guest_memfd {
+	__u64 fd;
+	__u64 flags;
+	__u64 reserved[6];
+  };
+
+KVM_LINK_GUEST_MEMFD creates a new guest_memfd file descriptor that is
+associated with the target VM (the VM on which the ioctl is invoked) and refers
+to the same underlying memory as the existing guest_memfd referred to by ``fd``,
+similar to how link() creates another name for an existing file.  The new file
+descriptor can be used in KVM_SET_USER_MEMORY_REGION2 for the target VM, and
+the memory's contents, size, flags and (with in-place conversion) memory
+attributes are shared by all linked guest_memfd instances.
+
+The primary use case is intra-host migration, where the destination VM needs to
+access the same guest memory as the source VM without copying it.
+
+No flags are defined yet; ``flags`` and ``reserved`` must be zero.
+
 .. _kvm_run:
 
 5. The kvm_run structure
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c081e4b9b25de..4427d32290c0f 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -10648,6 +10648,16 @@ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
 	return gmem_in_place_conversion || !kvm_arch_has_private_mem(kvm);
 }
 
+bool kvm_arch_gmem_link_allowed(struct kvm *dst, struct kvm *src)
+{
+	/*
+	 * Different VM types have different requirements for, and different
+	 * (vendor) state associated with, guest_memfd memory, e.g. memory that
+	 * is private to a TDX guest must never be mapped into a non-TDX VM.
+	 */
+	return dst->arch.vm_type == src->arch.vm_type;
+}
+
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
 int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
 			       kvm_pfn_t nr_pages)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 485f18454eb45..5cffb7ad376e1 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -2585,6 +2585,7 @@ static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn)
 #ifdef CONFIG_KVM_GUEST_MEMFD
 bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn);
 bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm);
+bool kvm_arch_gmem_link_allowed(struct kvm *dst, struct kvm *src);
 
 static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm)
 {
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 8dff2fc1972e9..53fe8dce57db2 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -1690,4 +1690,12 @@ struct kvm_pre_fault_memory {
 	__u64 padding[5];
 };
 
+#define KVM_LINK_GUEST_MEMFD	_IOWR(KVMIO,  0xd6, struct kvm_link_guest_memfd)
+
+struct kvm_link_guest_memfd {
+	__u64 fd;
+	__u64 flags;
+	__u64 reserved[6];
+};
+
 #endif /* __LINUX_KVM_H */
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index 3ca6f64d48040..54b11747a8c48 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -892,6 +892,15 @@ bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
 	return true;
 }
 
+/*
+ * Returns true if a guest_memfd instance associated with @src may be linked
+ * into @dst, i.e. if both VMs can safely share the underlying memory.
+ */
+bool __weak kvm_arch_gmem_link_allowed(struct kvm *dst, struct kvm *src)
+{
+	return kvm_arch_has_private_mem(dst) == kvm_arch_has_private_mem(src);
+}
+
 static int kvm_gmem_init_inode(struct inode *inode, loff_t size, u64 flags)
 {
 	struct gmem_inode *gi = GMEM_I(inode);
@@ -970,7 +979,10 @@ static struct file *kvm_gmem_alloc_file(struct kvm *kvm, struct inode *inode,
 	kvm_get_kvm(kvm);
 	f->kvm = kvm;
 	xa_init(&f->bindings);
+
+	filemap_invalidate_lock(inode->i_mapping);
 	list_add(&f->entry, &GMEM_I(inode)->gmem_file_list);
+	filemap_invalidate_unlock(inode->i_mapping);
 
 	return file;
 }
@@ -1026,6 +1038,77 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args)
 	return __kvm_gmem_create(kvm, size, flags);
 }
 
+int kvm_gmem_link(struct kvm *kvm, struct kvm_link_guest_memfd *args)
+{
+	static const char *name = "[kvm-gmem]";
+	u64 flags = args->flags;
+	u64 valid_flags = 0;
+	struct file *dst_file, *src_file;
+	struct gmem_file *gmem;
+	struct timespec64 ts;
+	struct inode *inode;
+	struct fd f;
+	int ret, fd, i;
+
+	if (flags & ~valid_flags)
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(args->reserved); i++) {
+		if (args->reserved[i])
+			return -EINVAL;
+	}
+
+	/* Don't let fdget() silently truncate the fd. */
+	if (args->fd > INT_MAX)
+		return -EINVAL;
+
+	f = fdget(args->fd);
+	src_file = fd_file(f);
+	if (!src_file)
+		return -EINVAL;
+
+	ret = -EINVAL;
+	if (src_file->f_op != &kvm_gmem_fops)
+		goto out;
+
+	/* Cannot link a gmem file with the same VM again. */
+	gmem = src_file->private_data;
+	if (gmem->kvm == kvm)
+		goto out;
+
+	if (!kvm_arch_gmem_link_allowed(kvm, gmem->kvm))
+		goto out;
+
+	fd = get_unused_fd_flags(0);
+	if (fd < 0) {
+		ret = fd;
+		goto out;
+	}
+
+	inode = file_inode(src_file);
+	ihold(inode);
+
+	dst_file = kvm_gmem_alloc_file(kvm, inode, name);
+	if (IS_ERR(dst_file)) {
+		iput(inode);
+		ret = PTR_ERR(dst_file);
+		goto out_fd;
+	}
+
+	ts = inode_set_ctime_current(inode);
+	inode_set_atime_to_ts(inode, ts);
+
+	fd_install(fd, dst_file);
+	fdput(f);
+	return fd;
+
+out_fd:
+	put_unused_fd(fd);
+out:
+	fdput(f);
+	return ret;
+}
+
 int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot,
 				   unsigned int fd, uoff_t offset)
 {
diff --git a/virt/kvm/guest_memfd.h b/virt/kvm/guest_memfd.h
index 01bd359d27e30..7ffa4fa7337e9 100644
--- a/virt/kvm/guest_memfd.h
+++ b/virt/kvm/guest_memfd.h
@@ -8,6 +8,7 @@
 int kvm_gmem_init(struct module *module);
 void kvm_gmem_exit(void);
 int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args);
+int kvm_gmem_link(struct kvm *kvm, struct kvm_link_guest_memfd *args);
 int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot,
 				   unsigned int fd, uoff_t offset);
 int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 3ed481c4a15b6..70521371b64ab 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -5453,6 +5453,16 @@ static long kvm_vm_ioctl(struct file *filp,
 		r = kvm_gmem_create(kvm, &guest_memfd);
 		break;
 	}
+	case KVM_LINK_GUEST_MEMFD: {
+		struct kvm_link_guest_memfd params;
+
+		r = -EFAULT;
+		if (copy_from_user(&params, argp, sizeof(params)))
+			goto out;
+
+		r = kvm_gmem_link(kvm, &params);
+		break;
+	}
 #endif
 	default:
 		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
-- 
2.56.0.257.gf7497c7d27-goog