Re: [PATCH v7 05/18] iommu: Implement device and IOMMU HW preservation

From: Baolu Lu

Date: Mon Nov 16 2026 - 05:19:28 EST


On 11/10/2026 12:08 PM, Samiullah Khawaja wrote:
Add IOMMU ops to preserve/unpreserve the device state handled by the
IOMMU driver. This is IOMMU specific state of the device that needs to
be preserved for liveupdate. These ops can be implemented by the IOMMU
drivers that support preservation of devices that have their IOMMU
domains preserved. During device preservation the state of the
associated IOMMU is also preserved as dependency.

Signed-off-by: Samiullah Khawaja <skhawaja@xxxxxxxxxx>
---
drivers/iommu/iommu-priv.h | 2 +
drivers/iommu/iommu.c | 32 +++++-
drivers/iommu/liveupdate.c | 191 +++++++++++++++++++++++++++++++
include/linux/iommu-liveupdate.h | 76 ++++++++++++
include/linux/iommu.h | 21 ++++
5 files changed, 321 insertions(+), 1 deletion(-)

diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h
index 1f9f886b2c10..8a14d38a3e70 100644
--- a/drivers/iommu/iommu-priv.h
+++ b/drivers/iommu/iommu-priv.h
@@ -21,6 +21,8 @@ static inline const struct iommu_ops *dev_iommu_ops(struct device *dev)
void dev_iommu_free(struct device *dev);
+size_t iommu_group_device_count(struct iommu_group *group);
+
const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode);
static inline const struct iommu_ops *iommu_fwspec_ops(struct iommu_fwspec *fwspec)
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index b486b8bbd1fc..932325e0af41 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -18,6 +18,7 @@
#include <linux/errno.h>
#include <linux/host1x_context_bus.h>
#include <linux/iommu.h>
+#include <linux/iommu-liveupdate.h>
#include <linux/iommufd.h>
#include <linux/idr.h>
#include <linux/err.h>
@@ -626,8 +627,8 @@ DEFINE_MUTEX(iommu_probe_device_lock);
static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
{
+ struct group_device *gdev, *gdev2;
struct iommu_group *group;
- struct group_device *gdev;
int ret;
/*
@@ -661,6 +662,13 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
goto err_put_group;
}
+ for_each_group_device(group, gdev2) {
+ if (dev_iommu_has_preserved_state(gdev2->dev)) {
+ ret = -EBUSY;
+ goto err_free_gdev;
+ }
+ }
+
/*
* The gdev must be in the list before calling
* iommu_setup_default_domain()
@@ -697,6 +705,7 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
err_remove_gdev:
list_del(&gdev->list);
+err_free_gdev:
__iommu_group_free_device(group, gdev);
err_put_group:
iommu_deinit_device(dev);
@@ -3543,6 +3552,27 @@ bool iommu_group_dma_owner_claimed(struct iommu_group *group)
}
EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
+/**
+ * iommu_group_device_count() - Get the number of devices in a group
+ * @group: The group.
+ *
+ * The device count may change after this function returns, as devices can
+ * be added to or removed from the group. The caller must recheck it if it
+ * needs the count to stay the same.
+ *
+ * Return: the number of devices in the group.
+ */
+size_t iommu_group_device_count(struct iommu_group *group)
+{
+ size_t count;
+
+ mutex_lock(&group->mutex);
+ count = list_count_nodes(&group->devices);
+ mutex_unlock(&group->mutex);
+
+ return count;
+}
+
static void iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
struct iommu_domain *domain)
{

[...]

+/**
+ * iommu_preserve_device() - Preserve device state across live update
+ * @domain: Associated IOMMU domain
+ * @dev: Device to preserve
+ * @dma_owner_token: Token to identify DMA owner of this device
+ *
+ * Return: 0 on success, or negative error code.
+ */
+int iommu_preserve_device(struct iommu_domain *domain,
+ struct device *dev, u64 dma_owner_token)
+{
+ struct iommu_device_ser *device_ser;
+ struct iommu_flb_obj *flb_obj;
+ struct dev_iommu *iommu;
+ struct pci_dev *pdev;
+ int ret;
+
+ if (!dev_is_pci(dev))
+ return -EOPNOTSUPP;
+
+ if (!iommu_group_dma_owner_claimed(dev->iommu_group))
+ return -EINVAL;
+
+ if (iommu_group_device_count(dev->iommu_group) != 1)
+ return -EOPNOTSUPP;
+
+ if (iommu_get_domain_for_dev(dev) != domain)
+ return -EINVAL;
+
+ pdev = to_pci_dev(dev);
+ iommu = dev->iommu;
+ if (!iommu->iommu_dev->ops->preserve_device ||
+ !iommu->iommu_dev->ops->unpreserve_device ||
+ !iommu->iommu_dev->ops->preserve ||
+ !iommu->iommu_dev->ops->unpreserve)
+ return -EOPNOTSUPP;
+
+ ret = liveupdate_flb_get_outgoing(&iommu_flb, (void **)&flb_obj);
+ if (ret)
+ return ret;
+
+ mutex_lock(&flb_obj->lock);
+ if (!domain->domain_ser) {
+ ret = -EINVAL;
+ goto out_unlock;
+ }
+
+ if (dev_iommu_preserved_state(dev)) {
+ ret = -EBUSY;
+ goto out_unlock;
+ }
+
+ device_ser = iommu_ser_set_alloc(&flb_obj->device_set);
+ if (IS_ERR(device_ser)) {
+ ret = PTR_ERR(device_ser);
+ goto out_unlock;
+ }
+
+ ret = iommu_preserve_locked(iommu->iommu_dev, flb_obj);
+ if (ret) {
+ device_ser->hdr.flags |= IOMMU_SER_FLAG_DELETED;
+ goto out_unlock;
+ }
+
+ device_ser->attachment.domain_phys = virt_to_phys(domain->domain_ser);
+ device_ser->iommu_phys = virt_to_phys(iommu->iommu_dev->iommu_ser);
+ device_ser->devid = pci_dev_id(pdev);
+ device_ser->pci_domain_nr = pci_domain_nr(pdev->bus);
+ device_ser->dma_owner_token = dma_owner_token;
+
+ ret = iommu->iommu_dev->ops->preserve_device(dev, device_ser);
+ if (!ret) {
+ WRITE_ONCE(dev->iommu->device_ser, device_ser);
+
+ /* Validate that no sibling device was added concurrently */
+ if (iommu_group_device_count(dev->iommu_group) != 1) {
+ iommu->iommu_dev->ops->unpreserve_device(dev, device_ser);
+ WRITE_ONCE(dev->iommu->device_ser, NULL);
+ ret = -EOPNOTSUPP;
+ }
+ }
+
+ if (ret) {
+ device_ser->hdr.flags |= IOMMU_SER_FLAG_DELETED;
+ iommu_unpreserve_locked(iommu->iommu_dev, flb_obj);
+ goto out_unlock;
+ }
+
+out_unlock:
+ mutex_unlock(&flb_obj->lock);
+ liveupdate_flb_put_outgoing(&iommu_flb);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(iommu_preserve_device);
Only devices in singleton iommu groups can be preserved, and no other
device may join the group while preservation is in progress or the
device remains preserved.

Could we simplify this by adding a flag to struct iommu_group, protected
by group->mutex? For example:

@@ -72,6 +72,9 @@ struct iommu_group {
*/
unsigned int recovery_cnt;
void *owner;
+#ifdef CONFIG_IOMMU_LIVEUPDATE
+ bool liveupdate_reserved;
+#endif
};

A helper could check that the group contains exactly one device and set
the flag atomically under group->mutex. Device probing would reject an
addition if the flag is set.

#ifdef CONFIG_IOMMU_LIVEUPDATE
int iommu_group_reserve_liveupdate(struct iommu_group *group)
{
int ret = 0;

mutex_lock(&group->mutex);
if (group->liveupdate_reserved)
ret = -EBUSY;
else if (list_count_nodes(&group->devices) != 1)
ret = -EOPNOTSUPP;
else
group->liveupdate_reserved = true;
mutex_unlock(&group->mutex);

return ret;
}

void iommu_group_unreserve_liveupdate(struct iommu_group *group)
{
mutex_lock(&group->mutex);
group->liveupdate_reserved = false;
mutex_unlock(&group->mutex);
}
#endif

The reservation could be taken before calling the driver's preserve
callback and cleared on every failure path and after unpreservation.
This would avoid the second device-count check.

Any thoughts?

Thanks,
baolu