[PATCH 2/3] PCI/IOV: Restore initial VF ReBAR sizes on SR-IOV disable/failure

[Marcin Bernatowicz]

PF drivers can resize a VF BAR via pci_iov_vf_bar_set_size(). The new
size persists in config space. A subsequent reprobe/unplug-rescan may
then cause the PCI core to reserve MMIO based on the inflated VF BAR
registers in the SR-IOV capability, often multiplied by TotalVFs.
On platforms with tight apertures, later SR-IOV enable can fail due to
lack of space.

Restore the initial VF BAR sizes saved at SR-IOV init time when SR-IOV
is disabled and when SR-IOV enable fails. Restore is only performed
when VF Memory Space is disabled, which is the precondition of
pci_iov_vf_bar_set_size() and is true at all the new call sites.

Note: this changes user-visible behavior for drivers that rely on the
resized VF BAR persisting across an enable/disable cycle. After this
change drivers must call pci_iov_vf_bar_set_size() before each
pci_enable_sriov() if a non-default size is desired.

Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/5937
Signed-off-by: Marcin Bernatowicz <marcin.bernatowicz@xxxxxxxxxxxxxxx>
---
 drivers/pci/iov.c | 61 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 54 insertions(+), 7 deletions(-)

diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 19f4dca4eec1..42b935265b3c 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -646,6 +646,40 @@ static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs)
 	return rc;
 }
 
+static void sriov_restore_vf_rebar_initial_sizes(struct pci_dev *dev)
+{
+	struct pci_sriov *iov = dev->sriov;
+	int i;
+
+	if (!iov || !iov->vf_rebar_cap)
+		return;
+
+	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
+		int resno = pci_resource_num_from_vf_bar(i);
+		resource_size_t orig_sz = iov->barsz_orig[i];
+		resource_size_t cur_sz = iov->barsz[i];
+		int size, cur, rc;
+
+		if (!orig_sz || cur_sz == orig_sz)
+			continue;
+
+		size = pci_rebar_bytes_to_size(orig_sz);
+		rc = pci_iov_vf_bar_set_size(dev, resno, size);
+		if (!rc)
+			continue;
+
+		pci_warn(dev, "failed to restore %s size %#llx -> %#llx: %d\n",
+			 pci_resource_name(dev, resno),
+			 (unsigned long long)cur_sz,
+			 (unsigned long long)orig_sz, rc);
+
+		/* Re-sync in-memory size with what hardware actually has. */
+		cur = pci_rebar_get_current_size(dev, resno);
+		if (cur >= 0)
+			iov->barsz[i] = pci_rebar_size_to_bytes(cur);
+	}
+}
+
 static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 {
 	int rc;
@@ -687,36 +721,42 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 	}
 	if (nres != iov->nres) {
 		pci_err(dev, "not enough MMIO resources for SR-IOV\n");
-		return -ENOMEM;
+		rc = -ENOMEM;
+		goto err_restore;
 	}
 
 	bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1);
 	if (bus > dev->bus->busn_res.end) {
 		pci_err(dev, "can't enable %d VFs (bus %02x out of range of %pR)\n",
 			nr_virtfn, bus, &dev->bus->busn_res);
-		return -ENOMEM;
+		rc = -ENOMEM;
+		goto err_restore;
 	}
 
 	if (pci_enable_resources(dev, bars)) {
 		pci_err(dev, "SR-IOV: IOV BARS not allocated\n");
-		return -ENOMEM;
+		rc = -ENOMEM;
+		goto err_restore;
 	}
 
 	if (iov->link != dev->devfn) {
 		pdev = pci_get_slot(dev->bus, iov->link);
-		if (!pdev)
-			return -ENODEV;
+		if (!pdev) {
+			rc = -ENODEV;
+			goto err_restore;
+		}
 
 		if (!pdev->is_physfn) {
 			pci_dev_put(pdev);
-			return -ENOSYS;
+			rc = -ENOSYS;
+			goto err_restore;
 		}
 
 		rc = sysfs_create_link(&dev->dev.kobj,
 					&pdev->dev.kobj, "dep_link");
 		pci_dev_put(pdev);
 		if (rc)
-			return rc;
+			goto err_restore;
 	}
 
 	iov->initial_VFs = initial;
@@ -758,6 +798,12 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 		sysfs_remove_link(&dev->dev.kobj, "dep_link");
 
 	pci_iov_set_numvfs(dev, 0);
+	sriov_restore_vf_rebar_initial_sizes(dev);
+	return rc;
+
+err_restore:
+	/* pcibios_sriov_enable() was not called on these early-exit paths. */
+	sriov_restore_vf_rebar_initial_sizes(dev);
 	return rc;
 }
 
@@ -791,6 +837,7 @@ static void sriov_disable(struct pci_dev *dev)
 
 	iov->num_VFs = 0;
 	pci_iov_set_numvfs(dev, 0);
+	sriov_restore_vf_rebar_initial_sizes(dev);
 }
 
 static int sriov_init(struct pci_dev *dev, int pos)
-- 
2.43.0