RE: [PATCH 1/1][v2] PCI: Support VF resizable BAR
From: Chang, HaiJun
Date: Fri May 31 2024 - 00:26:41 EST
[AMD Official Use Only - AMD Internal Distribution Only]
Hi, Bjorn and pci driver maintainers
Could you review the VF resizable BAR patch?
Thanks,
HaiJun
-----Original Message-----
From: Lianjie Shi <Lianjie.Shi@xxxxxxx>
Sent: Thursday, May 23, 2024 3:11 PM
To: linux-pci@xxxxxxxxxxxxxxx; linux-kernel@xxxxxxxxxxxxxxx; Bjorn Helgaas <bhelgaas@xxxxxxxxxx>
Cc: Chang, HaiJun <HaiJun.Chang@xxxxxxx>; Jiang, Jerry (SW) <Jerry.Jiang@xxxxxxx>; Zhang, Andy <Andy.Zhang@xxxxxxx>; Shi, Lianjie <Lianjie.Shi@xxxxxxx>
Subject: [PATCH 1/1][v2] PCI: Support VF resizable BAR
Add support for VF resizable BAR PCI extended cap.
Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init.
Based on following patch series:
Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/
Signed-off-by: Lianjie Shi <Lianjie.Shi@xxxxxxx>
---
drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++-
drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------
include/uapi/linux/pci_regs.h | 1 +
3 files changed, 85 insertions(+), 8 deletions(-)
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev)
}
}
+static void pci_restore_vf_rebar_state(struct pci_dev *pdev) { #ifdef
+CONFIG_PCI_IOV
+ unsigned int pos, nbars, i;
+ u32 ctrl;
+ u16 total;
+
+ pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR);
+ if (!pos)
+ return;
+
+ pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+ nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
+
+ for (i = 0; i < nbars; i++, pos += 8) {
+ struct resource *res;
+ int bar_idx, size;
+ u64 tmp;
+
+ pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+ bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
+ total = pdev->sriov->total_VFs;
+ if (!total)
+ return;
+
+ res = pdev->resource + bar_idx + PCI_IOV_RESOURCES;
+ tmp = resource_size(res);
+ do_div(tmp, total);
+ size = pci_rebar_bytes_to_size(tmp);
+ ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
+ ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
+ pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
+ }
+#endif
+}
+
/**
* pci_restore_state - Restore the saved state of a PCI device
* @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev)
pci_restore_ats_state(dev);
pci_restore_vc_state(dev);
pci_restore_rebar_state(dev);
+ pci_restore_vf_rebar_state(dev);
pci_restore_dpc_state(dev);
pci_restore_ptm_state(dev);
@@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev)
*/
static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) {
+ int cap = PCI_EXT_CAP_ID_REBAR;
unsigned int pos, nbars, i;
u32 ctrl;
- pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
+#ifdef CONFIG_PCI_IOV
+ if (bar >= PCI_IOV_RESOURCES) {
+ cap = PCI_EXT_CAP_ID_VF_REBAR;
+ bar -= PCI_IOV_RESOURCES;
+ }
+#endif
+
+ pos = pci_find_ext_capability(pdev, cap);
if (!pos)
return -ENOTSUPP;
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource);
+static int pci_memory_decoding(struct pci_dev *dev, int resno) {
+ u16 cmd;
+
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES) {
+ pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd);
+ if (cmd & PCI_SRIOV_CTRL_MSE)
+ return -EBUSY;
+ else
+ return 0;
+ }
+#endif
+ pci_read_config_word(dev, PCI_COMMAND, &cmd);
+ if (cmd & PCI_COMMAND_MEMORY)
+ return -EBUSY;
+
+ return 0;
+}
+
int pci_resize_resource(struct pci_dev *dev, int resno, int size) {
struct resource *res = dev->resource + resno;
struct pci_host_bridge *host;
int old, ret;
u32 sizes;
- u16 cmd;
/* Check if we must preserve the firmware's resource assignment */
host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
if (!(res->flags & IORESOURCE_UNSET))
return -EBUSY;
- pci_read_config_word(dev, PCI_COMMAND, &cmd);
- if (cmd & PCI_COMMAND_MEMORY)
- return -EBUSY;
+ ret = pci_memory_decoding(dev, resno);
+ if (ret)
+ return ret;
sizes = pci_rebar_get_possible_sizes(dev, resno);
if (!sizes)
@@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
if (ret)
return ret;
- res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES)
+ dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size);
+ else
+#endif
+ res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
/* Check if the new config works by trying to assign everything. */
if (dev->bus->self) {
ret = pci_reassign_bridge_resources(dev->bus->self, res->flags);
- if (ret)
+ if (ret && ret != -ENOENT)
goto error_resize;
}
return 0;
error_resize:
pci_rebar_set_size(dev, resno, old);
- res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
+
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES)
+ dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old);
+ else
+#endif
+ res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
+
return ret;
}
EXPORT_SYMBOL(pci_resize_resource);
diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644
--- a/include/uapi/linux/pci_regs.h
+++ b/include/uapi/linux/pci_regs.h
@@ -738,6 +738,7 @@
#define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */
#define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */
#define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */
+#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */
#define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */
#define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */
#define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */
--
2.34.1