[PATCH v19 11/14] PCI: Cache PCI DSN into pci_dev->dsn during probe

[Terry Bowman]

Subsequent CXL error-reporting code paths need to log the PCI Device
Serial Number (DSN) as part of trace events emitted from interrupt or
panic context. Computing the DSN there via pci_get_dsn() requires PCI
configuration space reads, which are slow, can fail when the link is
down or frozen, and may not be safe in some contexts.

Add a u64 dsn field to struct pci_dev and populate it from pci_get_dsn()
during pci_init_capabilities() at probe time via pci_dsn_init(). Only
write dev->dsn when the read succeeds. The zero initial value from
pci_dev allocation already represents 'no DSN available.'

Remove the now-redundant dsn member from the pciehp struct controller
along with its kernel-doc. Drop the pci_get_slot()/pci_dev_put() pairs
in pciehp_configure_device() and pcie_init() that existed solely to
read the DSN into ctrl->dsn. Use pdev->dsn in pciehp_device_replaced()
for the device-replacement comparison.

pci_get_dsn() is not modified because it remains a pure config-space read
with no side effects on pci_dev. The cache is written exclusively by
pci_dsn_init() at probe time.

Signed-off-by: Terry Bowman <terry.bowman@xxxxxxx>
Reviewed-by: Dave Jiang <dave.jiang@xxxxxxxxx>
Reviewed-by: Jonathan Cameron <jonathan.cameron@xxxxxxxxxxxxxxxx>

---

Changes in v18 -> v19:
- Update pciehp hotplug to use cached DSN.
- Swapped order in series with
("cxl: Add port and dport identifiers to CXL AER trace events ")
- Update function documentation for pci_dsn_init()

Changes in v17->v18:
- New commit.
---
 drivers/cxl/pci.c                |  2 +-
 drivers/pci/hotplug/pciehp.h     |  4 ----
 drivers/pci/hotplug/pciehp_hpc.c |  7 +------
 drivers/pci/hotplug/pciehp_pci.c |  4 ----
 drivers/pci/probe.c              | 17 +++++++++++++++++
 include/linux/pci.h              |  1 +
 6 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 46e60c53d3ca4..cb256d194de28 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -807,7 +807,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (!dvsec)
 		pci_warn(pdev, "Device DVSEC not present, skip CXL.mem init\n");
 
-	mds = cxl_memdev_state_create(&pdev->dev, pci_get_dsn(pdev), dvsec);
+	mds = cxl_memdev_state_create(&pdev->dev, pdev->dsn, dvsec);
 	if (IS_ERR(mds))
 		return PTR_ERR(mds);
 	cxlds = &mds->cxlds;
diff --git a/drivers/pci/hotplug/pciehp.h b/drivers/pci/hotplug/pciehp.h
index debc79b0adfb2..30aad3b66bab9 100644
--- a/drivers/pci/hotplug/pciehp.h
+++ b/drivers/pci/hotplug/pciehp.h
@@ -46,9 +46,6 @@ extern int pciehp_poll_time;
 /**
  * struct controller - PCIe hotplug controller
  * @pcie: pointer to the controller's PCIe port service device
- * @dsn: cached copy of Device Serial Number of Function 0 in the hotplug slot
- *	(PCIe r6.2 sec 7.9.3); used to determine whether a hotplugged device
- *	was replaced with a different one during system sleep
  * @slot_cap: cached copy of the Slot Capabilities register
  * @inband_presence_disabled: In-Band Presence Detect Disable supported by
  *	controller and disabled per spec recommendation (PCIe r5.0, appendix I
@@ -90,7 +87,6 @@ extern int pciehp_poll_time;
  */
 struct controller {
 	struct pcie_device *pcie;
-	u64 dsn;
 
 	u32 slot_cap;				/* capabilities and quirks */
 	unsigned int inband_presence_disabled:1;
diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c
index 4c62140a3cb44..5336e9c9003ae 100644
--- a/drivers/pci/hotplug/pciehp_hpc.c
+++ b/drivers/pci/hotplug/pciehp_hpc.c
@@ -587,7 +587,7 @@ bool pciehp_device_replaced(struct controller *ctrl)
 	     reg != (pdev->subsystem_vendor | (pdev->subsystem_device << 16))))
 		return true;
 
-	if (pci_get_dsn(pdev) != ctrl->dsn)
+	if (pci_get_dsn(pdev) != pdev->dsn)
 		return true;
 
 	return false;
@@ -1085,11 +1085,6 @@ struct controller *pcie_init(struct pcie_device *dev)
 		}
 	}
 
-	pdev = pci_get_slot(subordinate, PCI_DEVFN(0, 0));
-	if (pdev)
-		ctrl->dsn = pci_get_dsn(pdev);
-	pci_dev_put(pdev);
-
 	return ctrl;
 }
 
diff --git a/drivers/pci/hotplug/pciehp_pci.c b/drivers/pci/hotplug/pciehp_pci.c
index 65e50bee1a8c0..ad12515a4a121 100644
--- a/drivers/pci/hotplug/pciehp_pci.c
+++ b/drivers/pci/hotplug/pciehp_pci.c
@@ -72,10 +72,6 @@ int pciehp_configure_device(struct controller *ctrl)
 	pci_bus_add_devices(parent);
 	down_read_nested(&ctrl->reset_lock, ctrl->depth);
 
-	dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
-	ctrl->dsn = pci_get_dsn(dev);
-	pci_dev_put(dev);
-
  out:
 	pci_unlock_rescan_remove();
 	return ret;
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index dd0abbc63e18d..450e5090564e2 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -2638,6 +2638,22 @@ void pcie_report_downtraining(struct pci_dev *dev)
 	__pcie_print_link_status(dev, false);
 }
 
+/*
+ * Cache the Device Serial Number for use in contexts where config-space reads
+ * are unsafe (interrupt, panic).  Process-context callers that need a fresh
+ * value (e.g. hotplug device replacement) call pci_get_dsn() and compare it
+ * against this cached pdev->dsn to detect a changed device.  Note pdev->dsn
+ * is 0 for devices without the DSN capability, so such a comparison cannot
+ * distinguish a replacement.
+ */
+static void pci_dsn_init(struct pci_dev *dev)
+{
+	u64 dsn = pci_get_dsn(dev);
+
+	if (dsn)
+		dev->dsn = dsn;
+}
+
 static void pci_imm_ready_init(struct pci_dev *dev)
 {
 	u16 status;
@@ -2674,6 +2690,7 @@ static void pci_init_capabilities(struct pci_dev *dev)
 	pci_rebar_init(dev);		/* Resizable BAR */
 	pci_dev3_init(dev);		/* Device 3 capabilities */
 	pci_ide_init(dev);		/* Link Integrity and Data Encryption */
+	pci_dsn_init(dev);		/* Serial number */
 
 	pcie_report_downtraining(dev);
 	pci_init_reset_methods(dev);
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 64b308b6e61c1..48a1622639190 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -386,6 +386,7 @@ struct pci_dev {
 	unsigned long	*dma_alias_mask;/* Mask of enabled devfn aliases */
 
 	struct pci_driver *driver;	/* Driver bound to this device */
+	u64		dsn;		/* PCI Device Serial Number */
 	u64		dma_mask;	/* Mask of the bits of bus address this
 					   device implements.  Normally this is
 					   0xffffffff.  You only need to change
-- 
2.34.1