[PATCH 10/10] PCI: liveupdate: Avoid enabling INTx on incoming devices

[David Matlack]

Avoid clearing PCI_COMMAND_INTX_DISABLE on incoming preserved devices.
Pass whether INTx masking is broken on each preserved device to the
next kernel instead of probing it, and skip clearing
PCI_COMMAND_INTX_DISABLE in do_pci_enable_device() for incoming
devices.

Clearing PCI_COMMAND_INTX_DISABLE enables INTx on the device, even if
only temporarily. pci_intx_mask_broken() probes whether INTx masking
works by toggling PCI_COMMAND_INTX_DISABLE during enumeration, and
do_pci_enable_device() clears it when a device that is not using MSI or
MSI-X is enabled. A preserved device may assert INTx at any point
during the Live Update, before its driver in the next kernel has set up
interrupt handling, so enabling INTx could result in interrupts that
no handler claims, on a line that may be shared with other devices.

Pass the result of probing as a new bit in the probed results in
struct pci_dev_ser, PCI_DEV_SER_PROBED_BROKEN_INTX_MASKING, which
reflects struct pci_dev::broken_intx_masking. If the previous kernel
did not pass it, fall back to probing INTx masking.

Drivers bound to incoming devices are responsible for enabling INTx once
they are ready to handle it. pci_alloc_irq_vectors() already does so
with pci_intx() when it falls back to INTx, and vfio-pci clears
PCI_COMMAND_INTX_DISABLE itself in vfio_pci_core_enable().

Assisted-by: LLM
Signed-off-by: David Matlack <dmatlack@xxxxxxxxxx>
---
 drivers/pci/liveupdate.c    | 37 ++++++++++++++++++++++++++++++++++++-
 drivers/pci/liveupdate.h    |  6 ++++++
 drivers/pci/pci.c           |  8 ++++++++
 drivers/pci/probe.c         | 10 ++++++++++
 include/linux/kho/abi/pci.h |  3 +++
 5 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/drivers/pci/liveupdate.c b/drivers/pci/liveupdate.c
index ac4525568818..8a74762c3f7d 100644
--- a/drivers/pci/liveupdate.c
+++ b/drivers/pci/liveupdate.c
@@ -216,6 +216,11 @@
  *    bridge implements), since that would disrupt memory transactions
  *    targeting the device or devices below it. Instead, the previous kernel
  *    passes the results of probing in struct pci_dev_ser.
+ *
+ *  * The PCI core does not enable INTx (i.e. clear PCI_COMMAND_INTX_DISABLE)
+ *    on incoming preserved devices, neither while probing whether INTx masking
+ *    works nor when the device is enabled, since the device could then raise
+ *    interrupts before its driver is ready to handle them.
  */
 
 #define pr_fmt(fmt) "PCI: liveupdate: " fmt
@@ -650,7 +655,11 @@ static void pci_liveupdate_save_bar_masks(struct pci_dev *dev,
 static void pci_liveupdate_preserve_probed(struct pci_dev *dev,
 					   struct pci_dev_ser *dev_ser)
 {
-	u64 supported = 0, probed = 0;
+	u64 supported = PCI_DEV_SER_PROBED_BROKEN_INTX_MASKING;
+	u64 probed = 0;
+
+	if (dev->broken_intx_masking)
+		probed |= PCI_DEV_SER_PROBED_BROKEN_INTX_MASKING;
 
 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
 		supported |= PCI_DEV_SER_PROBED_WINDOWS;
@@ -1213,6 +1222,32 @@ int pci_liveupdate_read_bridge_windows(struct pci_dev *bridge)
 	return 0;
 }
 
+/**
+ * pci_liveupdate_intx_mask_broken() - Get whether INTx masking is broken on an
+ *                                     incoming device
+ * @dev: The PCI device
+ *
+ * Get whether INTx masking is broken on @dev as determined by the previous
+ * kernel, so that INTx masking does not need to be probed.
+ *
+ * Return: 1 if INTx masking is broken, 0 if it is not, or -ENODATA if @dev is
+ * not incoming-preserved or the previous kernel did not pass whether INTx
+ * masking is broken on @dev.
+ */
+int pci_liveupdate_intx_mask_broken(struct pci_dev *dev)
+{
+	u64 probed;
+
+	guard(rwsem_read)(&pci_liveupdate.rwsem);
+
+	if (!pci_liveupdate_read_probed(dev,
+					PCI_DEV_SER_PROBED_BROKEN_INTX_MASKING,
+					&probed))
+		return -ENODATA;
+
+	return !!probed;
+}
+
 /*
  * Check if the windows of @bridge have been read from hardware. Outside of x86
  * that does not happen during scanning.
diff --git a/drivers/pci/liveupdate.h b/drivers/pci/liveupdate.h
index 98d23e217308..7f86cc85deee 100644
--- a/drivers/pci/liveupdate.h
+++ b/drivers/pci/liveupdate.h
@@ -22,6 +22,7 @@ bool pci_liveupdate_is_outgoing(struct pci_dev *dev);
 int pci_liveupdate_read_bar_masks(struct pci_dev *dev, unsigned int nr_bars,
 				  u32 *bar_masks, u32 *rom_mask);
 int pci_liveupdate_read_bridge_windows(struct pci_dev *bridge);
+int pci_liveupdate_intx_mask_broken(struct pci_dev *dev);
 void pci_liveupdate_claim_bus_resources(struct pci_bus *bus);
 #else
 static inline void pci_liveupdate_setup_device(struct pci_dev *dev)
@@ -74,6 +75,11 @@ static inline int pci_liveupdate_read_bridge_windows(struct pci_dev *bridge)
 	return -EINVAL;
 }
 
+static inline int pci_liveupdate_intx_mask_broken(struct pci_dev *dev)
+{
+	return -EINVAL;
+}
+
 static inline void pci_liveupdate_claim_bus_resources(struct pci_bus *bus)
 {
 }
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 853c84a2c782..43af72d1e5c9 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -2184,6 +2184,14 @@ static int do_pci_enable_device(struct pci_dev *dev, int bars)
 	if (dev->msi_enabled || dev->msix_enabled)
 		return 0;
 
+	/*
+	 * Leave INTx disabled on devices preserved across Live Update, since
+	 * the device could raise interrupts before its driver is ready. It is
+	 * up to the driver to enable INTx once it is ready.
+	 */
+	if (pci_liveupdate_is_incoming(dev))
+		return 0;
+
 	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
 	if (pin) {
 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index a53d0077a08b..badce31b34f8 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -1976,6 +1976,16 @@ static u8 pci_hdr_type(struct pci_dev *dev)
 static int pci_intx_mask_broken(struct pci_dev *dev)
 {
 	u16 orig, toggle, new;
+	int ret;
+
+	/*
+	 * Toggling PCI_COMMAND_INTX_DISABLE could let a device preserved across
+	 * Live Update raise interrupts before its driver is ready, so use the
+	 * result determined by the previous kernel for those devices.
+	 */
+	ret = pci_liveupdate_intx_mask_broken(dev);
+	if (ret >= 0)
+		return ret;
 
 	pci_read_config_word(dev, PCI_COMMAND, &orig);
 	toggle = orig ^ PCI_COMMAND_INTX_DISABLE;
diff --git a/include/linux/kho/abi/pci.h b/include/linux/kho/abi/pci.h
index 84a1bf71eeca..badcf2f01be7 100644
--- a/include/linux/kho/abi/pci.h
+++ b/include/linux/kho/abi/pci.h
@@ -95,10 +95,13 @@ struct pci_dev_ser {
  *    window.
  *  * PCI_DEV_SER_PROBED_PREF_64_WINDOW: The bridge's prefetchable memory
  *    window is 64-bit.
+ *  * PCI_DEV_SER_PROBED_BROKEN_INTX_MASKING: INTx masking cannot be used on
+ *    the device, i.e. struct pci_dev::broken_intx_masking is set.
  */
 #define PCI_DEV_SER_PROBED_IO_WINDOW		BIT_ULL(0)
 #define PCI_DEV_SER_PROBED_PREF_WINDOW		BIT_ULL(1)
 #define PCI_DEV_SER_PROBED_PREF_64_WINDOW	BIT_ULL(2)
+#define PCI_DEV_SER_PROBED_BROKEN_INTX_MASKING	BIT_ULL(3)
 
 /*
  * The size of struct pci_dev_ser when this ABI was introduced. Since fields
-- 
2.56.0.212.g129ede20d9-goog