[RFC PATCH v2 1/2] PCI: Allow hpmmiosize and hpmmioprefsize for specific hotplug bridges

[Maciej Grochowski]

From: Maciej Grochowski <maciej.grochowski@xxxxxxxx>

A hotplug bridge can be sized before devices expected behind it appear.
After a reset of a two-level PM50052 hierarchy, the nested switch's link
returns after its downstream port has received an equal share of the
parent's window. Its 1 GiB prefetchable NTB BAR then fails assignment:

  pci 0000:8c:00.1: BAR 2 [mem size 0x40000000 64bit pref]: can't assign; no space

The global hotplug memory sizes are optional targets for every hotplug
bridge. They cannot reserve the required space and head alignment on one
specific port when the parent aperture cannot satisfy all its siblings.

Allow hpmmiosize and hpmmioprefsize to take "<size>@<pci_dev>" entries,
separated by semicolons, using the existing PCI device selectors. For
example:

  pci=hpmmioprefsize=1032M@0000:80:01.1/00.0/03.0

Use the selected size as a required total window minimum in pbus_size_mem()
and request head alignment of the largest power of two not above that
size. Apply size and alignment independently, preserving stronger existing
requirements. A 1032 MiB minimum therefore requests 1 GiB head alignment.

Reject scoped values that exceed the existing unsigned long size interface
before narrowing them. Also guard window adjustment when there is no
optional realloc entry: a required minimum equal to the global size can
leave no entry to trim.

Retain the existing global optional-space policy. Assigned firmware
windows are kept; the minimum applies when the kernel sizes a window,
such as after hot-add or resource reallocation. Allocation remains subject
to the parent aperture and normal placement rules. A bare option keeps
its existing global meaning.

Assisted-by: Opus 5.5 (implementation/testing), GPT-6.1 (review)
Signed-off-by: Maciej Grochowski <maciej.grochowski@xxxxxxxx>
---
 .../admin-guide/kernel-parameters.txt         | 25 +++++++
 drivers/pci/pci.c                             | 75 ++++++++++++++++++-
 drivers/pci/pci.h                             |  3 +
 drivers/pci/setup-bus.c                       | 25 +++++++
 4 files changed, 126 insertions(+), 2 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 0b3c140babd9..19defbe2a1d6 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -5218,9 +5218,34 @@ Kernel parameters
 		hpmmiosize=nn[KMG]	The fixed amount of bus space which is
 				reserved for hotplug bridge's MMIO window.
 				Default size is 2 megabytes.
+		hpmmiosize=nn[KMG]@<pci_dev>[; ...]
+				Reserve at least nn bytes of MMIO window for
+				the hotplug bridges specified (in the format
+				described above), e.g. for a switch that is
+				expected behind the bridge but whose link
+				comes up only after the bridge windows have
+				been sized.  Unlike the default amount, which
+				is optional space that is dropped if it does
+				not fit, this is required space: the window
+				is sized like it would be for devices already
+				present below the bridge, with head alignment
+				from the largest power of two not above nn.
+				The size must fit unsigned long.  Only hotplug bridges
+				are affected, and only windows the kernel
+				sizes, e.g. after a hot-add or with
+				pci=realloc; windows set up by firmware are
+				kept.
 		hpmmioprefsize=nn[KMG]	The fixed amount of bus space which is
 				reserved for hotplug bridge's MMIO_PREF window.
 				Default size is 2 megabytes.
+		hpmmioprefsize=nn[KMG]@<pci_dev>[; ...]
+				Like hpmmiosize=nn[KMG]@<pci_dev>, for the
+				MMIO_PREF window.  For example,
+				  pci=hpmmioprefsize=1032M@0000:80:01.1/00.0/03.0
+				requests a 1032 MiB prefetchable window with
+				1 GiB head alignment below the downstream
+				port at devfn 03.0 of the switch behind root
+				port 0000:80:01.1.
 		hpmemsize=nn[KMG]	The fixed amount of bus space which is
 				reserved for hotplug bridge's MMIO and
 				MMIO_PREF window.
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 7c6f2aba8144..7a6ce1d3360d 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -412,6 +412,68 @@ static int pci_dev_str_match(struct pci_dev *dev, const char *p,
 	return 1;
 }
 
+/*
+ * "<size>@<pci_dev>" entries of "pci=hpmmiosize=" and "pci=hpmmioprefsize=",
+ * set in pci_setup() and copied in pci_realloc_setup_params().
+ */
+static const char *hotplug_mmio_size_param;
+static const char *hotplug_mmio_pref_size_param;
+
+/* Match @bridge against a "<size>@<pci_dev>[;...]" list; first match wins */
+static bool pci_hotplug_size_match(struct pci_dev *bridge, const char *p,
+				   unsigned long *size)
+{
+	unsigned long long val;
+	char *end;
+	int ret;
+
+	while (p && *p) {
+		val = memparse(p, &end);
+		if (val > ULONG_MAX)
+			return false;
+		p = end;
+		if (*p == '@') {
+			ret = pci_dev_str_match(bridge, p + 1, &p);
+			if (ret == 1) {
+				*size = val;
+				return true;
+			}
+			if (ret < 0) {
+				pr_err_once("PCI: Can't parse hotplug size parameter\n");
+				return false;
+			}
+		}
+
+		if (*p != ';')
+			break;
+		p++;
+	}
+
+	return false;
+}
+
+/**
+ * pci_hotplug_dev_size - get a hotplug size given for a specific bridge
+ * @bridge: the bridge
+ * @type: IORESOURCE_MEM, optionally with IORESOURCE_PREFETCH
+ * @size: set to the size given for @bridge
+ *
+ * Look up @bridge in the "<size>@<pci_dev>" entries of "pci=hpmmiosize=" or
+ * "pci=hpmmioprefsize=".  Only hotplug bridges are considered.
+ *
+ * Return: true if a size was given for @bridge.
+ */
+bool pci_hotplug_dev_size(struct pci_dev *bridge, unsigned long type,
+			  unsigned long *size)
+{
+	if (!bridge->is_hotplug_bridge || !(type & IORESOURCE_MEM))
+		return false;
+
+	return pci_hotplug_size_match(bridge, type & IORESOURCE_PREFETCH ?
+				      hotplug_mmio_pref_size_param :
+				      hotplug_mmio_size_param, size);
+}
+
 static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
 			      u8 pos, int cap)
 {
@@ -6892,9 +6954,15 @@ static int __init pci_setup(char *str)
 			} else if (!strncmp(str, "hpiosize=", 9)) {
 				pci_hotplug_io_size = memparse(str + 9, &str);
 			} else if (!strncmp(str, "hpmmiosize=", 11)) {
-				pci_hotplug_mmio_size = memparse(str + 11, &str);
+				if (strchr(str + 11, '@'))
+					hotplug_mmio_size_param = str + 11;
+				else
+					pci_hotplug_mmio_size = memparse(str + 11, &str);
 			} else if (!strncmp(str, "hpmmioprefsize=", 15)) {
-				pci_hotplug_mmio_pref_size = memparse(str + 15, &str);
+				if (strchr(str + 15, '@'))
+					hotplug_mmio_pref_size_param = str + 15;
+				else
+					pci_hotplug_mmio_pref_size = memparse(str + 15, &str);
 			} else if (!strncmp(str, "hpmemsize=", 10)) {
 				pci_hotplug_mmio_size = memparse(str + 10, &str);
 				pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size;
@@ -6942,6 +7010,9 @@ static int __init pci_realloc_setup_params(void)
 					   GFP_KERNEL);
 	disable_acs_redir_param = kstrdup(disable_acs_redir_param, GFP_KERNEL);
 	config_acs_param = kstrdup(config_acs_param, GFP_KERNEL);
+	hotplug_mmio_size_param = kstrdup(hotplug_mmio_size_param, GFP_KERNEL);
+	hotplug_mmio_pref_size_param = kstrdup(hotplug_mmio_pref_size_param,
+					       GFP_KERNEL);
 
 	return 0;
 }
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 77efa4c5c960..3b681653b5d9 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -419,6 +419,9 @@ extern unsigned long pci_hotplug_mmio_size;
 extern unsigned long pci_hotplug_mmio_pref_size;
 extern unsigned long pci_hotplug_bus_size;
 
+bool pci_hotplug_dev_size(struct pci_dev *bridge, unsigned long type,
+			  unsigned long *size);
+
 static inline bool pci_is_cardbus_bridge(struct pci_dev *dev)
 {
 	return dev->hdr_type == PCI_HEADER_TYPE_CARDBUS;
diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c
index ed16ef7c26fa..f3992859d90f 100644
--- a/drivers/pci/setup-bus.c
+++ b/drivers/pci/setup-bus.c
@@ -26,6 +26,7 @@
 #include <linux/ioport.h>
 #include <linux/cache.h>
 #include <linux/limits.h>
+#include <linux/log2.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
 #include <linux/acpi.h>
@@ -1285,6 +1286,7 @@ static void pbus_size_mem(struct pci_bus *bus, struct resource *b_res,
 	int order, max_order;
 	resource_size_t children_add_size = 0;
 	resource_size_t add_align = 0;
+	unsigned long hp_size;
 
 	if (!b_res)
 		return;
@@ -1349,6 +1351,27 @@ static void pbus_size_mem(struct pci_bus *bus, struct resource *b_res,
 	win_align = pci_min_window_alignment(bus, b_res->flags);
 	min_align = calculate_head_align(aligns, max_order);
 	min_align = max(min_align, win_align);
+
+	/*
+	 * A hotplug size given for this bridge is a required minimum, not
+	 * optional space like the global size: aligned for the largest BAR
+	 * that fits in it, so it is in place before the devices that need
+	 * it appear below the bridge.
+	 */
+	if (bus->self &&
+	    pci_hotplug_dev_size(bus->self, b_res->flags, &hp_size)) {
+		if (hp_size) {
+			size = max(size, hp_size);
+			min_align = max_t(resource_size_t, min_align,
+					  rounddown_pow_of_two(hp_size));
+			pci_info(bus->self, "bridge window to %pR: reserving %s size %#lx align %#llx\n",
+				 &bus->busn_res,
+				 b_res->flags & IORESOURCE_PREFETCH ?
+				 "prefetchable" : "non-prefetchable",
+				 hp_size, (unsigned long long)min_align);
+		}
+	}
+
 	size0 = calculate_memsize(size, realloc_head ? 0 : add_size,
 				  0, win_align);
 
@@ -1897,6 +1920,8 @@ static void adjust_bridge_window(struct pci_dev *bridge, struct resource *res,
 		}
 
 		dev_res = res_to_dev_res(add_list, res);
+		if (!dev_res)
+			return;
 		add_size = size - new_size;
 		if (add_size < dev_res->add_size) {
 			dev_res->add_size -= add_size;