Re: [PATCH v6 20/27] cxl/pci: Add RCH downstream port AER register discovery

[Dave Jiang]

On 6/21/23 20:51, Terry Bowman wrote:
> Restricted CXL host (RCH) downstream port AER information is not currently
> logged while in the error state. One problem preventing the error logging
> is the AER and RAS registers are not accessible. The CXL driver requires
> changes to find RCH downstream port AER and RAS registers for purpose of
> error logging.
>
> RCH downstream ports are not enumerated during a PCI bus scan and are
> instead discovered using system firmware, ACPI in this case.[1] The
> downstream port is implemented as a Root Complex Register Block (RCRB).
> The RCRB is a 4k memory block containing PCIe registers based on the PCIe
> root port.[2] The RCRB includes AER extended capability registers used for
> reporting errors. Note, the RCH's AER Capability is located in the RCRB
> memory space instead of PCI configuration space, thus its register access
> is different. Existing kernel PCIe AER functions can not be used to manage
> the downstream port AER capabilities and RAS registers because the port was
> not enumerated during PCI scan and the registers are not PCI config
> accessible.
>
> Discover RCH downstream port AER extended capability registers. Use MMIO
> accesses to search for extended AER capability in RCRB register space.
>
> [1] CXL 3.0 Spec, 9.11.2 - System Firmware View of CXL 1.1 Hierarchy
> [2] CXL 3.0 Spec, 8.2.1.1 - RCH Downstream Port RCRB
>
> Co-developed-by: Robert Richter <rrichter@xxxxxxx>
> Signed-off-by: Robert Richter <rrichter@xxxxxxx>
> Signed-off-by: Terry Bowman <terry.bowman@xxxxxxx>
> Reviewed-by: Jonathan Cameron <Jonathan.Cameron@xxxxxxxxxx>

Reviewed-by: Dave Jiang <dave.jiang@xxxxxxxxx>

> ---
>   drivers/cxl/core/core.h |  1 +
>   drivers/cxl/core/port.c |  7 ++++++-
>   drivers/cxl/core/regs.c | 35 +++++++++++++++++++++++++++++++++++
>   3 files changed, 42 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
> index b001669a5133..87467c633123 100644
> --- a/drivers/cxl/core/core.h
> +++ b/drivers/cxl/core/core.h
> @@ -72,6 +72,7 @@ struct cxl_rcrb_info;
>   resource_size_t __rcrb_to_component(struct device *dev,
>   				    struct cxl_rcrb_info *ri,
>   				    enum cxl_rcrb which);
> +u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb);
>   
>   extern struct rw_semaphore cxl_dpa_rwsem;
>   
> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
> index cfae8307de90..37ecd756b224 100644
> --- a/drivers/cxl/core/port.c
> +++ b/drivers/cxl/core/port.c
> @@ -678,7 +678,6 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
>   	device_set_pm_not_required(dev);
>   	dev->bus = &cxl_bus_type;
>   	dev->type = &cxl_port_type;
> -

As Jonathan pointed out, stray blank line

DJ

>   	return port;
>   
>   err:
> @@ -974,6 +973,8 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
>   		return ERR_PTR(-ENOMEM);
>   
>   	if (rcrb != CXL_RESOURCE_NONE) {
> +		struct pci_host_bridge *host_bridge;
> +
>   		dport->rcrb.base = rcrb;
>   		component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
>   							 CXL_RCRB_DOWNSTREAM);
> @@ -982,6 +983,10 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
>   			return ERR_PTR(-ENXIO);
>   		}
>   
> +		host_bridge = to_pci_host_bridge(dport_dev);
> +		if (host_bridge->native_cxl_error)
> +			dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base);
> +
>   		dport->rch = true;
>   	}
>   
> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
> index e68848075bb6..60b7ba7f4030 100644
> --- a/drivers/cxl/core/regs.c
> +++ b/drivers/cxl/core/regs.c
> @@ -407,6 +407,41 @@ int cxl_setup_regs(struct cxl_register_map *map)
>   }
>   EXPORT_SYMBOL_NS_GPL(cxl_setup_regs, CXL);
>   
> +u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb)
> +{
> +	void __iomem *addr;
> +	u16 offset = 0;
> +	u32 cap_hdr;
> +
> +	if (WARN_ON_ONCE(rcrb == CXL_RESOURCE_NONE))
> +		return 0;
> +
> +	if (!request_mem_region(rcrb, SZ_4K, dev_name(dev)))
> +		return 0;
> +
> +	addr = ioremap(rcrb, SZ_4K);
> +	if (!addr) {
> +		release_mem_region(rcrb, SZ_4K);
> +		return 0;
> +	}
> +
> +	cap_hdr = readl(addr + offset);
> +	while (PCI_EXT_CAP_ID(cap_hdr) != PCI_EXT_CAP_ID_ERR) {
> +		offset = PCI_EXT_CAP_NEXT(cap_hdr);
> +		if (!offset)
> +			break;
> +		cap_hdr = readl(addr + offset);
> +	}
> +
> +	if (offset)
> +		dev_dbg(dev, "found AER extended capability (0x%x)\n", offset);
> +
> +	iounmap(addr);
> +	release_mem_region(rcrb, SZ_4K);
> +
> +	return offset;
> +}
> +
>   resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri,
>   				    enum cxl_rcrb which)
>   {