[RFC PATCH v4 8/8] bus/cdx: add cdx controller

[Nipun Gupta]

Add driver for CDX controller. The controller discovers the
CDX devices with the help of firmware and add those devices
to the CDX bus.

CDX controller also provide ops to be registered to CDX
bus for scanning, resetting and writing MSI data to CDX
devices.

Signed-off-by: Nipun Gupta <nipun.gupta@xxxxxxx>
---
 drivers/bus/cdx/Kconfig                     |   2 +
 drivers/bus/cdx/Makefile                    |   2 +-
 drivers/bus/cdx/controller/Kconfig          |  16 ++
 drivers/bus/cdx/controller/Makefile         |   8 +
 drivers/bus/cdx/controller/cdx_controller.c | 210 ++++++++++++++++++++
 drivers/bus/cdx/controller/mcdi_stubs.c     |  68 +++++++
 drivers/bus/cdx/controller/mcdi_stubs.h     | 101 ++++++++++
 7 files changed, 406 insertions(+), 1 deletion(-)
 create mode 100644 drivers/bus/cdx/controller/Kconfig
 create mode 100644 drivers/bus/cdx/controller/Makefile
 create mode 100644 drivers/bus/cdx/controller/cdx_controller.c
 create mode 100644 drivers/bus/cdx/controller/mcdi_stubs.c
 create mode 100644 drivers/bus/cdx/controller/mcdi_stubs.h

diff --git a/drivers/bus/cdx/Kconfig b/drivers/bus/cdx/Kconfig
index 062443080d6f..8036d754a0a6 100644
--- a/drivers/bus/cdx/Kconfig
+++ b/drivers/bus/cdx/Kconfig
@@ -12,3 +12,5 @@ config CDX_BUS
 		Driver to enable CDX Bus infrastructure. CDX bus uses
 		CDX controller and firmware to scan the FPGA based
 		devices.
+
+source "drivers/bus/cdx/controller/Kconfig"
diff --git a/drivers/bus/cdx/Makefile b/drivers/bus/cdx/Makefile
index 95f8fa3b4db8..285fd7d51174 100644
--- a/drivers/bus/cdx/Makefile
+++ b/drivers/bus/cdx/Makefile
@@ -5,4 +5,4 @@
 # Copyright (C) 2022, Advanced Micro Devices, Inc.
 #
 
-obj-$(CONFIG_CDX_BUS) += cdx.o cdx_msi.o
+obj-$(CONFIG_CDX_BUS) += cdx.o cdx_msi.o controller/
diff --git a/drivers/bus/cdx/controller/Kconfig b/drivers/bus/cdx/controller/Kconfig
new file mode 100644
index 000000000000..29bd45626d38
--- /dev/null
+++ b/drivers/bus/cdx/controller/Kconfig
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# CDX controller configuration
+#
+# Copyright (C) 2022, Advanced Micro Devices, Inc.
+#
+
+config CDX_CONTROLLER
+	tristate "CDX hardware driver"
+	depends on CDX_BUS
+	help
+	  CDX controller drives the CDX bus. It interacts with
+	  firmware to get the hardware devices and registers with
+	  the CDX bus. Say Y to enable the CDX hardware driver.
+
+	  If unsure, say N.
diff --git a/drivers/bus/cdx/controller/Makefile b/drivers/bus/cdx/controller/Makefile
new file mode 100644
index 000000000000..83bcd4ae1874
--- /dev/null
+++ b/drivers/bus/cdx/controller/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for CDX controller drivers
+#
+# Copyright (C) 2022, Advanced Micro Devices, Inc.
+#
+
+obj-$(CONFIG_CDX_CONTROLLER) += cdx_controller.o mcdi_stubs.o
diff --git a/drivers/bus/cdx/controller/cdx_controller.c b/drivers/bus/cdx/controller/cdx_controller.c
new file mode 100644
index 000000000000..792fbb8314d5
--- /dev/null
+++ b/drivers/bus/cdx/controller/cdx_controller.c
@@ -0,0 +1,210 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Platform driver for CDX bus.
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/of_platform.h>
+#include <linux/cdx/cdx_bus.h>
+
+#include "../cdx.h"
+#include "mcdi_stubs.h"
+
+static int cdx_reset_device(struct cdx_controller_t *cdx,
+			    uint8_t bus_num, uint8_t dev_num)
+{
+	return cdx_mcdi_reset_dev(cdx->priv, bus_num, dev_num);
+}
+
+static int cdx_scan_devices(struct cdx_controller_t *cdx)
+{
+	struct cdx_mcdi_t *cdx_mcdi = cdx->priv;
+	int num_cdx_bus, num_cdx_dev;
+	uint8_t bus_num, dev_num;
+	int ret;
+
+	/* MCDI FW Read: Fetch the number of CDX buses present*/
+	num_cdx_bus = cdx_mcdi_get_num_buses(cdx_mcdi);
+
+	for (bus_num = 0; bus_num < num_cdx_bus; bus_num++) {
+		/* MCDI FW Read: Fetch the number of devices present */
+		num_cdx_dev = cdx_mcdi_get_num_devs(cdx_mcdi, bus_num);
+
+		for (dev_num = 0; dev_num < num_cdx_dev; dev_num++) {
+			struct cdx_dev_params_t dev_params;
+
+			/* MCDI FW: Get the device config */
+			ret = cdx_mcdi_get_dev_config(cdx_mcdi, bus_num,
+						      dev_num, &dev_params);
+			if (ret) {
+				dev_err(cdx->dev,
+					"CDX device config get failed for bus: %d\n", ret);
+				return ret;
+			}
+			dev_params.cdx = cdx;
+
+			/* Add the device to the cdx bus */
+			ret = cdx_device_add(&dev_params);
+			if (ret) {
+				dev_err(cdx->dev, "registering cdx dev: %d failed: %d\n",
+					dev_num, ret);
+				return ret;
+			}
+
+			dev_dbg(cdx->dev, "CDX dev: %d on cdx bus: %d created\n",
+				dev_num, bus_num);
+		}
+	}
+
+	return 0;
+}
+
+static int cdx_write_msi(struct cdx_controller_t *cdx, uint8_t bus_num,
+			 uint8_t dev_num, uint16_t msi_index,
+			 uint32_t data, uint64_t addr)
+{
+	return cdx_mcdi_write_msi(cdx->priv, bus_num, dev_num,
+				  msi_index, data, addr);
+}
+
+static int cdx_probe(struct platform_device *pdev)
+{
+	struct cdx_mcdi_t *cdx_mcdi;
+	struct cdx_controller_t *cdx;
+	struct device_node *iommu_node;
+	struct platform_device *plat_iommu_dev;
+	int ret;
+
+	/*
+	 * Defer probe CDX controller in case IOMMU is not yet initialized.
+	 * We do not add CDX devices before the IOMMU device is probed, as
+	 * in cleanup or shutdown time (device_shutdonw()), which traverse
+	 * device list in reverse order of device addition, we need to remove
+	 * CDX devices, before IOMMU device is removed.
+	 */
+	iommu_node = of_parse_phandle(pdev->dev.of_node, "iommu-map", 1);
+	if (!of_device_is_available(iommu_node)) {
+		dev_err(&pdev->dev, "No associated IOMMU node found\n");
+		return -EINVAL;
+	}
+
+	plat_iommu_dev = of_find_device_by_node(iommu_node);
+	if (!plat_iommu_dev) {
+		ret = -EPROBE_DEFER;
+		goto iommu_of_cleanup;
+	}
+
+	if (!plat_iommu_dev->dev.driver) {
+		ret = -EPROBE_DEFER;
+		goto iommu_of_cleanup;
+	}
+
+	cdx_mcdi = kzalloc(sizeof(*cdx_mcdi), GFP_KERNEL);
+	if (!cdx_mcdi) {
+		dev_err(&pdev->dev, "Failed to allocate memory for cdx_mcdi\n");
+		ret = -ENOMEM;
+		goto iommu_of_cleanup;
+	}
+
+	/* MCDI FW: Initialize the FW path */
+	ret = cdx_mcdi_init(cdx_mcdi);
+	if (ret) {
+		dev_err(&pdev->dev, "MCDI Initialization failed: %d\n", ret);
+		goto mcdi_init_fail;
+	}
+
+	cdx = kzalloc(sizeof(*cdx), GFP_KERNEL);
+	if (!cdx) {
+		dev_err(&pdev->dev, "Failed to allocate memory for cdx\n");
+		ret = -ENOMEM;
+		goto cdx_alloc_fail;
+	}
+	platform_set_drvdata(pdev, cdx);
+
+	cdx->dev = &pdev->dev;
+	cdx->priv = cdx_mcdi;
+	cdx->ops.scan = cdx_scan_devices;
+	cdx->ops.reset_dev = cdx_reset_device;
+	cdx->ops.write_msi = cdx_write_msi;
+
+	/* Register CDX controller with CDX bus driver */
+	ret = cdx_register_controller(cdx);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to register CDX controller\n");
+		goto cdx_register_fail;
+	}
+
+	return 0;
+
+cdx_register_fail:
+	kfree(cdx);
+cdx_alloc_fail:
+	cdx_mcdi_finish(cdx_mcdi);
+mcdi_init_fail:
+	kfree(cdx_mcdi);
+iommu_of_cleanup:
+	of_node_put(iommu_node);
+
+	return ret;
+}
+
+static int cdx_remove(struct platform_device *pdev)
+{
+	struct cdx_controller_t *cdx = platform_get_drvdata(pdev);
+	struct cdx_mcdi_t *cdx_mcdi = cdx->priv;
+
+	cdx_unregister_controller(cdx);
+	kfree(cdx);
+
+	cdx_mcdi_finish(cdx_mcdi);
+	kfree(cdx_mcdi);
+
+	return 0;
+}
+
+static void cdx_shutdown(struct platform_device *pdev)
+{
+	cdx_remove(pdev);
+}
+
+static const struct of_device_id cdx_match_table[] = {
+	{.compatible = "xlnx,cdxbus-controller-1.0",},
+	{ },
+};
+
+MODULE_DEVICE_TABLE(of, cdx_match_table);
+
+static struct platform_driver cdx_pdriver = {
+	.driver = {
+		   .name = "cdx-controller",
+		   .pm = NULL,
+		   .of_match_table = cdx_match_table,
+		   },
+	.probe = cdx_probe,
+	.remove = cdx_remove,
+	.shutdown = cdx_shutdown,
+};
+
+static int __init cdx_controller_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&cdx_pdriver);
+	if (ret < 0)
+		pr_err("platform_driver_register() failed: %d\n", ret);
+
+	return ret;
+}
+module_init(cdx_controller_init);
+
+static void __exit cdx_controller_exit(void)
+{
+	platform_driver_unregister(&cdx_pdriver);
+}
+module_exit(cdx_controller_exit);
diff --git a/drivers/bus/cdx/controller/mcdi_stubs.c b/drivers/bus/cdx/controller/mcdi_stubs.c
new file mode 100644
index 000000000000..17737ffeae1c
--- /dev/null
+++ b/drivers/bus/cdx/controller/mcdi_stubs.c
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * MCDI Firmware interaction for CDX bus.
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/ioport.h>
+
+#include "mcdi_stubs.h"
+
+int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi)
+{
+	cdx_mcdi->id = 0;
+	cdx_mcdi->flags = 0;
+
+	return 0;
+}
+
+void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi)
+{
+}
+
+int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi)
+{
+	return 1;
+}
+
+int cdx_mcdi_get_num_devs(struct cdx_mcdi_t *cdx_mcdi, int bus_num)
+{
+	return 1;
+}
+
+int cdx_mcdi_get_dev_config(struct cdx_mcdi_t *cdx_mcdi,
+			    uint8_t bus_num, uint8_t dev_num,
+			    struct cdx_dev_params_t *dev_params)
+{
+	dev_params->res[0].start = 0xe4020000;
+	dev_params->res[0].end = 0xe4020FFF;
+	dev_params->res[0].flags = IORESOURCE_MEM;
+	dev_params->res[1].start = 0xe4100000;
+	dev_params->res[1].end = 0xE411FFFF;
+	dev_params->res[1].flags = IORESOURCE_MEM;
+	dev_params->res_count = 2;
+
+	dev_params->req_id = 0x250;
+	dev_params->num_msi = 4;
+	dev_params->vendor = 0x10ee;
+	dev_params->device = 0x8084;
+	dev_params->bus_num = bus_num;
+	dev_params->dev_num = dev_num;
+
+	return 0;
+}
+
+int cdx_mcdi_reset_dev(struct cdx_mcdi_t *cdx_mcdi,
+		       uint8_t bus_num, uint8_t dev_num)
+{
+	return 0;
+}
+
+int cdx_mcdi_write_msi(struct cdx_mcdi_t *cdx_mcdi,
+		       uint8_t bus_num, uint8_t dev_num,
+		       uint16_t msi_index, uint32_t data,
+		       uint64_t addr)
+{
+	return 0;
+}
diff --git a/drivers/bus/cdx/controller/mcdi_stubs.h b/drivers/bus/cdx/controller/mcdi_stubs.h
new file mode 100644
index 000000000000..3256f0e306fa
--- /dev/null
+++ b/drivers/bus/cdx/controller/mcdi_stubs.h
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Header file for MCDI FW interaction for CDX bus.
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _MCDI_STUBS_H_
+#define _MCDI_STUBS_H_
+
+#include "../cdx.h"
+
+/**
+ * struct cdx_mcdi_t - CDX MCDI Firmware interface, to interact
+ *		       with CDX controller.
+ * @id: ID for MCDI Firmware interface
+ * @flags: Associated flags
+ */
+struct cdx_mcdi_t {
+	u32 id;
+	u32 flags;
+	/* Have more MCDI interface related data */
+};
+
+/**
+ * cdx_mcdi_init - Initialize the MCDI Firmware interface
+ *	for the CDX controller.
+ * @cdx_mcdi: pointer to MCDI interface
+ *
+ * Return 0 on success, <0 on failure
+ */
+int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi);
+
+/**
+ * cdx_mcdi_finish - Close the MCDI Firmware interface.
+ * @cdx_mcdi: pointer to MCDI interface
+ */
+void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi);
+
+/**
+ * cdx_mcdi_get_num_buses - Get the total number of busses on
+ *	the controller.
+ * @cdx_mcdi: pointer to MCDI interface.
+ *
+ * Return total number of busses available on the controller,
+ *	<0 on failure
+ */
+int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi);
+
+/**
+ * cdx_mcdi_get_num_devs - Get the total number of devices on
+ *	a particular bus.
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ *
+ * Return total number of devices available on the bus, <0 on failure
+ */
+int cdx_mcdi_get_num_devs(struct cdx_mcdi_t *cdx_mcdi, int bus_num);
+
+/**
+ * cdx_mcdi_get_dev_config - Get configuration for a particular
+ *	bus_num:dev_num
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @dev_num: Device number.
+ * @dev_params: Pointer to cdx_dev_params_t, this is populated by this
+ *	function with the configuration corresponding to the provided
+ *	bus_num:dev_num.
+ *
+ * Return 0 total number of devices available on the bus, <0 on failure
+ */
+int cdx_mcdi_get_dev_config(struct cdx_mcdi_t *cdx_mcdi,
+			    uint8_t bus_num, uint8_t dev_num,
+			    struct cdx_dev_params_t *dev_params);
+
+/**
+ * cdx_mcdi_reset_dev - Reset cdx device represented by bus_num:dev_num
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @dev_num: Device number.
+ *
+ * Return 0 on success, <0 on failure
+ */
+int cdx_mcdi_reset_dev(struct cdx_mcdi_t *cdx_mcdi,
+		       uint8_t bus_num, uint8_t dev_num);
+
+/**
+ * cdx_mcdi_write_msi_data - Write MSI related info to the Firmware
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @dev_num: Device number.
+ * @msi_index: MSI Index for which the data and address is provided
+ * @data: MSI data (i.e. eventID) for the interrupt
+ * @addr: the address on which the MSI is to be raised
+ */
+int cdx_mcdi_write_msi(struct cdx_mcdi_t *cdx_mcdi,
+		       uint8_t bus_num, uint8_t dev_num,
+		       uint16_t msi_index, uint32_t data,
+		       uint64_t addr);
+
+#endif /* _MCDI_STUBS_H_ */
-- 
2.25.1