[PATCH net-next v3 01/14] sfc: add function personality support for EF100 devices

[Gautam Dawar]

A function personality defines the location and semantics of
registers in the BAR. EF100 NICs allow different personalities
of a PCIe function and changing it at run-time. A total of three
function personalities are defined as of now: EF100, vDPA and
None with EF100 being the default.
For now, vDPA net devices can be created on a EF100 virtual
function and the VF personality will be changed to vDPA in the
process.

Co-developed-by: Martin Habets <habetsm.xilinx@xxxxxxxxx>
Signed-off-by: Martin Habets <habetsm.xilinx@xxxxxxxxx>
Signed-off-by: Gautam Dawar <gautam.dawar@xxxxxxx>
---
 drivers/net/ethernet/sfc/ef100.c     |  7 ++-
 drivers/net/ethernet/sfc/ef100_nic.c | 93 +++++++++++++++++++++++++++-
 drivers/net/ethernet/sfc/ef100_nic.h | 14 ++++-
 3 files changed, 109 insertions(+), 5 deletions(-)

diff --git a/drivers/net/ethernet/sfc/ef100.c b/drivers/net/ethernet/sfc/ef100.c
index 6334992b0af4..35f1b74ba890 100644
--- a/drivers/net/ethernet/sfc/ef100.c
+++ b/drivers/net/ethernet/sfc/ef100.c
@@ -3,6 +3,7 @@
  * Driver for Solarflare network controllers and boards
  * Copyright 2005-2018 Solarflare Communications Inc.
  * Copyright 2019-2022 Xilinx Inc.
+ * Copyright (C) 2023, Advanced Micro Devices, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
@@ -428,8 +429,7 @@ static void ef100_pci_remove(struct pci_dev *pci_dev)
 	if (!efx)
 		return;
 
-	probe_data = container_of(efx, struct efx_probe_data, efx);
-	ef100_remove_netdev(probe_data);
+	efx_ef100_set_bar_config(efx, EF100_BAR_CONFIG_NONE);
 #ifdef CONFIG_SFC_SRIOV
 	efx_fini_struct_tc(efx);
 #endif
@@ -440,6 +440,7 @@ static void ef100_pci_remove(struct pci_dev *pci_dev)
 	pci_dbg(pci_dev, "shutdown successful\n");
 
 	pci_set_drvdata(pci_dev, NULL);
+	probe_data = container_of(efx, struct efx_probe_data, efx);
 	efx_fini_struct(efx);
 	kfree(probe_data);
 };
@@ -505,7 +506,7 @@ static int ef100_pci_probe(struct pci_dev *pci_dev,
 		goto fail;
 
 	efx->state = STATE_PROBED;
-	rc = ef100_probe_netdev(probe_data);
+	rc = efx_ef100_set_bar_config(efx, EF100_BAR_CONFIG_EF100);
 	if (rc)
 		goto fail;
 
diff --git a/drivers/net/ethernet/sfc/ef100_nic.c b/drivers/net/ethernet/sfc/ef100_nic.c
index 4dc643b0d2db..54b2ee7a5be6 100644
--- a/drivers/net/ethernet/sfc/ef100_nic.c
+++ b/drivers/net/ethernet/sfc/ef100_nic.c
@@ -3,6 +3,7 @@
  * Driver for Solarflare network controllers and boards
  * Copyright 2018 Solarflare Communications Inc.
  * Copyright 2019-2022 Xilinx Inc.
+ * Copyright (C) 2023, Advanced Micro Devices, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
@@ -772,6 +773,93 @@ static int efx_ef100_get_base_mport(struct efx_nic *efx)
 	return 0;
 }
 
+/* BAR configuration.
+ * To change BAR configuration, tear down the current configuration (which
+ * leaves the hardware in the PROBED state), and then initialise the new
+ * BAR state.
+ */
+struct ef100_bar_config_ops {
+	int (*init)(struct efx_probe_data *probe_data);
+	void (*fini)(struct efx_probe_data *probe_data);
+};
+
+static const struct ef100_bar_config_ops bar_config_ops[] = {
+	[EF100_BAR_CONFIG_EF100] = {
+		.init = ef100_probe_netdev,
+		.fini = ef100_remove_netdev
+	},
+#ifdef CONFIG_SFC_VDPA
+	[EF100_BAR_CONFIG_VDPA] = {
+		.init = NULL,
+		.fini = NULL
+	},
+#endif
+	[EF100_BAR_CONFIG_NONE] = {
+		.init = NULL,
+		.fini = NULL
+	},
+};
+
+/* Keep this in sync with the definition of enum ef100_bar_config. */
+static char *bar_config_name[] = {
+	[EF100_BAR_CONFIG_NONE] = "None",
+	[EF100_BAR_CONFIG_EF100] = "EF100",
+	[EF100_BAR_CONFIG_VDPA] = "vDPA",
+};
+
+int efx_ef100_set_bar_config(struct efx_nic *efx,
+			     enum ef100_bar_config new_config)
+{
+	const struct ef100_bar_config_ops *old_config_ops;
+	const struct ef100_bar_config_ops *new_config_ops;
+	struct ef100_nic_data *nic_data = efx->nic_data;
+	struct efx_probe_data *probe_data;
+	enum ef100_bar_config old_config;
+	int rc;
+
+	if (WARN_ON_ONCE(nic_data->bar_config > EF100_BAR_CONFIG_VDPA))
+		return -EINVAL;
+
+	/* Current EF100 hardware supports vDPA on VFs only */
+	if (IS_ENABLED(CONFIG_SFC_VDPA) &&
+	    new_config == EF100_BAR_CONFIG_VDPA &&
+	    !efx->type->is_vf) {
+		pci_err(efx->pci_dev, "vdpa over PF not supported : %s",
+			efx->name);
+		return -EOPNOTSUPP;
+	}
+
+	mutex_lock(&nic_data->bar_config_lock);
+	old_config = nic_data->bar_config;
+	if (new_config == old_config) {
+		mutex_unlock(&nic_data->bar_config_lock);
+		return 0;
+	}
+
+	old_config_ops = &bar_config_ops[old_config];
+	new_config_ops = &bar_config_ops[new_config];
+
+	probe_data = container_of(efx, struct efx_probe_data, efx);
+	if (old_config_ops->fini)
+		old_config_ops->fini(probe_data);
+	nic_data->bar_config = EF100_BAR_CONFIG_NONE;
+
+	if (new_config_ops->init) {
+		rc = new_config_ops->init(probe_data);
+		if (rc) {
+			mutex_unlock(&nic_data->bar_config_lock);
+			return rc;
+		}
+	}
+
+	nic_data->bar_config = new_config;
+	pci_dbg(efx->pci_dev, "BAR configuration changed to %s\n",
+		bar_config_name[new_config]);
+	mutex_unlock(&nic_data->bar_config_lock);
+
+	return 0;
+}
+
 static int compare_versions(const char *a, const char *b)
 {
 	int a_major, a_minor, a_point, a_patch;
@@ -1025,6 +1113,7 @@ static int ef100_probe_main(struct efx_nic *efx)
 		return -ENOMEM;
 	efx->nic_data = nic_data;
 	nic_data->efx = efx;
+	mutex_init(&nic_data->bar_config_lock);
 	efx->max_vis = EF100_MAX_VIS;
 
 	/* Populate design-parameter defaults */
@@ -1208,8 +1297,10 @@ void ef100_remove(struct efx_nic *efx)
 
 	efx_mcdi_detach(efx);
 	efx_mcdi_fini(efx);
-	if (nic_data)
+	if (nic_data) {
 		efx_nic_free_buffer(efx, &nic_data->mcdi_buf);
+		mutex_destroy(&nic_data->bar_config_lock);
+	}
 	kfree(nic_data);
 	efx->nic_data = NULL;
 }
diff --git a/drivers/net/ethernet/sfc/ef100_nic.h b/drivers/net/ethernet/sfc/ef100_nic.h
index f1ed481c1260..02e5ab4e9f1f 100644
--- a/drivers/net/ethernet/sfc/ef100_nic.h
+++ b/drivers/net/ethernet/sfc/ef100_nic.h
@@ -2,7 +2,8 @@
 /****************************************************************************
  * Driver for Solarflare network controllers and boards
  * Copyright 2018 Solarflare Communications Inc.
- * Copyright 2019-2020 Xilinx Inc.
+ * Copyright 2019-2022 Xilinx Inc.
+ * Copyright (C) 2023, Advanced Micro Devices, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
@@ -61,6 +62,13 @@ enum {
 	EF100_STAT_COUNT
 };
 
+/* Keep this in sync with the contents of bar_config_name. */
+enum ef100_bar_config {
+	EF100_BAR_CONFIG_NONE,
+	EF100_BAR_CONFIG_EF100,
+	EF100_BAR_CONFIG_VDPA,
+};
+
 struct ef100_nic_data {
 	struct efx_nic *efx;
 	struct efx_buffer mcdi_buf;
@@ -71,6 +79,8 @@ struct ef100_nic_data {
 	u16 warm_boot_count;
 	u8 port_id[ETH_ALEN];
 	DECLARE_BITMAP(evq_phases, EFX_MAX_CHANNELS);
+	enum ef100_bar_config bar_config;
+	struct mutex bar_config_lock; /* lock to control access to bar config */
 	u64 stats[EF100_STAT_COUNT];
 	u32 base_mport;
 	bool have_mport; /* base_mport was populated successfully */
@@ -95,4 +105,6 @@ int ef100_filter_table_probe(struct efx_nic *efx);
 int ef100_get_mac_address(struct efx_nic *efx, u8 *mac_address,
 			  int client_handle, bool empty_ok);
 int efx_ef100_lookup_client_id(struct efx_nic *efx, efx_qword_t pciefn, u32 *id);
+int efx_ef100_set_bar_config(struct efx_nic *efx,
+			     enum ef100_bar_config new_config);
 #endif	/* EFX_EF100_NIC_H */
-- 
2.30.1