[RFC net-next v1 2/3] net/mlx5: Add mdev sub device life cycle command support
From: Parav Pandit
Date: Fri Mar 08 2019 - 17:08:26 EST
Implement mdev hooks to to create mediated devices using mdev driver.
Actual mlx5_core driver in the host is expected to bind to these devices
using standard device driver model.
mdev devices are created using sysfs file as below example.
$ uuidgen
49d0e9ac-61b8-4c91-957e-6f6dbc42557d
$ echo 49d0e9ac-61b8-4c91-957e-6f6dbc42557d > \
./bus/pci/devices/0000:05:00.0/mdev_supported_types/mlx5_core-mgmt/create
$ echo 49d0e9ac-61b8-4c91-957e-6f6dbc42557d >
/sys/bus/mdev/drivers/vfio_mdev/unbind
Once mlx5 core driver is registered as mdev driver, mdev can be attached
to mlx5_core driver as below.
$ echo 49d0e9ac-61b8-4c91-957e-6f6dbc42557d >
/sys/bus/mdev/drivers/mlx5_core/bind
devlink output:
$ devlink dev show
pci/0000:05:00.0
mdev/69ea1551-d054-46e9-974d-8edae8f0aefe
Signed-off-by: Parav Pandit <parav@xxxxxxxxxxxx>
---
drivers/net/ethernet/mellanox/mlx5/core/Kconfig | 9 ++
drivers/net/ethernet/mellanox/mlx5/core/Makefile | 5 +
drivers/net/ethernet/mellanox/mlx5/core/main.c | 9 ++
drivers/net/ethernet/mellanox/mlx5/core/mdev.c | 120 +++++++++++++++++++++
.../net/ethernet/mellanox/mlx5/core/mlx5_core.h | 15 +++
include/linux/mlx5/driver.h | 5 +
6 files changed, 163 insertions(+)
create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/mdev.c
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
index 37a5514..881ae1a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
@@ -117,3 +117,12 @@ config MLX5_EN_TLS
Build support for TLS cryptography-offload accelaration in the NIC.
Note: Support for hardware with this capability needs to be selected
for this option to become available.
+
+config MLX5_MDEV
+ bool "Mellanox Technologies Mediated device support"
+ depends on MLX5_CORE
+ depends on VFIO_MDEV
+ default y
+ help
+ Build support for mediated devices. Mediated devices allow creating
+ multiple netdev and/or rdma device(s) on single PCI function.
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 82d636b..e5c0822c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -58,4 +58,9 @@ mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \
mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/tls.o en_accel/tls_rxtx.o en_accel/tls_stats.o
+#
+# Mdev basic
+#
+mlx5_core-$(CONFIG_MLX5_MDEV) += mdev.o
+
CFLAGS_tracepoint.o := -I$(src)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 40d591c..72b0072 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -851,10 +851,18 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
goto err_sriov_cleanup;
}
+ err = mlx5_mdev_init(dev);
+ if (err) {
+ dev_err(&pdev->dev, "Failed to init mdev device %d\n", err);
+ goto err_fpga_cleanup;
+ }
+
dev->tracer = mlx5_fw_tracer_create(dev);
return 0;
+err_fpga_cleanup:
+ mlx5_fpga_cleanup(dev);
err_sriov_cleanup:
mlx5_sriov_cleanup(dev);
err_eswitch_cleanup:
@@ -881,6 +889,7 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
{
mlx5_fw_tracer_destroy(dev->tracer);
+ mlx5_mdev_cleanup(dev);
mlx5_fpga_cleanup(dev);
mlx5_sriov_cleanup(dev);
mlx5_eswitch_cleanup(dev->priv.eswitch);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mdev.c b/drivers/net/ethernet/mellanox/mlx5/core/mdev.c
new file mode 100644
index 0000000..e8e4aac
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mdev.c
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018-19 Mellanox Technologies
+
+#include <net/devlink.h>
+#include <linux/mdev.h>
+
+#include "mlx5_core.h"
+
+#define MLX5_MAX_MDEVS 1
+
+struct mlx5_mdev {
+ struct mlx5_core_dev *dev;
+};
+
+static int mlx5_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
+{
+ struct mlx5_core_dev *mlx5_dev;
+ struct device *parent_dev;
+ struct mlx5_mdev *mmdev;
+ struct devlink *devlink;
+ bool added;
+ int err;
+
+ parent_dev = mdev_parent_dev(mdev);
+ mlx5_dev = pci_get_drvdata(to_pci_dev(parent_dev));
+
+ added = atomic_add_unless(&mlx5_dev->mdev_info.cnt, 1, MLX5_MAX_MDEVS);
+ if (!added)
+ return -ENOSPC;
+
+ devlink = devlink_alloc(NULL, sizeof(*mmdev));
+ if (!devlink) {
+ atomic_dec(&mlx5_dev->mdev_info.cnt);
+ return -ENOMEM;
+ }
+ mmdev = devlink_priv(devlink);
+ mmdev->dev = mlx5_dev;
+ mdev_set_drvdata(mdev, mmdev);
+
+ /* TODO: create a mediated device hw object that driver
+ * can work on later on to enable interrupts, create queues etc.
+ */
+ err = devlink_register(devlink, mdev_dev(mdev));
+ if (err) {
+ devlink_free(devlink);
+ atomic_dec(&mlx5_dev->mdev_info.cnt);
+ }
+ return err;
+}
+
+static int mlx5_mdev_remove(struct mdev_device *mdev)
+{
+ struct mlx5_core_dev *mlx5_dev;
+ struct device *parent_dev;
+ struct mlx5_mdev *mmdev;
+ struct devlink *devlink;
+
+ parent_dev = mdev_parent_dev(mdev);
+ mlx5_dev = pci_get_drvdata(to_pci_dev(parent_dev));
+
+ mmdev = mdev_get_drvdata(mdev);
+ devlink = priv_to_devlink(mmdev);
+
+ devlink_unregister(devlink);
+ devlink_free(devlink);
+ atomic_dec(&mlx5_dev->mdev_info.cnt);
+ return 0;
+}
+
+static ssize_t
+num_mdevs_show(struct kobject *kobj, struct device *dev, char *buf)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct mlx5_core_dev *mlx5_dev;
+
+ mlx5_dev = pci_get_drvdata(pdev);
+
+ return sprintf(buf, "%d\n", atomic_read(&mlx5_dev->mdev_info.cnt));
+}
+MDEV_TYPE_ATTR_RO(num_mdevs);
+
+static ssize_t
+available_instances_show(struct kobject *kobj, struct device *dev, char *buf)
+{
+ return sprintf(buf, "%d\n", MLX5_MAX_MDEVS);
+}
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static struct attribute *mdev_dev_attrs[] = {
+ &mdev_type_attr_num_mdevs.attr,
+ &mdev_type_attr_available_instances.attr,
+ NULL,
+};
+
+static struct attribute_group mdev_mgmt_group = {
+ .name = "mgmt",
+ .attrs = mdev_dev_attrs,
+};
+
+struct attribute_group *mlx5_mdev_groups[] = {
+ &mdev_mgmt_group,
+ NULL,
+};
+
+const struct mdev_parent_ops mlx5_mdev_ops = {
+ .create = mlx5_mdev_create,
+ .remove = mlx5_mdev_remove,
+ .supported_type_groups = mlx5_mdev_groups,
+};
+
+int mlx5_mdev_init(struct mlx5_core_dev *dev)
+{
+ return mdev_register_device(&dev->pdev->dev, &mlx5_mdev_ops);
+}
+
+void mlx5_mdev_cleanup(struct mlx5_core_dev *dev)
+{
+ mdev_unregister_device(&dev->pdev->dev);
+}
+
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index 9529cf9..0605a63 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -202,4 +202,19 @@ enum {
u8 mlx5_get_nic_state(struct mlx5_core_dev *dev);
void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state);
+
+#ifdef CONFIG_MLX5_MDEV
+int mlx5_mdev_init(struct mlx5_core_dev *mdev);
+void mlx5_mdev_cleanup(struct mlx5_core_dev *mdev);
+#else
+static inline int mlx5_mdev_init(struct mlx5_core_dev *mdev)
+{
+ return 0;
+}
+
+static inline void mlx5_mdev_cleanup(struct mlx5_core_dev *mdev)
+{
+}
+#endif
+
#endif /* __MLX5_CORE_H__ */
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index c2de50f..6836ab9 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -637,6 +637,10 @@ struct mlx5_clock {
struct mlx5_fw_tracer;
struct mlx5_vxlan;
+struct mlx5_mdevs_info {
+ atomic_t cnt;
+};
+
struct mlx5_core_dev {
struct pci_dev *pdev;
/* sync pci state */
@@ -679,6 +683,7 @@ struct mlx5_core_dev {
struct mlx5_ib_clock_info *clock_info;
struct page *clock_info_page;
struct mlx5_fw_tracer *tracer;
+ struct mlx5_mdevs_info mdev_info;
};
struct mlx5_db {
--
1.8.3.1