[PATCH v2] drivers/virt/vSMP: new driver

[Czerwacki, Eial]

Introducing the vSMP guest driver which allows interaction with the
vSMP control device when running a Linux OS atop of the vSMP hypervisor.
vSMP is a resource aggregation hypervisor from SAP.

The driver comprises of api part which facilitates communication with
the hypervisor and version which displays the hypervisor's version.

This patch s based on previous patches sent to the staging tree mailing
lists

Signed-off-by: Eial Czerwacki <eial.czerwacki@xxxxxxx>
Acked-by: Leonid Arsh <leonid.arsh@xxxxxxx>
Acked-by: Oren Twaig <oren.twaig@xxxxxxx>
CC: SAP vSMP Linux Maintainer <linux.vsmp@xxxxxxx>
CC: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx>
CC: Arnd Bergmann <arnd@xxxxxxxx>
CC: Dan Carpenter <dan.carpenter@xxxxxxxxxx>
CC: Andra Paraschiv <andraprs@xxxxxxxxxx>
CC: Borislav Petkov <bp@xxxxxxx>
CC: Brijesh Singh <brijesh.singh@xxxxxxx>
CC: Eric Biggers <ebiggers@xxxxxxxxxx>
CC: Fei Li <fei1.li@xxxxxxxxx>
CC: Hans de Goede <hdegoede@xxxxxxxxxx>
CC: Jens Axboe <axboe@xxxxxxxxx>
CC: Mauro Carvalho Chehab <mchehab+huawei@xxxxxxxxxx>

v1 -> v2:
	- fix -0 var init in add_sysfs_entries (pointed out by Dan Carpenter)
---
 Documentation/ABI/stable/sysfs-driver-vsmp |   5 +
 MAINTAINERS                                |   6 +
 drivers/virt/Kconfig                       |   2 +
 drivers/virt/Makefile                      |   2 +
 drivers/virt/vsmp/Kconfig                  |  11 +
 drivers/virt/vsmp/Makefile                 |   7 +
 drivers/virt/vsmp/api/api.c                | 249 +++++++++++++++++++++
 drivers/virt/vsmp/api/api.h                |  69 ++++++
 drivers/virt/vsmp/include/registers.h      |  12 +
 drivers/virt/vsmp/version/version.c        | 118 ++++++++++
 drivers/virt/vsmp/version/version.h        |  14 ++
 drivers/virt/vsmp/vsmp_main.c              | 110 +++++++++
 12 files changed, 605 insertions(+)
 create mode 100644 Documentation/ABI/stable/sysfs-driver-vsmp
 create mode 100644 drivers/virt/vsmp/Kconfig
 create mode 100644 drivers/virt/vsmp/Makefile
 create mode 100644 drivers/virt/vsmp/api/api.c
 create mode 100644 drivers/virt/vsmp/api/api.h
 create mode 100644 drivers/virt/vsmp/include/registers.h
 create mode 100644 drivers/virt/vsmp/version/version.c
 create mode 100644 drivers/virt/vsmp/version/version.h
 create mode 100644 drivers/virt/vsmp/vsmp_main.c

diff --git a/Documentation/ABI/stable/sysfs-driver-vsmp b/Documentation/ABI/stable/sysfs-driver-vsmp
new file mode 100644
index 000000000000..18a0a62f40ed
--- /dev/null
+++ b/Documentation/ABI/stable/sysfs-driver-vsmp
@@ -0,0 +1,5 @@
+What:           /sys/hypervisor/vsmp/version
+Date:           Aug 2022
+Contact:        Eial Czerwacki <eial.czerwacki@xxxxxxx>
+		linux-vsmp@xxxxxxx
+Description:    Shows the full version of the vSMP hypervisor
diff --git a/MAINTAINERS b/MAINTAINERS
index f512b430c7cb..cf74089c4d19 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -21783,6 +21783,12 @@ F:	lib/test_printf.c
 F:	lib/test_scanf.c
 F:	lib/vsprintf.c

+VSMP GUEST DRIVER
+M:	Eial Czerwacki <eial.czerwacki@xxxxxxx>
+M:	linux-vsmp@xxxxxxx
+S:	Maintained
+F:	drivers/virt/vsmp
+
 VT1211 HARDWARE MONITOR DRIVER
 M:	Juerg Haefliger <juergh@xxxxxxxxx>
 L:	linux-hwmon@xxxxxxxxxxxxxxx
diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig
index 87ef258cec64..9f283f476674 100644
--- a/drivers/virt/Kconfig
+++ b/drivers/virt/Kconfig
@@ -52,4 +52,6 @@ source "drivers/virt/coco/efi_secret/Kconfig"

 source "drivers/virt/coco/sev-guest/Kconfig"

+source "drivers/virt/vsmp/Kconfig"
+
 endif
diff --git a/drivers/virt/Makefile b/drivers/virt/Makefile
index 093674e05c40..159ba37cb471 100644
--- a/drivers/virt/Makefile
+++ b/drivers/virt/Makefile
@@ -11,3 +11,5 @@ obj-$(CONFIG_NITRO_ENCLAVES)	+= nitro_enclaves/
 obj-$(CONFIG_ACRN_HSM)		+= acrn/
 obj-$(CONFIG_EFI_SECRET)	+= coco/efi_secret/
 obj-$(CONFIG_SEV_GUEST)		+= coco/sev-guest/
+
+obj-$(CONFIG_VSMP)             += vsmp/
diff --git a/drivers/virt/vsmp/Kconfig b/drivers/virt/vsmp/Kconfig
new file mode 100644
index 000000000000..4e1d7e0dc746
--- /dev/null
+++ b/drivers/virt/vsmp/Kconfig
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VSMP
+	tristate "vSMP Guest Support"
+	depends on SYS_HYPERVISOR && X86_64 && PCI
+	help
+	  Support for vSMP Guest Driver.
+
+	  This driver allows information gathering of data from the vSMP hypervisor when
+	  running on top of a vSMP-based hypervisor.
+
+	  If unsure, say no.
diff --git a/drivers/virt/vsmp/Makefile b/drivers/virt/vsmp/Makefile
new file mode 100644
index 000000000000..f637097e19f2
--- /dev/null
+++ b/drivers/virt/vsmp/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for vSMP Guest drivers
+#
+
+obj-$(CONFIG_VSMP) = vsmp.o
+vsmp-y := vsmp_main.o api/api.o version/version.o
diff --git a/drivers/virt/vsmp/api/api.c b/drivers/virt/vsmp/api/api.c
new file mode 100644
index 000000000000..6e40935907bc
--- /dev/null
+++ b/drivers/virt/vsmp/api/api.c
@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * vSMP driver api
+ * (C) Copyright 2022 SAP SE
+ */
+
+#include "api.h"
+
+static void __iomem *cfg_addr;
+static struct kobject *vsmp_sysfs_kobj;
+static struct pci_dev *vsmp_dev_obj;
+
+/* R/W ops handlers */
+
+/*
+ * Init a vsmp firmware operation object
+ */
+int vsmp_init_op(struct fw_ops *op, ssize_t max_size,
+		 enum vsmp_fw_action action)
+{
+	op->hwi_block_size = max_size;
+	op->action = action;
+	op->buff_offset = op->hwi_block_size;
+
+	op->buff = kzalloc(op->hwi_block_size, GFP_KERNEL);
+	if (!op->buff)
+		return -ENOMEM;
+
+	vsmp_reset_op(op);
+
+	return 0;
+}
+
+/*
+ * Release an vsmp firmware operation object
+ */
+void vsmp_release_op(struct fw_ops *op)
+{
+	if (!op) {
+		WARN_ON(!op);
+		return;
+	}
+
+	if (!op->buff) {
+		WARN_ON(!op->buff);
+		return;
+	}
+
+	kfree(op->buff);
+	memset(op, 0, sizeof(*op));
+}
+
+/*
+ * Reset a vsmp firmware operation object
+ */
+void vsmp_reset_op(struct fw_ops *op)
+{
+	memset(op->buff, 0, op->hwi_block_size);
+	op->buff_offset = op->hwi_block_size;
+}
+
+/* Regs/Buffs R/W handlers */
+
+/*
+ * Read a value from a specific register in the vSMP's device config space
+ */
+u64 vsmp_read_reg_from_cfg(u64 reg, enum reg_size_type type)
+{
+	u64 ret_val;
+
+	switch (type) {
+	case VSMP_CTL_REG_SIZE_8BIT:
+		ret_val = readb(cfg_addr + reg);
+		break;
+
+	case VSMP_CTL_REG_SIZE_16BIT:
+		ret_val = readw(cfg_addr + reg);
+		break;
+
+	case VSMP_CTL_REG_SIZE_32BIT:
+		ret_val = readl(cfg_addr + reg);
+		break;
+
+	case VSMP_CTL_REG_SIZE_64BIT:
+		ret_val = readq(cfg_addr + reg);
+		break;
+
+	default:
+		dev_err(get_dev(), "Unsupported reg size type %d.\n", type);
+		ret_val = (u64) -EINVAL;
+	}
+
+	dev_dbg(get_dev(), "%s: read 0x%llx from reg 0x%llx of %d bits\n",
+		__func__, ret_val, reg, (type + 1) * 8);
+	return ret_val;
+}
+
+/*
+ * Read a buffer from the bar byte by byte for halt on
+ * null termination.
+ * Expected buffs are strings.
+ */
+static ssize_t read_buff_from_bar_in_bytes(char *out, u8 __iomem *buff, ssize_t len)
+{
+	u32 i;
+
+	for (i = 0; i < len; i++) {
+		out[i] = ioread8(&buff[i]);
+		if (!out[i])
+			break;
+	}
+
+	return i;
+}
+
+/*
+ * Read a buffer from a specific offset in a specific bar,
+ * maxed to a predefined len size-wise from the vSMP device
+ */
+int vsmp_read_buff_from_bar(u8 bar, u32 offset, char *out, ssize_t len,
+			    bool halt_on_null)
+{
+	u8 __iomem *buff;
+	u64 bar_start = pci_resource_start(vsmp_dev_obj, bar);
+	u32 bar_len = pci_resource_len(vsmp_dev_obj, bar);
+	ssize_t actual_len = len;
+
+	/* incase of overflow, warn and use max len possible */
+	if ((offset + len) > bar_len) {
+		WARN_ON((offset + len) > actual_len);
+		actual_len = bar_len - offset;
+		dev_dbg(get_dev(), "%lu overflows bar len, using %ld len instead\n",
+			len, actual_len);
+	}
+
+	buff = ioremap(bar_start + offset, actual_len);
+	if (!buff)
+		return -ENOMEM;
+
+	if (halt_on_null)
+		read_buff_from_bar_in_bytes(out, buff, len);
+	else
+		memcpy_fromio(out, buff, len);
+
+	iounmap(buff);
+
+	return 0;
+}
+
+/*
+ * Generic function to read from the bar
+ */
+ssize_t vsmp_generic_buff_read(struct fw_ops *op, u8 bar, u64 reg,
+			       char *buf, loff_t off, ssize_t count)
+{
+	ssize_t ret_val = 0;
+
+	if (op->buff_offset >= op->hwi_block_size) {	/* perform H/W op */
+		vsmp_reset_op(op);
+
+		ret_val = vsmp_read_buff_from_bar(bar, reg, op->buff, op->hwi_block_size, false);
+		if (ret_val) {
+			dev_err(get_dev(), "%s operation failed\n",
+				(op->action == FW_READ) ? "read" : "write");
+		}
+		op->buff_offset = 0;
+	}
+
+	if (ret_val)
+		return ret_val;
+
+	return memory_read_from_buffer(buf, count, &op->buff_offset, op->buff, op->hwi_block_size);
+}
+
+/* sysfs handlers */
+
+/*
+ * Register the vSMP sysfs object for user space interaction
+ */
+int vsmp_register_sysfs_group(const struct bin_attribute *bin_attr)
+{
+	int error = -EINVAL;
+
+	if (vsmp_sysfs_kobj && bin_attr) {
+		error = sysfs_create_bin_file(vsmp_sysfs_kobj, bin_attr);
+		if (error)
+			dev_err(get_dev(), "Failed to register sysfs entry (%d)\n", error);
+	}
+
+	return error;
+}
+
+/*
+ * Deregister the vSMP sysfs object for user space interaction
+ */
+void vsmp_deregister_sysfs_group(const struct bin_attribute *bin_attr)
+{
+	if (vsmp_sysfs_kobj && bin_attr)
+		sysfs_remove_bin_file(vsmp_sysfs_kobj, bin_attr);
+}
+
+/* Generic functions */
+
+/*
+ * Open the cfg address space of the vSDP device
+ */
+int open_cfg_addr(struct pci_dev *pdev)
+{
+	u64 cfg_start;
+	u32 cfg_len;
+
+	vsmp_dev_obj = pdev;
+	cfg_start = pci_resource_start(vsmp_dev_obj, 0);
+	cfg_len = pci_resource_len(vsmp_dev_obj, 0);
+
+	dev_dbg(get_dev(), "Mapping bar 0: [0x%llx,0x%llx]\n",
+		cfg_start, cfg_start + cfg_len);
+
+	cfg_addr = ioremap(cfg_start, cfg_len);
+	if (!cfg_addr) {
+		dev_err(get_dev(), "Failed to map vSMP pci controller, exiting.\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+int init_sysfs(void)
+{
+	vsmp_sysfs_kobj = kobject_create_and_add("vsmp", hypervisor_kobj);
+	if (!vsmp_sysfs_kobj) {
+		dev_err(get_dev(), "Failed to create vSMP sysfs entry, exiting.\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void cleanup(void)
+{
+	iounmap(cfg_addr);
+	kobject_put(vsmp_sysfs_kobj);
+}
+
+const struct device *get_dev(void)
+{
+	return &vsmp_dev_obj->dev;
+}
diff --git a/drivers/virt/vsmp/api/api.h b/drivers/virt/vsmp/api/api.h
new file mode 100644
index 000000000000..6142e947979f
--- /dev/null
+++ b/drivers/virt/vsmp/api/api.h
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * vSMP driver api header
+ * (C) Copyright 2022 SAP SE
+ */
+
+#ifndef VSMP_API_H
+#define VSMP_API_H
+
+#include <linux/pci.h>
+
+// R/W ops handlers
+#define vsmp_read_reg32_from_cfg(_reg_) \
+	((u32) vsmp_read_reg_from_cfg((_reg_), VSMP_CTL_REG_SIZE_32BIT))
+
+enum reg_size_type {
+	VSMP_CTL_REG_SIZE_8BIT = 0,
+	VSMP_CTL_REG_SIZE_16BIT,
+	VSMP_CTL_REG_SIZE_32BIT,
+	VSMP_CTL_REG_SIZE_64BIT
+};
+
+enum vsmp_fw_action {
+	FW_READ = 0,
+	FW_WRITE = 1
+};
+
+struct fw_ops {
+	enum vsmp_fw_action action;
+	ssize_t hwi_block_size;
+	unsigned char *buff;
+	loff_t buff_offset;
+	bool in_progress;
+};
+
+int vsmp_init_op(struct fw_ops *op, ssize_t max_size,
+		 enum vsmp_fw_action action);
+void vsmp_release_op(struct fw_ops *op);
+void vsmp_reset_op(struct fw_ops *op);
+
+#define FILE_PREM 0444
+
+/* Regs/Buffs R/W handlers */
+#define vsmp_read_reg32_from_cfg(_reg_) \
+	((u32) vsmp_read_reg_from_cfg((_reg_), VSMP_CTL_REG_SIZE_32BIT))
+
+u64 vsmp_read_reg_from_cfg(u64 reg, enum reg_size_type type);
+ssize_t vsmp_generic_buff_read(struct fw_ops *op, u8 bar, u64 reg,
+			       char *buf, loff_t off, ssize_t count);
+int vsmp_read_buff_from_bar(u8 bar, u32 offset, char *out, ssize_t len,
+			    bool halt_on_null);
+
+typedef int (*sysfs_register_cb)(void);
+typedef void (*sysfs_deregister_cb)(void);
+
+struct sysfs_entry_cbs {
+	sysfs_register_cb reg_cb;
+	sysfs_deregister_cb dereg_cb;
+};
+
+int vsmp_register_sysfs_group(const struct bin_attribute *bin_attr);
+void vsmp_deregister_sysfs_group(const struct bin_attribute *bin_attr);
+
+int open_cfg_addr(struct pci_dev *pdev);
+int init_sysfs(void);
+void cleanup(void);
+const struct device *get_dev(void);
+#endif /* VSMP_API_H */
diff --git a/drivers/virt/vsmp/include/registers.h b/drivers/virt/vsmp/include/registers.h
new file mode 100644
index 000000000000..b6458d25e3b7
--- /dev/null
+++ b/drivers/virt/vsmp/include/registers.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * vSMP driver registers
+ * (C) Copyright 2022 SAP SE
+ */
+
+#ifndef VSMP_REGSITERS_H
+#define VSMP_REGSITERS_H
+
+#define VSMP_VERSION_REG 0x0c
+
+#endif /* VSMP_REGSITERS_H */
diff --git a/drivers/virt/vsmp/version/version.c b/drivers/virt/vsmp/version/version.c
new file mode 100644
index 000000000000..d8ad771daf28
--- /dev/null
+++ b/drivers/virt/vsmp/version/version.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * vSMP driver version module
+ * (C) Copyright 2022 SAP SE
+ */
+
+#include <linux/slab.h>
+#include <linux/kobject.h>
+
+#include "../api/api.h"
+#include "../include/registers.h"
+
+/*
+ * This is the maximal possible length of the version which is a text string
+ * the real len is usually much smaller, thus the driver uses this once to read
+ * the version string and record it's actual len.
+ * From that point and on, the actual len will be used in each call.
+ */
+#define VERSION_MAX_LEN (1 << 19)
+
+static struct fw_ops op;
+
+static ssize_t version_read(struct file *filp, struct kobject *kobj,
+			    struct bin_attribute *bin_attr,
+			    char *buf, loff_t off, size_t count)
+{
+	u64 reg_val = vsmp_read_reg32_from_cfg(VSMP_VERSION_REG);
+	ssize_t ret_val;
+
+	if (reg_val < 0) {
+		dev_err(get_dev(), "Failed to value of reg 0x%x\n", VSMP_VERSION_REG);
+		return 0;
+	}
+
+	ret_val = vsmp_generic_buff_read(&op, 0, reg_val, buf, off, count);
+	if (ret_val < 0) {
+		dev_err(get_dev(), "Failed to read version (%ld)\n", ret_val);
+		return 0;
+	}
+
+	buf[ret_val++] = '\n';
+
+	return ret_val;
+}
+
+struct bin_attribute version_raw_attr = __BIN_ATTR(version, FILE_PREM,
+						   version_read, NULL, VERSION_MAX_LEN);
+
+/*
+ * Retrieve str in order to determine the proper length.
+ * This is the best way to maintain backwards compatibility with all
+ * vSMP versions.
+ */
+static ssize_t get_version_len(void)
+{
+	ssize_t len = 0;
+	u64 reg_val = vsmp_read_reg32_from_cfg(VSMP_VERSION_REG);
+	char *version_str = kzalloc(VERSION_MAX_LEN, GFP_KERNEL);
+
+	if (!version_str)
+		return len;
+
+	if (vsmp_read_reg32_from_cfg(VSMP_VERSION_REG) < 0) {
+		kfree(version_str);
+		dev_err(get_dev(), "Failed to read value of reg 0x%x\n", VSMP_VERSION_REG);
+		return len;
+	}
+
+	memset(version_str, 0, VERSION_MAX_LEN);
+	if (vsmp_read_buff_from_bar(0, reg_val, version_str, VERSION_MAX_LEN, true)) {
+		kfree(version_str);
+		dev_err(get_dev(), "Failed to read buffer from bar\n");
+		return len;
+	}
+
+	len = strlen(version_str);
+	kfree(version_str);
+
+	return len;
+}
+
+/*
+ * Register the version sysfs entry
+ */
+int sysfs_register_version_cb(void)
+{
+	ssize_t len = get_version_len();
+	int ret_val;
+
+	if (!len) {
+		dev_err(get_dev(), "Failed to init vSMP version module\n");
+		return -EINVAL;
+	}
+	version_raw_attr.size = len;
+
+	if (vsmp_init_op(&op, version_raw_attr.size, FW_READ)) {
+		dev_err(get_dev(), "Failed to init vSMP version op\n");
+		return -ENODEV;
+	}
+
+	ret_val = vsmp_register_sysfs_group(&version_raw_attr);
+	if (ret_val) {
+		dev_err(get_dev(), "Failed to init vSMP version support\n");
+		vsmp_release_op(&op);
+	}
+
+	return ret_val;
+}
+
+/*
+ * Deregister the version sysfs entry
+ */
+void sysfs_deregister_version_cb(void)
+{
+	vsmp_deregister_sysfs_group(&version_raw_attr);
+	vsmp_release_op(&op);
+}
diff --git a/drivers/virt/vsmp/version/version.h b/drivers/virt/vsmp/version/version.h
new file mode 100644
index 000000000000..c4430b3065e4
--- /dev/null
+++ b/drivers/virt/vsmp/version/version.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * vSMP driver version module header
+ * (C) Copyright 2022 SAP SE
+ */
+
+#ifndef VSMP_VERSION_COMMON_H
+#define VSMP_VERSION_COMMON_H
+
+int sysfs_register_version_cb(void);
+void sysfs_deregister_version_cb(void);
+
+#endif /* VSMP_VERSION_COMMON_H */
diff --git a/drivers/virt/vsmp/vsmp_main.c b/drivers/virt/vsmp/vsmp_main.c
new file mode 100644
index 000000000000..95704bc7a32f
--- /dev/null
+++ b/drivers/virt/vsmp/vsmp_main.c
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * vSMP driver main
+ * (C) Copyright 2022 SAP SE
+ */
+
+#include <linux/module.h>
+
+#include "api/api.h"
+#include "version/version.h"
+
+/* modules info */
+#define DEVICE_NAME "vSMP"
+#define DRIVER_LICENSE "GPL v2"
+#define DRIVER_AUTHOR "Eial Czerwacki <eial.czerwacki@xxxxxxx>"
+#define DRIVER_DESC "vSMP hypervisor driver"
+#define DRIVER_VERSION "0.1"
+
+#define PCI_DEVICE_ID_SAP_FLX_VSMP_CTL 0x1011
+
+MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_VERSION(DRIVER_VERSION);
+
+/* Sysfs handlers */
+#define create_entry(_label_) \
+	{ \
+		.reg_cb = sysfs_register_ ## _label_ ## _cb, \
+		.dereg_cb = sysfs_deregister_ ## _label_ ## _cb, \
+	}
+
+static struct sysfs_entry_cbs cbs_arr[] = {
+	create_entry(version),
+};
+
+static const struct pci_device_id vsmp_pci_table[] = {
+	{ PCI_VDEVICE(SCALEMP, PCI_DEVICE_ID_SAP_FLX_VSMP_CTL), 0, },
+	{ 0, },			/* terminate list */
+};
+
+/*
+ * Init all submodules's sysfs entries
+ */
+static int add_sysfs_entries(void)
+{
+	int ret_val = 0, i;
+
+	for (i = 0; (i < ARRAY_SIZE(cbs_arr) && !ret_val); i++) {
+		ret_val = cbs_arr[i].reg_cb();
+		if (ret_val) {
+			dev_err(get_dev(), "Failed to init sysfs entry %d (%d).\n",
+				i, ret_val);
+		}
+	}
+
+	return ret_val;
+}
+
+/*
+ * Remove all submodules's sysfs entries
+ */
+static void remove_sysfs_entries(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(cbs_arr); i++)
+		cbs_arr[i].dereg_cb();
+}
+
+static int vsmp_pci_probe(struct pci_dev *pci, const struct pci_device_id *id)
+{
+	int ret_val;
+
+	ret_val = open_cfg_addr(pci);
+	if (ret_val) {
+		dev_err(get_dev(), "Failed to open cfg addr\n");
+		return ret_val;
+	}
+
+	if (init_sysfs()) {
+		dev_err(get_dev(), "Failed to create sysfs folder\n");
+		return -ENODEV;
+	}
+
+	if (add_sysfs_entries()) {
+		dev_err(get_dev(), "Failed to create sysfs entries\n");
+		return -ENODEV;
+	}
+
+	dev_info(get_dev(), "%s up and running\n", DRIVER_DESC);
+
+	return 0;
+}
+
+static void vsmp_pci_remove(struct pci_dev *pci)
+{
+	remove_sysfs_entries();
+	cleanup();
+}
+
+static struct pci_driver vsmp_pci_driver = {
+	.name		= DEVICE_NAME,
+	.id_table	= vsmp_pci_table,
+	.probe		= vsmp_pci_probe,
+	.remove	= vsmp_pci_remove,
+};
+
+module_pci_driver(vsmp_pci_driver);
--
2.25.1