Re: [PATCH v6 08/21] s390: vfio-ap: register matrix device with VFIO mdev framework

[Harald Freudenberger]

On 09.07.2018 16:17, Pierre Morel wrote:
> On 29/06/2018 23:11, Tony Krowiak wrote:
>> Registers the matrix device created by the VFIO AP device
>> driver with the VFIO mediated device framework.
>> Registering the matrix device will create the sysfs
>> structures needed to create mediated matrix devices
>> each of which will be used to configure the AP matrix
>> for a guest and connect it to the VFIO AP device driver.
>>
>> Registering the matrix device with the VFIO mediated device
>> framework will create the following sysfs structures:
>>
>> /sys/devices/vfio_ap
>> ... [matrix]
>> ...... [mdev_supported_types]
>> ......... [vfio_ap-passthrough]
>> ............ create
>>
>> To create a mediated device for the AP matrix device, write a UUID
>> to the create file:
>>
>> ÂÂÂÂuuidgen > create
>>
>> A symbolic link to the mediated device's directory will be created in the
>> devices subdirectory named after the generated $uuid:
>>
>> /sys/devices/vfio_ap
>> ... [matrix]
>> ...... [mdev_supported_types]
>> ......... [vfio_ap-passthrough]
>> ............ [devices]
>> ............... [$uuid]
>>
>> Signed-off-by: Tony Krowiak <akrowiak@xxxxxxxxxxxxx>
>> ---
>> Â MAINTAINERSÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ |ÂÂÂ 1 +
>> Â drivers/s390/crypto/MakefileÂÂÂÂÂÂÂÂÂ |ÂÂÂ 2 +-
>> Â drivers/s390/crypto/vfio_ap_drv.cÂÂÂÂ |ÂÂÂ 9 ++
>> Â drivers/s390/crypto/vfio_ap_ops.cÂÂÂÂ |Â 131 +++++++++++++++++++++++++++++++++
>> Â drivers/s390/crypto/vfio_ap_private.h |ÂÂ 22 +++++-
>> Â 5 files changed, 161 insertions(+), 4 deletions(-)
>> Â create mode 100644 drivers/s390/crypto/vfio_ap_ops.c
>>
>> diff --git a/MAINTAINERS b/MAINTAINERS
>> index 0515dae..3217803 100644
>> --- a/MAINTAINERS
>> +++ b/MAINTAINERS
>> @@ -12410,6 +12410,7 @@ W:ÂÂÂ http://www.ibm.com/developerworks/linux/linux390/
>> Â S:ÂÂÂ Supported
>> Â F:ÂÂÂ drivers/s390/crypto/vfio_ap_drv.c
>> Â F:ÂÂÂ drivers/s390/crypto/vfio_ap_private.h
>> +F:ÂÂÂ drivers/s390/crypto/vfio_ap_ops.c
>>
>> Â S390 ZFCP DRIVER
>> Â M:ÂÂÂ Steffen Maier <maier@xxxxxxxxxxxxx>
>> diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile
>> index 48e466e..8d36b05 100644
>> --- a/drivers/s390/crypto/Makefile
>> +++ b/drivers/s390/crypto/Makefile
>> @@ -17,5 +17,5 @@ pkey-objs := pkey_api.o
>> Â obj-$(CONFIG_PKEY) += pkey.o
>>
>> Â # adjunct processor matrix
>> -vfio_ap-objs := vfio_ap_drv.o
>> +vfio_ap-objs := vfio_ap_drv.o vfio_ap_ops.o
>> Â obj-$(CONFIG_VFIO_AP) += vfio_ap.o
>> diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c
>> index 93db312..b6ff7a4 100644
>> --- a/drivers/s390/crypto/vfio_ap_drv.c
>> +++ b/drivers/s390/crypto/vfio_ap_drv.c
>> @@ -127,11 +127,20 @@ int __init vfio_ap_init(void)
>> ÂÂÂÂÂÂÂÂÂ return ret;
>> ÂÂÂÂÂ }
>>
>> +ÂÂÂ ret = vfio_ap_mdev_register(matrix_dev);
>> +ÂÂÂ if (ret) {
>> +ÂÂÂÂÂÂÂ ap_driver_unregister(&vfio_ap_drv);
>> +ÂÂÂÂÂÂÂ vfio_ap_matrix_dev_destroy(matrix_dev);
>> +
>> +ÂÂÂÂÂÂÂ return ret;
>> +ÂÂÂ }
>> +
>> ÂÂÂÂÂ return 0;
>> Â }
>>
>> Â void __exit vfio_ap_exit(void)
>> Â {
>> +ÂÂÂ vfio_ap_mdev_unregister(matrix_dev);
>> ÂÂÂÂÂ ap_driver_unregister(&vfio_ap_drv);
>> ÂÂÂÂÂ vfio_ap_matrix_dev_destroy(matrix_dev);
>> Â }
>> diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
>> new file mode 100644
>> index 0000000..4e61e33
>> --- /dev/null
>> +++ b/drivers/s390/crypto/vfio_ap_ops.c
>> @@ -0,0 +1,131 @@
>> +// SPDX-License-Identifier: GPL-2.0+
>> +/*
>> + * Adjunct processor matrix VFIO device driver callbacks.
>> + *
>> + * Copyright IBM Corp. 2018
>> + * Author(s): Tony Krowiak <akrowiak@xxxxxxxxxxxxx>
>> + *
>> + */
>> +#include <linux/string.h>
>> +#include <linux/vfio.h>
>> +#include <linux/device.h>
>> +#include <linux/list.h>
>> +#include <linux/ctype.h>
>> +
>> +#include "vfio_ap_private.h"
>> +
>> +#define VFOP_AP_MDEV_TYPE_HWVIRT "passthrough"
>> +#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
>> +
>> +DEFINE_SPINLOCK(mdev_list_lock);
>> +LIST_HEAD(mdev_list);
>> +
>> +static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
>> +{
>> +ÂÂÂ struct ap_matrix_dev *matrix_dev =
>> +ÂÂÂÂÂÂÂ to_ap_matrix_dev(mdev_parent_dev(mdev));
>> +ÂÂÂ struct ap_matrix_mdev *matrix_mdev;
>> +
>> +ÂÂÂ matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
>> +ÂÂÂ if (!matrix_mdev)
>> +ÂÂÂÂÂÂÂ return -ENOMEM;
>> +
>> +ÂÂÂ matrix_mdev->name = dev_name(mdev_dev(mdev));
>> +ÂÂÂ mdev_set_drvdata(mdev, matrix_mdev);
>> +
>> +ÂÂÂ if (atomic_dec_if_positive(&matrix_dev->available_instances) < 0) {
>> +ÂÂÂÂÂÂÂ kfree(matrix_mdev);
>> +ÂÂÂÂÂÂÂ return -EPERM;
>> +ÂÂÂ }
>> +
>> +ÂÂÂ spin_lock_bh(&mdev_list_lock);
>> +ÂÂÂ list_add(&matrix_mdev->list, &mdev_list);
>> +ÂÂÂ spin_unlock_bh(&mdev_list_lock);
>> +
>> +ÂÂÂ return 0;
>> +}
>> +
>> +static int vfio_ap_mdev_remove(struct mdev_device *mdev)
>> +{
>> +ÂÂÂ struct ap_matrix_dev *matrix_dev =
>> +ÂÂÂÂÂÂÂ to_ap_matrix_dev(mdev_parent_dev(mdev));
>> +ÂÂÂ struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
>> +
>> +ÂÂÂ spin_lock_bh(&mdev_list_lock);
>> +ÂÂÂ list_del(&matrix_mdev->list);
>> +ÂÂÂ spin_unlock_bh(&mdev_list_lock);
>> +ÂÂÂ kfree(matrix_mdev);
>> +ÂÂÂ mdev_set_drvdata(mdev, NULL);
>> +ÂÂÂ atomic_inc(&matrix_dev->available_instances);
>> +
>> +ÂÂÂ return 0;
>> +}
>> +
>> +static ssize_t name_show(struct kobject *kobj, struct device *dev, char *buf)
>> +{
>> +ÂÂÂ return sprintf(buf, "%s\n", VFIO_AP_MDEV_NAME_HWVIRT);
>> +}
>> +
>> +MDEV_TYPE_ATTR_RO(name);
>> +
>> +static ssize_t available_instances_show(struct kobject *kobj,
>> +ÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ struct device *dev, char *buf)
>> +{
>> +ÂÂÂ struct ap_matrix_dev *matrix_dev = to_ap_matrix_dev(dev);
>> +
>> +ÂÂÂ return sprintf(buf, "%d\n",
>> +ÂÂÂÂÂÂÂÂÂÂÂÂÂÂ atomic_read(&matrix_dev->available_instances));
>> +}
>> +
>> +MDEV_TYPE_ATTR_RO(available_instances);
>> +
>> +static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
>> +ÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ char *buf)
>> +{
>> +ÂÂÂ return sprintf(buf, "%s\n", VFIO_DEVICE_API_AP_STRING);
>> +}
>> +
>> +MDEV_TYPE_ATTR_RO(device_api);
>> +
>> +static struct attribute *vfio_ap_mdev_type_attrs[] = {
>> +ÂÂÂ &mdev_type_attr_name.attr,
>> +ÂÂÂ &mdev_type_attr_device_api.attr,
>> +ÂÂÂ &mdev_type_attr_available_instances.attr,
>> +ÂÂÂ NULL,
>> +};
>> +
>> +static struct attribute_group vfio_ap_mdev_hwvirt_type_group = {
>> +ÂÂÂ .name = VFOP_AP_MDEV_TYPE_HWVIRT,
>> +ÂÂÂ .attrs = vfio_ap_mdev_type_attrs,
>> +};
>> +
>> +static struct attribute_group *vfio_ap_mdev_type_groups[] = {
>> +ÂÂÂ &vfio_ap_mdev_hwvirt_type_group,
>> +ÂÂÂ NULL,
>> +};
>> +
>> +static const struct mdev_parent_ops vfio_ap_matrix_ops = {
>> +ÂÂÂ .ownerÂÂÂÂÂÂÂÂÂÂÂ = THIS_MODULE,
>> +ÂÂÂ .supported_type_groupsÂÂÂ = vfio_ap_mdev_type_groups,
>> +ÂÂÂ .createÂÂÂÂÂÂÂÂÂÂÂ = vfio_ap_mdev_create,
>> +ÂÂÂ .removeÂÂÂÂÂÂÂÂÂÂÂ = vfio_ap_mdev_remove,
>> +};
>> +
>> +int vfio_ap_mdev_register(struct ap_matrix_dev *matrix_dev)
>> +{
>> +ÂÂÂ int ret;
>> +
>> +ÂÂÂ ret = mdev_register_device(&matrix_dev->device, &vfio_ap_matrix_ops);
>> +ÂÂÂ if (ret)
>> +ÂÂÂÂÂÂÂ return ret;
>> +
>> +ÂÂÂ atomic_set(&matrix_dev->available_instances,
>> +ÂÂÂÂÂÂÂÂÂÂ AP_MATRIX_MAX_AVAILABLE_INSTANCES);
>> +
>> +ÂÂÂ return 0;
>> +}
>> +
>> +void vfio_ap_mdev_unregister(struct ap_matrix_dev *matrix_dev)
>> +{
>> +ÂÂÂ mdev_unregister_device(&matrix_dev->device);
>> +}
>> diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
>> index 19c0b60..3de1275 100644
>> --- a/drivers/s390/crypto/vfio_ap_private.h
>> +++ b/drivers/s390/crypto/vfio_ap_private.h
>> @@ -10,20 +10,36 @@
>> Â #define _VFIO_AP_PRIVATE_H_
>>
>> Â #include <linux/types.h>
>> +#include <linux/device.h>
>> +#include <linux/mdev.h>
>>
>> Â #include "ap_bus.h"
>>
>> Â #define VFIO_AP_MODULE_NAME "vfio_ap"
>> Â #define VFIO_AP_DRV_NAME "vfio_ap"
>> +/**
>> + * There must be one mediated matrix device for every guest using AP devices.
>> + * If every APQN is assigned to a guest, then the maximum number of guests with
>> + * a unique APQN assigned would be 255 adapters x 255 domains = 72351 guests.
>> + */
>> +#define AP_MATRIX_MAX_AVAILABLE_INSTANCES 72351
>
> Why isn't it 256 x 256 ?
In zcrypt.h there are defines for these:

#define MAX_ZDEV_CARDIDS_EXT 256
#define MAX_ZDEV_DOMAINS_EXT 256

/* Maximum number of zcrypt devices */
#define MAX_ZDEV_ENTRIES_EXT (MAX_ZDEV_CARDIDS_EXT * MAX_ZDEV_DOMAINS_EXT)
>> Â struct ap_matrix_dev {
>> ÂÂÂÂÂ struct device device;
>> +ÂÂÂ atomic_t available_instances;
>> +};
>> +
>> +struct ap_matrix_mdev {
>> +ÂÂÂ const char *name;
>> +ÂÂÂ struct list_head list;
>> Â };
>>
>> -static inline struct ap_matrix_dev
>> -*to_ap_matrix_parent_dev(struct device *dev)
>> +static struct ap_matrix_dev *to_ap_matrix_dev(struct device *dev)
>> Â {
>> -ÂÂÂ return container_of(dev, struct ap_matrix_dev, device.parent);
>> +ÂÂÂ return container_of(dev, struct ap_matrix_dev, device);
>> Â }
>>
>> +extern int vfio_ap_mdev_register(struct ap_matrix_dev *matrix_dev);
>> +extern void vfio_ap_mdev_unregister(struct ap_matrix_dev *matrix_dev);
>> +
>> Â #endif /* _VFIO_AP_PRIVATE_H_ */
>
>