Re: [PATCH v2 06/18] fpga: dfl: fme: add DFL_FPGA_FME_PORT_RELEASE/ASSIGN ioctl support.

[Moritz Fischer]

On Mon, Apr 29, 2019 at 04:55:39PM +0800, Wu Hao wrote:
> In order to support virtualization usage via PCIe SRIOV, this patch
> adds two ioctls under FPGA Management Engine (FME) to release and
> assign back the port device. In order to safely turn Port from PF
> into VF and enable PCIe SRIOV, it requires user to invoke this
> PORT_RELEASE ioctl to release port firstly to remove userspace
> interfaces, and then configure the PF/VF access register in FME.
> After disable SRIOV, it requires user to invoke this PORT_ASSIGN
> ioctl to attach the port back to PF.
> 
>  Ioctl interfaces:
>  * DFL_FPGA_FME_PORT_RELEASE
>    Release platform device of given port, it deletes port platform
>    device to remove related userspace interfaces on PF, then
>    configures PF/VF access mode to VF.
> 
>  * DFL_FPGA_FME_PORT_ASSIGN
>    Assign platform device of given port back to PF, it configures
>    PF/VF access mode to PF, then adds port platform device back to
>    re-enable related userspace interfaces on PF.
> 
> Signed-off-by: Zhang Yi Z <yi.z.zhang@xxxxxxxxx>
> Signed-off-by: Xu Yilun <yilun.xu@xxxxxxxxx>
> Signed-off-by: Wu Hao <hao.wu@xxxxxxxxx>
> Acked-by: Alan Tull <atull@xxxxxxxxxx>
Acked-by: Moritz Fischer <mdf@xxxxxxxxxx>
> ---
>  drivers/fpga/dfl-fme-main.c   |  54 +++++++++++++++++++++
>  drivers/fpga/dfl.c            | 107 +++++++++++++++++++++++++++++++++++++-----
>  drivers/fpga/dfl.h            |  10 ++++
>  include/uapi/linux/fpga-dfl.h |  32 +++++++++++++
>  4 files changed, 191 insertions(+), 12 deletions(-)
> 
> diff --git a/drivers/fpga/dfl-fme-main.c b/drivers/fpga/dfl-fme-main.c
> index 076d74f..8b2a337 100644
> --- a/drivers/fpga/dfl-fme-main.c
> +++ b/drivers/fpga/dfl-fme-main.c
> @@ -16,6 +16,7 @@
>  
>  #include <linux/kernel.h>
>  #include <linux/module.h>
> +#include <linux/uaccess.h>
>  #include <linux/fpga-dfl.h>
>  
>  #include "dfl.h"
> @@ -105,9 +106,62 @@ static void fme_hdr_uinit(struct platform_device *pdev,
>  	sysfs_remove_files(&pdev->dev.kobj, fme_hdr_attrs);
>  }
>  
> +static long fme_hdr_ioctl_release_port(struct dfl_feature_platform_data *pdata,
> +				       void __user *arg)
> +{
> +	struct dfl_fpga_cdev *cdev = pdata->dfl_cdev;
> +	struct dfl_fpga_fme_port_release release;
> +	unsigned long minsz;
> +
> +	minsz = offsetofend(struct dfl_fpga_fme_port_release, port_id);
> +
> +	if (copy_from_user(&release, arg, minsz))
> +		return -EFAULT;
> +
> +	if (release.argsz < minsz || release.flags)
> +		return -EINVAL;
> +
> +	return dfl_fpga_cdev_config_port(cdev, release.port_id, true);
> +}
> +
> +static long fme_hdr_ioctl_assign_port(struct dfl_feature_platform_data *pdata,
> +				      void __user *arg)
> +{
> +	struct dfl_fpga_cdev *cdev = pdata->dfl_cdev;
> +	struct dfl_fpga_fme_port_assign assign;
> +	unsigned long minsz;
> +
> +	minsz = offsetofend(struct dfl_fpga_fme_port_assign, port_id);
> +
> +	if (copy_from_user(&assign, arg, minsz))
> +		return -EFAULT;
> +
> +	if (assign.argsz < minsz || assign.flags)
> +		return -EINVAL;
> +
> +	return dfl_fpga_cdev_config_port(cdev, assign.port_id, false);
> +}
> +
> +static long fme_hdr_ioctl(struct platform_device *pdev,
> +			  struct dfl_feature *feature,
> +			  unsigned int cmd, unsigned long arg)
> +{
> +	struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
> +
> +	switch (cmd) {
> +	case DFL_FPGA_FME_PORT_RELEASE:
> +		return fme_hdr_ioctl_release_port(pdata, (void __user *)arg);
> +	case DFL_FPGA_FME_PORT_ASSIGN:
> +		return fme_hdr_ioctl_assign_port(pdata, (void __user *)arg);
> +	}
> +
> +	return -ENODEV;
> +}
> +
>  static const struct dfl_feature_ops fme_hdr_ops = {
>  	.init = fme_hdr_init,
>  	.uinit = fme_hdr_uinit,
> +	.ioctl = fme_hdr_ioctl,
>  };
>  
>  static struct dfl_feature_driver fme_feature_drvs[] = {
> diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c
> index 2c09e50..a6b6d38 100644
> --- a/drivers/fpga/dfl.c
> +++ b/drivers/fpga/dfl.c
> @@ -224,16 +224,20 @@ void dfl_fpga_port_ops_del(struct dfl_fpga_port_ops *ops)
>   */
>  int dfl_fpga_check_port_id(struct platform_device *pdev, void *pport_id)
>  {
> -	struct dfl_fpga_port_ops *port_ops = dfl_fpga_port_ops_get(pdev);
> -	int port_id;
> +	struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
> +	struct dfl_fpga_port_ops *port_ops;
> +
> +	if (pdata->id != FEATURE_DEV_ID_UNUSED)
> +		return pdata->id == *(int *)pport_id;
>  
> +	port_ops = dfl_fpga_port_ops_get(pdev);
>  	if (!port_ops || !port_ops->get_id)
>  		return 0;
>  
> -	port_id = port_ops->get_id(pdev);
> +	pdata->id = port_ops->get_id(pdev);
>  	dfl_fpga_port_ops_put(port_ops);
>  
> -	return port_id == *(int *)pport_id;
> +	return pdata->id == *(int *)pport_id;
>  }
>  EXPORT_SYMBOL_GPL(dfl_fpga_check_port_id);
>  
> @@ -462,6 +466,7 @@ static int build_info_commit_dev(struct build_feature_devs_info *binfo)
>  	pdata->dev = fdev;
>  	pdata->num = binfo->feature_num;
>  	pdata->dfl_cdev = binfo->cdev;
> +	pdata->id = FEATURE_DEV_ID_UNUSED;
>  	mutex_init(&pdata->lock);
>  
>  	/*
> @@ -959,25 +964,27 @@ void dfl_fpga_feature_devs_remove(struct dfl_fpga_cdev *cdev)
>  {
>  	struct dfl_feature_platform_data *pdata, *ptmp;
>  
> -	remove_feature_devs(cdev);
> -
>  	mutex_lock(&cdev->lock);
> -	if (cdev->fme_dev) {
> -		/* the fme should be unregistered. */
> -		WARN_ON(device_is_registered(cdev->fme_dev));
> +	if (cdev->fme_dev)
>  		put_device(cdev->fme_dev);
> -	}
>  
>  	list_for_each_entry_safe(pdata, ptmp, &cdev->port_dev_list, node) {
>  		struct platform_device *port_dev = pdata->dev;
>  
> -		/* the port should be unregistered. */
> -		WARN_ON(device_is_registered(&port_dev->dev));
> +		/* remove released ports */
> +		if (!device_is_registered(&port_dev->dev)) {
> +			dfl_id_free(feature_dev_id_type(port_dev),
> +				    port_dev->id);
> +			platform_device_put(port_dev);
> +		}
> +
>  		list_del(&pdata->node);
>  		put_device(&port_dev->dev);
>  	}
>  	mutex_unlock(&cdev->lock);
>  
> +	remove_feature_devs(cdev);
> +
>  	fpga_region_unregister(cdev->region);
>  	devm_kfree(cdev->parent, cdev);
>  }
> @@ -1015,6 +1022,82 @@ struct platform_device *
>  }
>  EXPORT_SYMBOL_GPL(__dfl_fpga_cdev_find_port);
>  
> +static int attach_port_dev(struct dfl_fpga_cdev *cdev, u32 port_id)
> +{
> +	struct platform_device *port_pdev;
> +	int ret = -ENODEV;
> +
> +	mutex_lock(&cdev->lock);
> +	port_pdev = __dfl_fpga_cdev_find_port(cdev, &port_id,
> +					      dfl_fpga_check_port_id);
> +	if (!port_pdev)
> +		goto unlock_exit;
> +
> +	if (device_is_registered(&port_pdev->dev)) {
> +		ret = -EBUSY;
> +		goto put_dev_exit;
> +	}
> +
> +	ret = platform_device_add(port_pdev);
> +	if (ret)
> +		goto put_dev_exit;
> +
> +	dfl_feature_dev_use_end(dev_get_platdata(&port_pdev->dev));
> +	cdev->released_port_num--;
> +put_dev_exit:
> +	put_device(&port_pdev->dev);
> +unlock_exit:
> +	mutex_unlock(&cdev->lock);
> +	return ret;
> +}
> +
> +static int detach_port_dev(struct dfl_fpga_cdev *cdev, u32 port_id)
> +{
> +	struct platform_device *port_pdev;
> +	int ret = -ENODEV;
> +
> +	mutex_lock(&cdev->lock);
> +	port_pdev = __dfl_fpga_cdev_find_port(cdev, &port_id,
> +					      dfl_fpga_check_port_id);
> +	if (!port_pdev)
> +		goto unlock_exit;
> +
> +	if (!device_is_registered(&port_pdev->dev)) {
> +		ret = -EBUSY;
> +		goto put_dev_exit;
> +	}
> +
> +	ret = dfl_feature_dev_use_begin(dev_get_platdata(&port_pdev->dev));
> +	if (ret)
> +		goto put_dev_exit;
> +
> +	platform_device_del(port_pdev);
> +	cdev->released_port_num++;
> +put_dev_exit:
> +	put_device(&port_pdev->dev);
> +unlock_exit:
> +	mutex_unlock(&cdev->lock);
> +	return ret;
> +}
> +
> +/**
> + * dfl_fpga_cdev_config_port - configure a port feature dev
> + * @cdev: parent container device.
> + * @port_id: id of the port feature device.
> + * @release: release port or assign port back.
> + *
> + * This function allows user to release port platform device or assign it back.
> + * e.g. to safely turn one port from PF into VF for PCI device SRIOV support,
> + * release port platform device is one necessary step.
> + */
> +int dfl_fpga_cdev_config_port(struct dfl_fpga_cdev *cdev,
> +			      u32 port_id, bool release)
> +{
> +	return release ? detach_port_dev(cdev, port_id) :
> +			 attach_port_dev(cdev, port_id);
> +}
> +EXPORT_SYMBOL_GPL(dfl_fpga_cdev_config_port);
> +
>  static int __init dfl_fpga_init(void)
>  {
>  	int ret;
> diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h
> index 8851c6c..63f39ab 100644
> --- a/drivers/fpga/dfl.h
> +++ b/drivers/fpga/dfl.h
> @@ -183,6 +183,8 @@ struct dfl_feature {
>  
>  #define DEV_STATUS_IN_USE	0
>  
> +#define FEATURE_DEV_ID_UNUSED	(-1)
> +
>  /**
>   * struct dfl_feature_platform_data - platform data for feature devices
>   *
> @@ -191,6 +193,7 @@ struct dfl_feature {
>   * @cdev: cdev of feature dev.
>   * @dev: ptr to platform device linked with this platform data.
>   * @dfl_cdev: ptr to container device.
> + * @id: id used for this feature device.
>   * @disable_count: count for port disable.
>   * @num: number for sub features.
>   * @dev_status: dev status (e.g. DEV_STATUS_IN_USE).
> @@ -203,6 +206,7 @@ struct dfl_feature_platform_data {
>  	struct cdev cdev;
>  	struct platform_device *dev;
>  	struct dfl_fpga_cdev *dfl_cdev;
> +	int id;
>  	unsigned int disable_count;
>  	unsigned long dev_status;
>  	void *private;
> @@ -378,6 +382,7 @@ int dfl_fpga_enum_info_add_dfl(struct dfl_fpga_enum_info *info,
>   * @fme_dev: FME feature device under this container device.
>   * @lock: mutex lock to protect the port device list.
>   * @port_dev_list: list of all port feature devices under this container device.
> + * @released_port_num: released port number under this container device.
>   */
>  struct dfl_fpga_cdev {
>  	struct device *parent;
> @@ -385,6 +390,7 @@ struct dfl_fpga_cdev {
>  	struct device *fme_dev;
>  	struct mutex lock;
>  	struct list_head port_dev_list;
> +	int released_port_num;
>  };
>  
>  struct dfl_fpga_cdev *
> @@ -412,4 +418,8 @@ struct platform_device *
>  
>  	return pdev;
>  }
> +
> +int dfl_fpga_cdev_config_port(struct dfl_fpga_cdev *cdev,
> +			      u32 port_id, bool release);
> +
>  #endif /* __FPGA_DFL_H */
> diff --git a/include/uapi/linux/fpga-dfl.h b/include/uapi/linux/fpga-dfl.h
> index 2e324e5..e9a00e0 100644
> --- a/include/uapi/linux/fpga-dfl.h
> +++ b/include/uapi/linux/fpga-dfl.h
> @@ -176,4 +176,36 @@ struct dfl_fpga_fme_port_pr {
>  
>  #define DFL_FPGA_FME_PORT_PR	_IO(DFL_FPGA_MAGIC, DFL_FME_BASE + 0)
>  
> +/**
> + * DFL_FPGA_FME_PORT_RELEASE - _IOW(DFL_FPGA_MAGIC, DFL_FME_BASE + 1,
> + *					struct dfl_fpga_fme_port_release)
> + *
> + * Driver releases the port per Port ID provided by caller.
> + * Return: 0 on success, -errno on failure.
> + */
> +struct dfl_fpga_fme_port_release {
> +	/* Input */
> +	__u32 argsz;		/* Structure length */
> +	__u32 flags;		/* Zero for now */
> +	__u32 port_id;
> +};
> +
> +#define DFL_FPGA_FME_PORT_RELEASE	_IO(DFL_FPGA_MAGIC, DFL_FME_BASE + 1)
> +
> +/**
> + * DFL_FPGA_FME_PORT_ASSIGN - _IOW(DFL_FPGA_MAGIC, DFL_FME_BASE + 2,
> + *					struct dfl_fpga_fme_port_assign)
> + *
> + * Driver assigns the port back per Port ID provided by caller.
> + * Return: 0 on success, -errno on failure.
> + */
> +struct dfl_fpga_fme_port_assign {
> +	/* Input */
> +	__u32 argsz;		/* Structure length */
> +	__u32 flags;		/* Zero for now */
> +	__u32 port_id;
> +};
> +
> +#define DFL_FPGA_FME_PORT_ASSIGN	_IO(DFL_FPGA_MAGIC, DFL_FME_BASE + 2)
> +
>  #endif /* _UAPI_LINUX_FPGA_DFL_H */
> -- 
> 1.8.3.1
>