Re: [PATCH v4 09/24] fpga: dfl-pci: add enumeration for feature devices

[Wu Hao]

On Tue, Mar 13, 2018 at 01:30:24PM -0500, Alan Tull wrote:
> On Tue, Feb 13, 2018 at 3:24 AM, Wu Hao <hao.wu@xxxxxxxxx> wrote:
> 
> Hi Hao,
> 
> Thanks again for splitting the pci part of the code from enumeration
> and everything else.
> 
> One thing that may need to be fixed below, so with that fixed, adding my ack.

Hi Alan

Thanks a lot for your review and acked-by on these patches,

please see my replies below. : )

> 
> > The Device Feature List (DFL) is implemented in MMIO, and features
> > are linked via the DFLs. This patch enables pcie driver to prepare
> > enumeration information (e.g locations of all device feature lists
> > in MMIO) and use common APIs provided by the Device Feature List
> > framework to enumerate each feature device linked.
> >
> > Signed-off-by: Tim Whisonant <tim.whisonant@xxxxxxxxx>
> > Signed-off-by: Enno Luebbers <enno.luebbers@xxxxxxxxx>
> > Signed-off-by: Shiva Rao <shiva.rao@xxxxxxxxx>
> > Signed-off-by: Christopher Rauer <christopher.rauer@xxxxxxxxx>
> > Signed-off-by: Zhang Yi <yi.z.zhang@xxxxxxxxx>
> > Signed-off-by: Xiao Guangrong <guangrong.xiao@xxxxxxxxxxxxxxx>
> > Signed-off-by: Wu Hao <hao.wu@xxxxxxxxx>
> Acked-by: Alan Tull <atull@xxxxxxxxxx>
> 
> > ---
> > v3: split from another patch
> >     use common functions from DFL framework for enumeration.
> > v4: rebase
> > ---
> >  drivers/fpga/dfl-pci.c | 199 ++++++++++++++++++++++++++++++++++++++++++++++++-
> >  1 file changed, 197 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c
> > index d91ea42..8ce8a94 100644
> > --- a/drivers/fpga/dfl-pci.c
> > +++ b/drivers/fpga/dfl-pci.c
> > @@ -22,9 +22,52 @@
> >  #include <linux/errno.h>
> >  #include <linux/aer.h>
> >
> > +#include "dfl.h"
> > +
> >  #define DRV_VERSION    "0.8"
> >  #define DRV_NAME       "dfl-pci"
> >
> > +struct cci_drvdata {
> > +       struct fpga_cdev *cdev; /* container device */
> > +       struct list_head regions; /* list of pci bar mapping region */
> > +};
> > +
> > +/* pci bar mapping info */
> > +struct cci_region {
> > +       int bar;
> > +       void __iomem *ioaddr;   /* pointer to mapped bar region */
> > +       struct list_head node;
> > +};
> > +
> > +static void __iomem *cci_pci_ioremap_bar(struct pci_dev *pcidev, int bar)
> > +{
> > +       struct cci_drvdata *drvdata = pci_get_drvdata(pcidev);
> > +       struct cci_region *region;
> > +
> > +       list_for_each_entry(region, &drvdata->regions, node)
> > +               if (region->bar == bar) {
> > +                       dev_dbg(&pcidev->dev, "BAR %d region exists\n", bar);
> > +                       return region->ioaddr;
> > +               }
> > +
> > +       region = devm_kzalloc(&pcidev->dev, sizeof(*region), GFP_KERNEL);
> > +       if (!region)
> > +               return NULL;
> > +
> > +       region->bar = bar;
> > +       region->ioaddr = pci_ioremap_bar(pcidev, bar);
> > +       if (!region->ioaddr) {
> > +               dev_err(&pcidev->dev, "can't ioremap memory from BAR %d.\n",
> > +                       bar);
> > +               devm_kfree(&pcidev->dev, region);
> > +               return NULL;
> > +       }
> > +
> > +       list_add(&region->node, &drvdata->regions);
> > +
> > +       return region->ioaddr;
> > +}
> > +
> >  /* PCI Device ID */
> >  #define PCIE_DEVICE_ID_PF_INT_5_X      0xBCBD
> >  #define PCIE_DEVICE_ID_PF_INT_6_X      0xBCC0
> > @@ -45,6 +88,143 @@ static struct pci_device_id cci_pcie_id_tbl[] = {
> >  };
> >  MODULE_DEVICE_TABLE(pci, cci_pcie_id_tbl);
> >
> > +static int cci_init_drvdata(struct pci_dev *pcidev)
> > +{
> > +       struct cci_drvdata *drvdata;
> > +
> > +       drvdata = devm_kzalloc(&pcidev->dev, sizeof(*drvdata), GFP_KERNEL);
> > +       if (!drvdata)
> > +               return -ENOMEM;
> > +
> > +       INIT_LIST_HEAD(&drvdata->regions);
> > +
> > +       pci_set_drvdata(pcidev, drvdata);
> > +
> > +       return 0;
> > +}
> > +
> > +static void cci_pci_release_regions(struct pci_dev *pcidev)
> > +{
> > +       struct cci_drvdata *drvdata = pci_get_drvdata(pcidev);
> > +       struct cci_region *tmp, *region;
> > +
> > +       list_for_each_entry_safe(region, tmp, &drvdata->regions, node) {
> > +               list_del(&region->node);
> > +               if (region->ioaddr)
> > +                       pci_iounmap(pcidev, region->ioaddr);
> > +               devm_kfree(&pcidev->dev, region);
> > +       }
> > +}
> > +
> > +static void cci_remove_drvdata(struct pci_dev *pcidev)
> > +{
> > +       struct cci_drvdata *drvdata = pci_get_drvdata(pcidev);
> > +
> > +       cci_pci_release_regions(pcidev);
> 
> Would it make sense to call fpga_enum_info_free here?  I understand
> fpga_enum_info_alloc uses devm, but it does a get_device which needs
> to be put.
> 
> > +       pci_set_drvdata(pcidev, NULL);
> > +       devm_kfree(&pcidev->dev, drvdata);
> > +}
> > +
> > +static void cci_remove_feature_devs(struct pci_dev *pcidev)
> > +{
> > +       struct cci_drvdata *drvdata = pci_get_drvdata(pcidev);
> > +
> > +       /* remove all children feature devices */
> > +       fpga_remove_feature_devs(drvdata->cdev);
> > +}
> > +
> > +/* enumerate feature devices under pci device */
> > +static int cci_enumerate_feature_devs(struct pci_dev *pcidev)
> > +{
> > +       struct cci_drvdata *drvdata = pci_get_drvdata(pcidev);
> > +       struct fpga_cdev *cdev;
> > +       struct fpga_enum_info *info;
> > +       resource_size_t start, len;
> > +       void __iomem *base;
> > +       int port_num, bar, i, ret = 0;
> > +       u32 offset;
> > +       u64 v;
> > +
> > +       /* allocate enumeration info via pci_dev */
> > +       info = fpga_enum_info_alloc(&pcidev->dev);
> > +       if (!info)
> > +               return -ENOMEM;
> > +
> > +       /* start to find Device Feature List from Bar 0 */
> > +       base = cci_pci_ioremap_bar(pcidev, 0);
> > +       if (!base) {
> > +               ret = -ENOMEM;
> > +               goto enum_info_free_exit;
> > +       }
> > +
> > +       /*
> > +        * PF device has FME and Ports/AFUs, and VF device only has 1 Port/AFU.
> > +        * check them and add related "Device Feature List" info for the next
> > +        * step enumeration.
> > +        */
> > +       if (feature_is_fme(base)) {
> > +               start = pci_resource_start(pcidev, 0);
> > +               len = pci_resource_len(pcidev, 0);
> > +
> > +               fpga_enum_info_add_dfl(info, start, len, base);
> > +
> > +               /*
> > +                * find more Device Feature Lists (e.g Ports) per information
> > +                * indicated by FME module.
> > +                */
> > +               v = readq(base + FME_HDR_CAP);
> > +               port_num = FIELD_GET(FME_CAP_NUM_PORTS, v);
> > +
> > +               WARN_ON(port_num > MAX_FPGA_PORT_NUM);
> > +
> > +               for (i = 0; i < port_num; i++) {
> > +                       v = readq(base + FME_HDR_PORT_OFST(i));
> > +
> > +                       /* skip ports which are not implemented. */
> > +                       if (!(v & FME_PORT_OFST_IMP))
> > +                               continue;
> > +
> > +                       /*
> > +                        * add Port's Device Feature List information for next
> > +                        * step enumeration.
> > +                        */
> > +                       bar = FIELD_GET(FME_PORT_OFST_BAR_ID, v);
> > +                       offset = FIELD_GET(FME_PORT_OFST_DFH_OFST, v);
> > +                       base = cci_pci_ioremap_bar(pcidev, bar);
> > +                       if (!base)
> > +                               continue;
> > +
> > +                       start = pci_resource_start(pcidev, bar) + offset;
> > +                       len = pci_resource_len(pcidev, bar) - offset;
> > +
> > +                       fpga_enum_info_add_dfl(info, start, len, base + offset);
> > +               }
> > +       } else if (feature_is_port(base)) {
> > +               start = pci_resource_start(pcidev, 0);
> > +               len = pci_resource_len(pcidev, 0);
> > +
> > +               fpga_enum_info_add_dfl(info, start, len, base);
> > +       } else {
> > +               ret = -ENODEV;
> > +               goto enum_info_free_exit;
> > +       }
> > +
> > +       /* start enumeration with prepared enumeration information */
> > +       cdev = fpga_enumerate_feature_devs(info);
> > +       if (IS_ERR(cdev)) {
> > +               dev_err(&pcidev->dev, "Enumeration failure\n");
> > +               ret = PTR_ERR(cdev);
> > +               goto enum_info_free_exit;
> > +       }
> > +
> > +       drvdata->cdev = cdev;
> > +
> > +enum_info_free_exit:
> > +       fpga_enum_info_free(info);
> 
> This is the only place I saw fpga_enum_info_free being called.

It doesn't need to keep the enumeration inforamtion data structure once
the enumeration done, so in the driver, it always did fpga_enum_info_free
once fpga_enumerate_feature_devs(info) returned in this function. so
no need to consider it in other places per my understanding. : )

Thanks
Hao

> 
> Thanks,
> Alan
> 
> > +
> > +       return ret;
> > +}
> > +
> >  static
> >  int cci_pci_probe(struct pci_dev *pcidev, const struct pci_device_id *pcidevid)
> >  {
> > @@ -82,9 +262,22 @@ int cci_pci_probe(struct pci_dev *pcidev, const struct pci_device_id *pcidevid)
> >                 goto release_region_exit;
> >         }
> >
> > -       /* TODO: create and add the platform device per feature list */
> > -       return 0;
> > +       ret = cci_init_drvdata(pcidev);
> > +       if (ret) {
> > +               dev_err(&pcidev->dev, "Fail to init drvdata %d.\n", ret);
> > +               goto release_region_exit;
> > +       }
> > +
> > +       ret = cci_enumerate_feature_devs(pcidev);
> > +       if (ret) {
> > +               dev_err(&pcidev->dev, "enumeration failure %d.\n", ret);
> > +               goto remove_drvdata_exit;
> > +       }
> > +
> > +       return ret;
> >
> > +remove_drvdata_exit:
> > +       cci_remove_drvdata(pcidev);
> >  release_region_exit:
> >         pci_release_regions(pcidev);
> >  disable_error_report_exit:
> > @@ -95,6 +288,8 @@ int cci_pci_probe(struct pci_dev *pcidev, const struct pci_device_id *pcidevid)
> >
> >  static void cci_pci_remove(struct pci_dev *pcidev)
> >  {
> > +       cci_remove_feature_devs(pcidev);
> > +       cci_remove_drvdata(pcidev);
> >         pci_release_regions(pcidev);
> >         pci_disable_pcie_error_reporting(pcidev);
> >         pci_disable_device(pcidev);
> > --
> > 2.7.4
> >