Re: [PATCH v4 13/17] remoteproc: create vdev subdevice with specific dma memory pool

[Suman Anna]

On 9/27/18 3:18 PM, Wendy Liang wrote:
> Hi Loic,
> 
> 
> On Thu, Sep 27, 2018 at 12:22 PM Loic PALLARDY <loic.pallardy@xxxxxx> wrote:
>>
>> Hi Wendy
>>
>>> -----Original Message-----
>>> From: Wendy Liang <sunnyliangjy@xxxxxxxxx>
>>> Sent: Thursday, September 27, 2018 7:17 PM
>>> To: Loic PALLARDY <loic.pallardy@xxxxxx>
>>> Cc: Bjorn Andersson <bjorn.andersson@xxxxxxxxxx>; Ohad Ben-Cohen
>>> <ohad@xxxxxxxxxx>; linux-remoteproc@xxxxxxxxxxxxxxx; Linux Kernel
>>> Mailing List <linux-kernel@xxxxxxxxxxxxxxx>; Arnaud POULIQUEN
>>> <arnaud.pouliquen@xxxxxx>; benjamin.gaignard@xxxxxxxxxx; Suman Anna
>>> <s-anna@xxxxxx>
>>> Subject: Re: [PATCH v4 13/17] remoteproc: create vdev subdevice with
>>> specific dma memory pool
>>>
>>> On Fri, Jul 27, 2018 at 6:16 AM Loic Pallardy <loic.pallardy@xxxxxx> wrote:
>>>>
>>>> This patch creates a dedicated vdev subdevice for each vdev declared
>>>> in firmware resource table and associates carveout named "vdev%dbuffer"
>>>> (with %d vdev index in resource table) if any as dma coherent memory
>>> pool.
>>>>
>>>> Then vdev subdevice is used as parent for virtio device.
>>>>
>>>> Signed-off-by: Loic Pallardy <loic.pallardy@xxxxxx>
>>>> ---
>>>>  drivers/remoteproc/remoteproc_core.c     | 35
>>> +++++++++++++++++++++++---
>>>>  drivers/remoteproc/remoteproc_internal.h |  1 +
>>>>  drivers/remoteproc/remoteproc_virtio.c   | 42
>>> +++++++++++++++++++++++++++++++-
>>>>  include/linux/remoteproc.h               |  1 +
>>>>  4 files changed, 75 insertions(+), 4 deletions(-)
>>>>
>>>> diff --git a/drivers/remoteproc/remoteproc_core.c
>>> b/drivers/remoteproc/remoteproc_core.c
>>>> index 4edc6f0..adcc66e 100644
>>>> --- a/drivers/remoteproc/remoteproc_core.c
>>>> +++ b/drivers/remoteproc/remoteproc_core.c
>>>> @@ -39,6 +39,7 @@
>>>>  #include <linux/idr.h>
>>>>  #include <linux/elf.h>
>>>>  #include <linux/crc32.h>
>>>> +#include <linux/of_reserved_mem.h>
>>>>  #include <linux/virtio_ids.h>
>>>>  #include <linux/virtio_ring.h>
>>>>  #include <asm/byteorder.h>
>>>> @@ -145,7 +146,7 @@ static void rproc_disable_iommu(struct rproc
>>> *rproc)
>>>>         iommu_domain_free(domain);
>>>>  }
>>>>
>>>> -static phys_addr_t rproc_va_to_pa(void *cpu_addr)
>>>> +phys_addr_t rproc_va_to_pa(void *cpu_addr)
>>>>  {
>>>>         /*
>>>>          * Return physical address according to virtual address location
>>>> @@ -160,6 +161,7 @@ static phys_addr_t rproc_va_to_pa(void
>>> *cpu_addr)
>>>>         WARN_ON(!virt_addr_valid(cpu_addr));
>>>>         return virt_to_phys(cpu_addr);
>>>>  }
>>>> +EXPORT_SYMBOL(rproc_va_to_pa);
>>>>
>>>>  /**
>>>>   * rproc_da_to_va() - lookup the kernel virtual address for a remoteproc
>>> address
>>>> @@ -423,6 +425,20 @@ static void rproc_vdev_do_stop(struct
>>> rproc_subdev *subdev, bool crashed)
>>>>  }
>>>>
>>>>  /**
>>>> + * rproc_rvdev_release() - release the existence of a rvdev
>>>> + *
>>>> + * @dev: the subdevice's dev
>>>> + */
>>>> +static void rproc_rvdev_release(struct device *dev)
>>>> +{
>>>> +       struct rproc_vdev *rvdev = container_of(dev, struct rproc_vdev, dev);
>>>> +
>>>> +       of_reserved_mem_device_release(dev);
>>>> +
>>>> +       kfree(rvdev);
>>>> +}
>>>> +
>>>> +/**
>>>>   * rproc_handle_vdev() - handle a vdev fw resource
>>>>   * @rproc: the remote processor
>>>>   * @rsc: the vring resource descriptor
>>>> @@ -455,6 +471,7 @@ static int rproc_handle_vdev(struct rproc *rproc,
>>> struct fw_rsc_vdev *rsc,
>>>>         struct device *dev = &rproc->dev;
>>>>         struct rproc_vdev *rvdev;
>>>>         int i, ret;
>>>> +       char name[16];
>>>>
>>>>         /* make sure resource isn't truncated */
>>>>         if (sizeof(*rsc) + rsc->num_of_vrings * sizeof(struct
>>> fw_rsc_vdev_vring)
>>>> @@ -488,6 +505,18 @@ static int rproc_handle_vdev(struct rproc *rproc,
>>> struct fw_rsc_vdev *rsc,
>>>>         rvdev->rproc = rproc;
>>>>         rvdev->index = rproc->nb_vdev++;
>>>>
>>>> +       /* Initialise vdev subdevice */
>>>> +       snprintf(name, sizeof(name), "vdev%dbuffer", rvdev->index);
>>>> +       rvdev->dev.parent = rproc->dev.parent;
>>>> +       rvdev->dev.release = rproc_rvdev_release;
>>>> +       dev_set_name(&rvdev->dev, "%s#%s", dev_name(rvdev-
>>>> dev.parent), name);
>>>> +       dev_set_drvdata(&rvdev->dev, rvdev);
>>>> +       dma_set_coherent_mask(&rvdev->dev, DMA_BIT_MASK(32));
>>> I tried the latest kernel, this function will not set the DMA coherent mask as
>>> dma_supported() of the &rvdev->dev will return false.
>>> As this is a device created at run time, should it be force to support DMA?
>>> should it directly set the dma_coherent_mask?
>>
>> Thanks for pointing me this issue. I tested on top of 4.18-rc1 few months ago...
>> Could you please give me kernel version on which you are testing the series?
>> Is you platform 32bit or 64bit ?
>> I'll rebase and check on my side.
> 
> I am testing with 4.19-rc4 on aarch64 platform.

Btw, I ran into this on my v7 platform as well (4.19-rc6). The
dma_set_coherent_mask fails with error EIO. I did get my allocations
through though.

regards
Suman

> 
> Best Regards,
> Wendy
>>
>> Regards,
>> Loic
>>
>>>
>>>> +
>>>> +       ret = device_register(&rvdev->dev);
>>>> +       if (ret)
>>>> +               goto free_rvdev;
>>>> +
>>>>         /* parse the vrings */
>>>>         for (i = 0; i < rsc->num_of_vrings; i++) {
>>>>                 ret = rproc_parse_vring(rvdev, rsc, i);
>>>> @@ -518,7 +547,7 @@ static int rproc_handle_vdev(struct rproc *rproc,
>>> struct fw_rsc_vdev *rsc,
>>>>         for (i--; i >= 0; i--)
>>>>                 rproc_free_vring(&rvdev->vring[i]);
>>>>  free_rvdev:
>>>> -       kfree(rvdev);
>>>> +       device_unregister(&rvdev->dev);
>>>>         return ret;
>>>>  }
>>>>
>>>> @@ -536,7 +565,7 @@ void rproc_vdev_release(struct kref *ref)
>>>>
>>>>         rproc_remove_subdev(rproc, &rvdev->subdev);
>>>>         list_del(&rvdev->node);
>>>> -       kfree(rvdev);
>>>> +       device_unregister(&rvdev->dev);
>>>>  }
>>>>
>>>>  /**
>>>> diff --git a/drivers/remoteproc/remoteproc_internal.h
>>> b/drivers/remoteproc/remoteproc_internal.h
>>>> index f6cad24..bfeacfd 100644
>>>> --- a/drivers/remoteproc/remoteproc_internal.h
>>>> +++ b/drivers/remoteproc/remoteproc_internal.h
>>>> @@ -52,6 +52,7 @@ struct dentry *rproc_create_trace_file(const char
>>> *name, struct rproc *rproc,
>>>>  int rproc_alloc_vring(struct rproc_vdev *rvdev, int i);
>>>>
>>>>  void *rproc_da_to_va(struct rproc *rproc, u64 da, int len);
>>>> +phys_addr_t rproc_va_to_pa(void *cpu_addr);
>>>>  int rproc_trigger_recovery(struct rproc *rproc);
>>>>
>>>>  int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw);
>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c
>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>> index de21f62..9ee63c0 100644
>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>> @@ -17,7 +17,9 @@
>>>>   * GNU General Public License for more details.
>>>>   */
>>>>
>>>> +#include <linux/dma-mapping.h>
>>>>  #include <linux/export.h>
>>>> +#include <linux/of_reserved_mem.h>
>>>>  #include <linux/remoteproc.h>
>>>>  #include <linux/virtio.h>
>>>>  #include <linux/virtio_config.h>
>>>> @@ -315,10 +317,48 @@ static void rproc_virtio_dev_release(struct device
>>> *dev)
>>>>  int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id)
>>>>  {
>>>>         struct rproc *rproc = rvdev->rproc;
>>>> -       struct device *dev = &rproc->dev;
>>>> +       struct device *dev = &rvdev->dev;
>>>>         struct virtio_device *vdev = &rvdev->vdev;
>>>> +       struct rproc_mem_entry *mem;
>>>>         int ret;
>>>>
>>>> +       /* Try to find dedicated vdev buffer carveout */
>>>> +       mem = rproc_find_carveout_by_name(rproc, "vdev%dbuffer", rvdev-
>>>> index);
>>>> +       if (mem) {
>>>> +               phys_addr_t pa;
>>>> +
>>>> +               if (mem->of_resm_idx != -1) {
>>>> +                       struct device_node *np = rproc->dev.parent->of_node;
>>>> +
>>>> +                       /* Associate reserved memory to vdev device */
>>>> +                       ret = of_reserved_mem_device_init_by_idx(dev, np,
>>>> +                                                                mem->of_resm_idx);
>>>> +                       if (ret) {
>>>> +                               dev_err(dev, "Can't associate reserved memory\n");
>>>> +                               goto out;
>>>> +                       }
>>>> +               } else {
>>>> +                       if (mem->va) {
>>>> +                               dev_warn(dev, "vdev %d buffer already mapped\n",
>>>> +                                        rvdev->index);
>>>> +                               pa = rproc_va_to_pa(mem->va);
>>>> +                       } else {
>>>> +                               /* Use dma address as carveout no memmapped yet */
>>>> +                               pa = (phys_addr_t)mem->dma;
>>>> +                       }
>>>> +
>>>> +                       /* Associate vdev buffer memory pool to vdev subdev */
>>>> +                       ret = dmam_declare_coherent_memory(dev, pa,
>>>> +                                                          mem->da,
>>>> +                                                          mem->len,
>>>> +                                                          DMA_MEMORY_EXCLUSIVE);
>>>> +                       if (ret < 0) {
>>>> +                               dev_err(dev, "Failed to associate buffer\n");
>>>> +                               goto out;
>>>> +                       }
>>>> +               }
>>>> +       }
>>>> +
>>>>         vdev->id.device = id,
>>>>         vdev->config = &rproc_virtio_config_ops,
>>>>         vdev->dev.parent = dev;
>>>> diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h
>>>> index 6b3a234..2921dd2 100644
>>>> --- a/include/linux/remoteproc.h
>>>> +++ b/include/linux/remoteproc.h
>>>> @@ -547,6 +547,7 @@ struct rproc_vdev {
>>>>         struct kref refcount;
>>>>
>>>>         struct rproc_subdev subdev;
>>>> +       struct device dev;
>>>>
>>>>         unsigned int id;
>>>>         struct list_head node;
>>>> --
>>>> 1.9.1
>>>>
>>>> --
>>>> To unsubscribe from this list: send the line "unsubscribe linux-remoteproc"
>>> in
>>>> the body of a message to majordomo@xxxxxxxxxxxxxxx
>>>> More majordomo info at  http://vger.kernel.org/majordomo-info.html