Re: [PATCH v4 4/7] s390/pci: Use dma-iommu layer

[Matthew Rosato]

On 1/4/23 7:05 AM, Niklas Schnelle wrote:
> While s390 already has a standard IOMMU driver and previous changes have
> added I/O TLB flushing operations this driver is currently only used for
> user-space PCI access such as vfio-pci. For the DMA API s390 instead
> utilizes its own implementation in arch/s390/pci/pci_dma.c which drives
> the same hardware and shares some code but requires a complex and
> fragile hand over between DMA API and IOMMU API use of a device and
> despite code sharing still leads to significant duplication and
> maintenance effort. Let's utilize the common code DMAP API
> implementation from drivers/iommu/dma-iommu.c instead allowing us to
> get rid of arch/s390/pci/pci_dma.c.
> 
> Signed-off-by: Niklas Schnelle <schnelle@xxxxxxxxxxxxx>
> diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c
> index ef38b1514c77..6b0fe8761509 100644
> --- a/arch/s390/pci/pci.c
> +++ b/arch/s390/pci/pci.c
> @@ -124,7 +124,11 @@ int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
>  
>  	WARN_ON_ONCE(iota & 0x3fff);
>  	fib.pba = base;
> -	fib.pal = limit;
> +	/* Work around off by one in ISM virt device */
> +	if (zdev->pft == 0x5 && limit > base)

Nit: maybe a named #define for the ISM pft rather than hard-coding 0x5 here

[...]

> diff --git a/arch/s390/pci/pci_bus.c b/arch/s390/pci/pci_bus.c
> index 6a8da1b742ae..b15ad15999f8 100644
> --- a/arch/s390/pci/pci_bus.c
> +++ b/arch/s390/pci/pci_bus.c
> @@ -49,11 +49,6 @@ static int zpci_bus_prepare_device(struct zpci_dev *zdev)
>  		rc = zpci_enable_device(zdev);
>  		if (rc)
>  			return rc;
> -		rc = zpci_dma_init_device(zdev);
> -		if (rc) {
> -			zpci_disable_device(zdev);
> -			return rc;
> -		}
>  	}
>  
>  	if (!zdev->has_resources) {
> diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c
> index ca6bd98eec13..60cec57a3907 100644
> --- a/arch/s390/pci/pci_debug.c
> +++ b/arch/s390/pci/pci_debug.c
> @@ -53,9 +53,12 @@ static char *pci_fmt3_names[] = {
>  };
>  
>  static char *pci_sw_names[] = {
> -	"Allocated pages",
> +/* TODO "Allocated pages", */

? Forgot to finish this?

>  	"Mapped pages",
>  	"Unmapped pages",
> +	"Global RPCITs",
> +	"Sync Map RPCITs",
> +	"Sync RPCITs",
>  };

[...]

> diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
> index 4dfa557270f4..73144ea0adfc 100644
> --- a/drivers/iommu/s390-iommu.c
> +++ b/drivers/iommu/s390-iommu.c
> @@ -14,16 +14,300 @@

> -static void __s390_iommu_detach_device(struct zpci_dev *zdev)
> +static void s390_iommu_detach_device(struct iommu_domain *domain,
> +				     struct device *dev)
>  {
> -	struct s390_domain *s390_domain = zdev->s390_domain;
> +	struct s390_domain *s390_domain = to_s390_domain(domain);
> +	struct zpci_dev *zdev = to_zpci_dev(dev);
>  	unsigned long flags;
>  
> +	WARN_ON(zdev->s390_domain != to_s390_domain(domain));
>  	if (!s390_domain)
>  		return;
>  
> @@ -120,9 +412,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
>  		return -EINVAL;
>  
>  	if (zdev->s390_domain)
> -		__s390_iommu_detach_device(zdev);
> -	else if (zdev->dma_table)
> -		zpci_dma_exit_device(zdev);
> +		s390_iommu_detach_device(&zdev->s390_domain->domain, dev);
>  
>  	cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
>  				virt_to_phys(s390_domain->dma_table), &status);
> @@ -144,17 +434,6 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
>  	return 0;
>  }
>  
> -static void s390_iommu_detach_device(struct iommu_domain *domain,
> -				     struct device *dev)
> -{
> -	struct zpci_dev *zdev = to_zpci_dev(dev);
> -
> -	WARN_ON(zdev->s390_domain != to_s390_domain(domain));
> -
> -	__s390_iommu_detach_device(zdev);
> -	zpci_dma_init_device(zdev);
> -}
> ->  static void s390_iommu_get_resv_regions(struct device *dev,
>  					struct list_head *list)
>  {
> @@ -207,7 +486,7 @@ static void s390_iommu_release_device(struct device *dev)
>  	 * to the device, but keep it attached to other devices in the group.
>  	 */
>  	if (zdev)
> -		__s390_iommu_detach_device(zdev);
> +		s390_iommu_detach_device(&zdev->s390_domain->domain, dev);
>  }
>  
Looks good overall, but I think these hits above collide with Baolu's series that recently went into -next:

https://lore.kernel.org/linux-iommu/20230110025408.667767-1-baolu.lu@xxxxxxxxxxxxxxx/