Re: d63e2e1f3df breaks sparc/T5-8
From: Yinghai Lu
Date: Fri Mar 27 2015 - 19:57:30 EST
On Fri, Mar 27, 2015 at 2:01 PM, Yinghai Lu <yinghai@xxxxxxxxxx> wrote:
> On Thu, Mar 26, 2015 at 4:27 PM, David Ahern <david.ahern@xxxxxxxxxx> wrote:
> Also please make sure your config have
>
> CONFIG_PCI_DEBUG=y
>
> and capture serial console with "debug ignore_loglevel", so we check if
> pci 0000:00:01.0 really have resource assigned.
Please check attached patch and send out boot log with above config.
Thanks
Yinghai
Subject: [RFC PATCH] PCI: Introduce pci_bus_addr_t
David Ahern found commit d63e2e1f3df9 ("sparc/PCI: Clip bridge windows
to fit in upstream windows") broke sparc/T5-8.
In the boot log, there is
pci 0000:06:00.0: reg 0x184: can't handle BAR above 4GB (bus address
0x110204000)
but that only could happen when dma_addr_t is 32-bit.
According to David Miller, all DMA occurs behind an IOMMU and these
IOMMUs only support 32-bit addressing, therefore dma_addr_t is
32-bit on sparc64.
Let's introduce pci_bus_addr instead of using dma_addr_t.
Fixes: commit d63e2e1f3df9 ("sparc/PCI: Clip bridge windows to fit in upstream windows")
Fixes: commit 23b13bc76f35 ("PCI: Fail safely if we can't handle BARs larger than 4GB")
Link: http://lkml.kernel.org/r/CAE9FiQU1gJY1LYrxs+ma5LCTEEe4xmtjRG0aXJ9K_Tsu+m9Wuw@xxxxxxxxxxxxxx
Reported-by: David Ahern <david.ahern@xxxxxxxxxx>
Signed-off-by: Yinghai Lu <yinghai@xxxxxxxxxx>
---
arch/alpha/Kconfig | 3 +++
arch/arm/Kconfig | 1 +
arch/arm/mach-axxia/Kconfig | 1 +
arch/arm/mach-exynos/Kconfig | 1 +
arch/arm/mach-highbank/Kconfig | 1 +
arch/arm/mach-shmobile/Kconfig | 1 +
arch/arm/mm/Kconfig | 3 +++
arch/arm64/Kconfig | 3 +++
arch/ia64/Kconfig | 3 +++
arch/mips/Kconfig | 3 +++
arch/powerpc/Kconfig | 3 +++
arch/sparc/Kconfig | 3 +++
arch/tile/Kconfig | 3 +++
arch/x86/Kconfig | 4 ++++
drivers/pci/bus.c | 8 ++++----
drivers/pci/probe.c | 12 ++++++------
include/linux/pci.h | 6 +++---
include/linux/types.h | 7 +++++++
18 files changed, 53 insertions(+), 13 deletions(-)
Index: linux-2.6/arch/alpha/Kconfig
===================================================================
--- linux-2.6.orig/arch/alpha/Kconfig
+++ linux-2.6/arch/alpha/Kconfig
@@ -66,6 +66,9 @@ config ZONE_DMA
config ARCH_DMA_ADDR_T_64BIT
def_bool y
+config ARCH_PCI_BUS_ADDR_T_64BIT
+ def_bool y
+
config NEED_DMA_MAP_STATE
def_bool y
Index: linux-2.6/arch/arm/Kconfig
===================================================================
--- linux-2.6.orig/arch/arm/Kconfig
+++ linux-2.6/arch/arm/Kconfig
@@ -1779,6 +1779,7 @@ config XEN
depends on !GENERIC_ATOMIC64
depends on MMU
select ARCH_DMA_ADDR_T_64BIT
+ select ARCH_PCI_BUS_ADDR_T_64BIT
select ARM_PSCI
select SWIOTLB_XEN
help
Index: linux-2.6/arch/arm/mach-axxia/Kconfig
===================================================================
--- linux-2.6.orig/arch/arm/mach-axxia/Kconfig
+++ linux-2.6/arch/arm/mach-axxia/Kconfig
@@ -1,6 +1,7 @@
config ARCH_AXXIA
bool "LSI Axxia platforms" if (ARCH_MULTI_V7 && ARM_LPAE)
select ARCH_DMA_ADDR_T_64BIT
+ select ARCH_PCI_BUS_ADDR_T_64BIT
select ARM_AMBA
select ARM_GIC
select ARM_TIMER_SP804
Index: linux-2.6/arch/arm/mach-exynos/Kconfig
===================================================================
--- linux-2.6.orig/arch/arm/mach-exynos/Kconfig
+++ linux-2.6/arch/arm/mach-exynos/Kconfig
@@ -106,6 +106,7 @@ config SOC_EXYNOS5440
default y
depends on ARCH_EXYNOS5
select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
+ select ARCH_PCI_BUS_ADDR_T_64BIT if ARM_LPAE
select HAVE_ARM_ARCH_TIMER
select AUTO_ZRELADDR
select MIGHT_HAVE_PCI
Index: linux-2.6/arch/arm/mach-highbank/Kconfig
===================================================================
--- linux-2.6.orig/arch/arm/mach-highbank/Kconfig
+++ linux-2.6/arch/arm/mach-highbank/Kconfig
@@ -1,6 +1,7 @@
config ARCH_HIGHBANK
bool "Calxeda ECX-1000/2000 (Highbank/Midway)" if ARCH_MULTI_V7
select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
+ select ARCH_PCI_BUS_ADDR_T_64BIT if ARM_LPAE
select ARCH_HAS_HOLES_MEMORYMODEL
select ARCH_SUPPORTS_BIG_ENDIAN
select ARM_AMBA
Index: linux-2.6/arch/arm/mach-shmobile/Kconfig
===================================================================
--- linux-2.6.orig/arch/arm/mach-shmobile/Kconfig
+++ linux-2.6/arch/arm/mach-shmobile/Kconfig
@@ -36,6 +36,7 @@ menuconfig ARCH_SHMOBILE_MULTI
select HAVE_ARM_TWD if SMP
select ARM_GIC
select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
+ select ARCH_PCI_BUS_ADDR_T_64BIT if ARM_LPAE
select NO_IOPORT_MAP
select PINCTRL
select ARCH_REQUIRE_GPIOLIB
Index: linux-2.6/arch/arm/mm/Kconfig
===================================================================
--- linux-2.6.orig/arch/arm/mm/Kconfig
+++ linux-2.6/arch/arm/mm/Kconfig
@@ -630,6 +630,9 @@ config ARCH_PHYS_ADDR_T_64BIT
config ARCH_DMA_ADDR_T_64BIT
bool
+config ARCH_PCI_BUS_ADDR_T_64BIT
+ bool
+
config ARM_THUMB
bool "Support Thumb user binaries" if !CPU_THUMBONLY
depends on CPU_ARM720T || CPU_ARM740T || CPU_ARM920T || CPU_ARM922T || \
Index: linux-2.6/arch/arm64/Kconfig
===================================================================
--- linux-2.6.orig/arch/arm64/Kconfig
+++ linux-2.6/arch/arm64/Kconfig
@@ -125,6 +125,9 @@ config HAVE_GENERIC_RCU_GUP
config ARCH_DMA_ADDR_T_64BIT
def_bool y
+config ARCH_PCI_BUS_ADDR_T_64BIT
+ def_bool y
+
config NEED_DMA_MAP_STATE
def_bool y
Index: linux-2.6/arch/ia64/Kconfig
===================================================================
--- linux-2.6.orig/arch/ia64/Kconfig
+++ linux-2.6/arch/ia64/Kconfig
@@ -74,6 +74,9 @@ config MMU
config ARCH_DMA_ADDR_T_64BIT
def_bool y
+config ARCH_PCI_BUS_ADDR_T_64BIT
+ def_bool y
+
config NEED_DMA_MAP_STATE
def_bool y
Index: linux-2.6/arch/mips/Kconfig
===================================================================
--- linux-2.6.orig/arch/mips/Kconfig
+++ linux-2.6/arch/mips/Kconfig
@@ -984,6 +984,9 @@ config FW_CFE
config ARCH_DMA_ADDR_T_64BIT
def_bool (HIGHMEM && ARCH_PHYS_ADDR_T_64BIT) || 64BIT
+config ARCH_PCI_BUS_ADDR_T_64BIT
+ def_bool (HIGHMEM && ARCH_PHYS_ADDR_T_64BIT) || 64BIT
+
config DMA_MAYBE_COHERENT
select DMA_NONCOHERENT
bool
Index: linux-2.6/arch/powerpc/Kconfig
===================================================================
--- linux-2.6.orig/arch/powerpc/Kconfig
+++ linux-2.6/arch/powerpc/Kconfig
@@ -23,6 +23,9 @@ config ARCH_PHYS_ADDR_T_64BIT
config ARCH_DMA_ADDR_T_64BIT
def_bool ARCH_PHYS_ADDR_T_64BIT
+config ARCH_PCI_ADDR_T_64BIT
+ def_bool ARCH_PHYS_ADDR_T_64BIT
+
config MMU
bool
default y
Index: linux-2.6/arch/sparc/Kconfig
===================================================================
--- linux-2.6.orig/arch/sparc/Kconfig
+++ linux-2.6/arch/sparc/Kconfig
@@ -143,6 +143,9 @@ config GENERIC_ISA_DMA
bool
default y if SPARC32
+config ARCH_PCI_BUS_ADDR_T_64BIT
+ def_bool y if SPARC64
+
config ARCH_SUPPORTS_DEBUG_PAGEALLOC
def_bool y if SPARC64
Index: linux-2.6/arch/tile/Kconfig
===================================================================
--- linux-2.6.orig/arch/tile/Kconfig
+++ linux-2.6/arch/tile/Kconfig
@@ -86,6 +86,9 @@ config ARCH_PHYS_ADDR_T_64BIT
config ARCH_DMA_ADDR_T_64BIT
def_bool y
+config ARCH_PCI_BUS_ADDR_T_64BIT
+ def_bool y
+
config NEED_DMA_MAP_STATE
def_bool y
Index: linux-2.6/arch/x86/Kconfig
===================================================================
--- linux-2.6.orig/arch/x86/Kconfig
+++ linux-2.6/arch/x86/Kconfig
@@ -1295,6 +1295,10 @@ config ARCH_DMA_ADDR_T_64BIT
def_bool y
depends on X86_64 || HIGHMEM64G
+config ARCH_PCI_BUS_ADDR_T_64BIT
+ def_bool y
+ depends on X86_64 || HIGHMEM64G
+
config X86_DIRECT_GBPAGES
def_bool y
depends on X86_64 && !DEBUG_PAGEALLOC && !KMEMCHECK
Index: linux-2.6/drivers/pci/bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/bus.c
+++ linux-2.6/drivers/pci/bus.c
@@ -92,11 +92,11 @@ void pci_bus_remove_resources(struct pci
}
static struct pci_bus_region pci_32_bit = {0, 0xffffffffULL};
-#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+#ifdef CONFIG_ARCH_PCI_BUS_ADDR_T_64BIT
static struct pci_bus_region pci_64_bit = {0,
- (dma_addr_t) 0xffffffffffffffffULL};
-static struct pci_bus_region pci_high = {(dma_addr_t) 0x100000000ULL,
- (dma_addr_t) 0xffffffffffffffffULL};
+ (pci_bus_addr_t) 0xffffffffffffffffULL};
+static struct pci_bus_region pci_high = {(pci_bus_addr_t) 0x100000000ULL,
+ (pci_bus_addr_t) 0xffffffffffffffffULL};
#endif
/*
Index: linux-2.6/drivers/pci/probe.c
===================================================================
--- linux-2.6.orig/drivers/pci/probe.c
+++ linux-2.6/drivers/pci/probe.c
@@ -254,8 +254,8 @@ int __pci_read_base(struct pci_dev *dev,
}
if (res->flags & IORESOURCE_MEM_64) {
- if ((sizeof(dma_addr_t) < 8 || sizeof(resource_size_t) < 8) &&
- sz64 > 0x100000000ULL) {
+ if ((sizeof(pci_bus_addr_t) < 8 || sizeof(resource_size_t) < 8)
+ && sz64 > 0x100000000ULL) {
res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
res->start = 0;
res->end = 0;
@@ -264,7 +264,7 @@ int __pci_read_base(struct pci_dev *dev,
goto out;
}
- if ((sizeof(dma_addr_t) < 8) && l) {
+ if ((sizeof(pci_bus_addr_t) < 8) && l) {
/* Above 32-bit boundary; try to reallocate */
res->flags |= IORESOURCE_UNSET;
res->start = 0;
@@ -399,7 +399,7 @@ static void pci_read_bridge_mmio_pref(st
struct pci_dev *dev = child->self;
u16 mem_base_lo, mem_limit_lo;
u64 base64, limit64;
- dma_addr_t base, limit;
+ pci_bus_addr_t base, limit;
struct pci_bus_region region;
struct resource *res;
@@ -426,8 +426,8 @@ static void pci_read_bridge_mmio_pref(st
}
}
- base = (dma_addr_t) base64;
- limit = (dma_addr_t) limit64;
+ base = (pci_bus_addr_t) base64;
+ limit = (pci_bus_addr_t) limit64;
if (base != base64) {
dev_err(&dev->dev, "can't handle bridge window above 4GB (bus address %#010llx)\n",
Index: linux-2.6/include/linux/pci.h
===================================================================
--- linux-2.6.orig/include/linux/pci.h
+++ linux-2.6/include/linux/pci.h
@@ -577,8 +577,8 @@ int raw_pci_write(unsigned int domain, u
int reg, int len, u32 val);
struct pci_bus_region {
- dma_addr_t start;
- dma_addr_t end;
+ pci_bus_addr_t start;
+ pci_bus_addr_t end;
};
struct pci_dynids {
@@ -1127,7 +1127,7 @@ int __must_check pci_bus_alloc_resource(
int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr);
-static inline dma_addr_t pci_bus_address(struct pci_dev *pdev, int bar)
+static inline pci_bus_addr_t pci_bus_address(struct pci_dev *pdev, int bar)
{
struct pci_bus_region region;
Index: linux-2.6/include/linux/types.h
===================================================================
--- linux-2.6.orig/include/linux/types.h
+++ linux-2.6/include/linux/types.h
@@ -146,6 +146,13 @@ typedef u64 dma_addr_t;
typedef u32 dma_addr_t;
#endif /* dma_addr_t */
+/* A pci_bus_addr_t can hold pci bus address for the platform */
+#ifdef CONFIG_ARCH_PCI_BUS_ADDR_T_64BIT
+typedef u64 pci_bus_addr_t;
+#else
+typedef u32 pci_bus_addr_t;
+#endif /* pci_bus_addr_t */
+
#ifdef __CHECKER__
#else
#endif