Re: x2apic boot failure on recent sandy bridge system

From: Yinghai Lu
Date: Sat Dec 10 2011 - 15:17:17 EST

Next message: Thomas Gleixner: "Re: [patch 0/3] kvm tool: Serial emulation overhaul"
Previous message: Fabio Estevam: "[PATCH] rtc: Kconfig: Place RTC_DRV_IMXDI and RTC_MXC under "on-CPU RTC drivers""
In reply to: Berck E. Nash: "x2apic boot failure on recent sandy bridge system"
Next in thread: Berck E. Nash: "Re: x2apic boot failure on recent sandy bridge system"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

On Fri, Dec 9, 2011 at 5:56 PM, Berck E. Nash <flyboy@xxxxxxxxx> wrote:
> X2APIC has never worked on this system, and continues to not work
> 3.2.0-rc5. Simply hangs on boot with an error message that helpfully
> indicates that x2apic is the problem. The system is an ASRock P67
> Extreme 6. Log attached, let me know if I can do anything to help.
> Please cc as I'm not subscribed.

madt said cpu is x2apic.
[ 0.000000] ACPI: X2APIC (apic_id[0x00] uid[0x01] enabled)
[ 0.000000] ACPI: X2APIC (apic_id[0x02] uid[0x02] enabled)
[ 0.000000] ACPI: X2APIC (apic_id[0x04] uid[0x03] enabled)
[ 0.000000] ACPI: X2APIC (apic_id[0x06] uid[0x04] enabled)
[ 0.000000] ACPI: X2APIC (apic_id[0x01] uid[0x05] enabled)
[ 0.000000] ACPI: X2APIC (apic_id[0x03] uid[0x06] enabled)
[ 0.000000] ACPI: X2APIC (apic_id[0x05] uid[0x07] enabled)
[ 0.000000] ACPI: X2APIC (apic_id[0x07] uid[0x08] enabled)
[ 0.000000] ACPI: X2APIC_NMI (uid[0xffffffff] high edge lint[0x1])

but bios does not provide DMAR table...

[ 0.000000] ACPI: RSDP 00000000000f0450 00024 (v02 ALASKA)
[ 0.000000] ACPI: XSDT 00000000bef32068 00054 (v01 ALASKA A M I
01072009 AMI 00010013)
[ 0.000000] ACPI: FACP 00000000bef398f8 000F4 (v04 ALASKA A M I
01072009 AMI 00010013)
[ 0.000000] ACPI: DSDT 00000000bef32150 077A6 (v02 ALASKA A M I
00000000 INTL 20051117)
[ 0.000000] ACPI: FACS 00000000bf60af80 00040
[ 0.000000] ACPI: APIC 00000000bef399f0 000D8 (v03 ALASKA A M I
01072009 AMI 00010013)
[ 0.000000] ACPI: SSDT 00000000bef39ac8 001DE (v01 AMICPU PROC
00000001 MSFT 03000001)
[ 0.000000] ACPI: MCFG 00000000bef39ca8 0003C (v01 ALASKA A M I
01072009 MSFT 00000097)
[ 0.000000] ACPI: AAFT 00000000bef39ce8 0006F (v01 ALASKA OEMAAFT
01072009 MSFT 00000097)
[ 0.000000] ACPI: HPET 00000000bef39d58 00038 (v01 ALASKA A M I
01072009 AMI. 00000004)

so kernel can not enable interrput remapping, then it will panic..

[ 0.085717] Kernel panic - not syncing: x2apic: enabled by BIOS but
kernel init failed.
[ 0.093697] Pid: 1, comm: swapper/0 Not tainted 3.2.0-rc5 #2
[ 0.099337] Call Trace:
[ 0.101788] [<ffffffff8137b7ce>] ? panic+0x95/0x18e
[ 0.106755] [<ffffffff81016259>] ? __ioapic_write_entry+0x16/0x32
[ 0.112925] [<ffffffff8152d5bf>] ? enable_IR_x2apic+0x19a/0x1d7
[ 0.118913] [<ffffffff8152f281>] ? default_setup_apic_routing+0xd/0x71
[ 0.125506] [<ffffffff8152c536>] ? native_smp_prepare_cpus+0x2f1/0x325
[ 0.132102] [<ffffffff81522abf>] ? kernel_init+0x46/0x10f
[ 0.137571] [<ffffffff813841f4>] ? kernel_thread_helper+0x4/0x10
[ 0.143650] [<ffffffff81522a79>] ? start_kernel+0x341/0x341
[ 0.149297] [<ffffffff813841f0>] ? gs_change+0xb/0xb

your bios should not enable x2apic in this case.

please get one bios update.

also I have one old patch that could disable apic. but it is not get
merged and it may not be applied to current tree.

let us know if you can not get BIOS update or can not disable x2apic
in BIOS setup.

Thanks

Yinghai Lu
Subject: [PATCH -v3] x86: Disable x2apic if nox2apic is specified

For
1. x2apic preenabled system
2. first kernel have x2apic enabled, and try to kexec second kernel with "nox2apic"

Will put back cpu with apic id < 255 into xapic mode, instead of panic.

-v2: use x2apic_disabled instead of nox2apic, Suggested by Thomas
update x2apic_supported with x2apic_disabled, Suggested by Thomas

-v3: add checking for boot cpu apic id > 255. in that case will just panic
--- pointed out by Suresh.

Signed-off-by: Yinghai Lu <yinghai@xxxxxxxxxx>

---
arch/x86/include/asm/apic.h | 6 +++-
arch/x86/include/asm/apicdef.h | 1
arch/x86/include/asm/processor.h | 1
arch/x86/kernel/acpi/boot.c | 10 ++++++-
arch/x86/kernel/apic/apic.c | 51 +++++++++++++++++++++++++++++++++------
arch/x86/kernel/cpu/topology.c | 21 ++++++++++++++++
arch/x86/mm/srat.c | 7 ++++-
7 files changed, 85 insertions(+), 12 deletions(-)

Index: linux-2.6/arch/x86/include/asm/apic.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/apic.h
+++ linux-2.6/arch/x86/include/asm/apic.h
@@ -175,6 +175,7 @@ static inline u64 native_x2apic_icr_read
}

extern int x2apic_phys;
+extern int x2apic_disabled;
extern void check_x2apic(void);
extern void enable_x2apic(void);
extern void x2apic_icr_write(u32 low, u32 id);
@@ -182,7 +183,7 @@ static inline int x2apic_enabled(void)
{
u64 msr;

- if (!cpu_has_x2apic)
+ if (!cpu_has_x2apic || x2apic_disabled)
return 0;

rdmsrl(MSR_IA32_APICBASE, msr);
@@ -191,7 +192,7 @@ static inline int x2apic_enabled(void)
return 0;
}

-#define x2apic_supported() (cpu_has_x2apic)
+#define x2apic_supported() (cpu_has_x2apic && !x2apic_disabled)
static inline void x2apic_force_phys(void)
{
x2apic_phys = 1;
@@ -213,6 +214,7 @@ static inline void x2apic_force_phys(voi

#define x2apic_preenabled 0
#define x2apic_supported() 0
+#define x2apic_disabled 1
#endif

extern void enable_IR_x2apic(void);
Index: linux-2.6/arch/x86/kernel/acpi/boot.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/acpi/boot.c
+++ linux-2.6/arch/x86/kernel/acpi/boot.c
@@ -219,6 +219,8 @@ static int __init
acpi_parse_x2apic(struct acpi_subtable_header *header, const unsigned long end)
{
struct acpi_madt_local_x2apic *processor = NULL;
+ int apic_id;
+ u8 enabled;

processor = (struct acpi_madt_local_x2apic *)header;

@@ -227,6 +229,8 @@ acpi_parse_x2apic(struct acpi_subtable_h

acpi_table_print_madt_entry(header);

+ apic_id = processor->local_apic_id;
+ enabled = processor->lapic_flags & ACPI_MADT_ENABLED;
#ifdef CONFIG_X86_X2APIC
/*
* We need to register disabled CPU as well to permit
@@ -235,8 +239,10 @@ acpi_parse_x2apic(struct acpi_subtable_h
* to not preallocating memory for all NR_CPUS
* when we use CPU hotplug.
*/
- acpi_register_lapic(processor->local_apic_id, /* APIC ID */
- processor->lapic_flags & ACPI_MADT_ENABLED);
+ if (x2apic_disabled && (apic_id >= 0xff) && enabled)
+ printk(KERN_WARNING PREFIX "x2apic entry ignored\n");
+ else
+ acpi_register_lapic(apic_id, enabled);
#else
printk(KERN_WARNING PREFIX "x2apic entry ignored\n");
#endif
Index: linux-2.6/arch/x86/kernel/apic/apic.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/apic/apic.c
+++ linux-2.6/arch/x86/kernel/apic/apic.c
@@ -147,15 +147,14 @@ int x2apic_mode;
#ifdef CONFIG_X86_X2APIC
/* x2apic enabled before OS handover */
static int x2apic_preenabled;
+int x2apic_disabled;
static __init int setup_nox2apic(char *str)
{
- if (x2apic_enabled()) {
- pr_warning("Bios already enabled x2apic, "
- "can't enforce nox2apic");
- return 0;
- }
+ if (x2apic_enabled())
+ pr_warning("Bios already enabled x2apic, will disable it");
+
+ x2apic_disabled = 1;

- setup_clear_cpu_cap(X86_FEATURE_X2APIC);
return 0;
}
early_param("nox2apic", setup_nox2apic);
@@ -1412,8 +1411,38 @@ void __init bsp_end_local_APIC_setup(voi
}

#ifdef CONFIG_X86_X2APIC
+
+static void disable_x2apic(void)
+{
+ int msr, msr2;
+
+ if (!cpu_has_x2apic)
+ return;
+
+ rdmsr(MSR_IA32_APICBASE, msr, msr2);
+ if (msr & X2APIC_ENABLE) {
+ u32 x2apic_id = x2apic_cpuid_initial_apicid();
+
+ if (x2apic_id > 255)
+ panic("Can not disable x2apic, id: %08x\n", x2apic_id);
+
+ pr_info("Disabling x2apic\n");
+ /*
+ * Need to disable xapic and x2apic at the same time at first
+ * then enable xapic
+ */
+ wrmsr(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE),
+ 0);
+ wrmsr(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE, 0);
+ }
+}
void check_x2apic(void)
{
+ if (x2apic_disabled) {
+ disable_x2apic();
+ return;
+ }
+
if (x2apic_enabled()) {
pr_info("x2apic enabled by BIOS, switching to x2apic ops\n");
x2apic_preenabled = x2apic_mode = 1;
@@ -1424,6 +1453,11 @@ void enable_x2apic(void)
{
int msr, msr2;

+ if (x2apic_disabled) {
+ disable_x2apic();
+ return;
+ }
+
if (!x2apic_mode)
return;

@@ -1485,6 +1519,9 @@ void __init enable_IR_x2apic(void)
else
ret = enable_IR();

+ if (x2apic_disabled)
+ goto nox2apic;
+
if (!ret) {
/* IR is required if there is APIC ID > 255 even when running
* under KVM
@@ -1519,7 +1556,7 @@ out:

if (x2apic_preenabled)
panic("x2apic: enabled by BIOS but kernel init failed.");
- else if (cpu_has_x2apic)
+ else if (cpu_has_x2apic && !x2apic_disabled)
pr_info("Not enabling x2apic, Intr-remapping init failed.\n");
}

Index: linux-2.6/arch/x86/mm/srat.c
===================================================================
--- linux-2.6.orig/arch/x86/mm/srat.c
+++ linux-2.6/arch/x86/mm/srat.c
@@ -69,6 +69,12 @@ acpi_numa_x2apic_affinity_init(struct ac
if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
return;
pxm = pa->proximity_domain;
+ apic_id = pa->apic_id;
+ if (x2apic_disabled && (apic_id >= 0xff)) {
+ printk(KERN_INFO "SRAT: PXM %u -> X2APIC 0x%04x ignored\n",
+ pxm, apic_id);
+ return;
+ }
node = setup_node(pxm);
if (node < 0) {
printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
@@ -76,7 +82,6 @@ acpi_numa_x2apic_affinity_init(struct ac
return;
}

- apic_id = pa->apic_id;
if (apic_id >= MAX_LOCAL_APIC) {
printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
return;
Index: linux-2.6/arch/x86/include/asm/apicdef.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/apicdef.h
+++ linux-2.6/arch/x86/include/asm/apicdef.h
@@ -142,6 +142,7 @@

#define APIC_BASE (fix_to_virt(FIX_APIC_BASE))
#define APIC_BASE_MSR 0x800
+#define XAPIC_ENABLE (1UL << 11)
#define X2APIC_ENABLE (1UL << 10)

#ifdef CONFIG_X86_32
Index: linux-2.6/arch/x86/include/asm/processor.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/processor.h
+++ linux-2.6/arch/x86/include/asm/processor.h
@@ -167,6 +167,7 @@ extern void init_scattered_cpuid_feature
extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
extern unsigned short num_cache_leaves;

+u32 x2apic_cpuid_initial_apicid(void);
extern void detect_extended_topology(struct cpuinfo_x86 *c);
extern void detect_ht(struct cpuinfo_x86 *c);

Index: linux-2.6/arch/x86/kernel/cpu/topology.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/topology.c
+++ linux-2.6/arch/x86/kernel/cpu/topology.c
@@ -21,6 +21,27 @@
#define BITS_SHIFT_NEXT_LEVEL(eax) ((eax) & 0x1f)
#define LEVEL_MAX_SIBLINGS(ebx) ((ebx) & 0xffff)

+u32 x2apic_cpuid_initial_apicid(void)
+{
+ unsigned int eax, ebx, ecx, edx;
+
+ if (boot_cpu_data.cpuid_level < 0xb)
+ return 0;
+
+ cpuid_count(0xb, SMT_LEVEL, &eax, &ebx, &ecx, &edx);
+
+ /*
+ * check if the cpuid leaf 0xb is actually implemented.
+ */
+ if (ebx == 0 || (LEAFB_SUBTYPE(ecx) != SMT_TYPE))
+ return 0;
+
+ /*
+ * initial apic id, which also represents 32-bit extended x2apic id.
+ */
+ return edx;
+}
+
/*
* Check for extended topology enumeration cpuid leaf 0xb and if it
* exists, use it for populating initial_apicid and cpu topology

Next message: Thomas Gleixner: "Re: [patch 0/3] kvm tool: Serial emulation overhaul"
Previous message: Fabio Estevam: "[PATCH] rtc: Kconfig: Place RTC_DRV_IMXDI and RTC_MXC under "on-CPU RTC drivers""
In reply to: Berck E. Nash: "x2apic boot failure on recent sandy bridge system"
Next in thread: Berck E. Nash: "Re: x2apic boot failure on recent sandy bridge system"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]