Re: IO-APIC on nforce2

[Ross Dickson]

Christian Kröner wrote

> I got a problem using LOCAL APIC and IO-APIC on my uniprocessor nforce2 board. 
> With recent kernels (latest -mm and 2.6.5-linus) the timer irq gets set to 
> XT-PIC, which results in having a constant hi-load of 15% (after booting) to 
> about 25% (after having the system run about 12 h). Earlier versions of -mm 
> set the timer-irq to IO-APIC-level (or edge, i dont remember it well) and i 
> never had any constant hi-load with these versions. Since mainline kernel 
> versions never ever set the timer irq to IO-APIC-{level,edge} i used to patch 
> them with the ross' nforce-patches, so that the timer-irq gets to be 
> IO-APCI-edge, which worked even though the patch applied with offset. Anyways 
> with the latest mm-kernels these patches dont work anymore. I could apply 
> them with offset but it seems the code isn't used or something else is wrong 
> since the timer-irq stays XT-PIC, which results in the problems above. Could 
> anyone point out, how to resolve this problem or tell me what I could do, to 
> get my timer-irq right? I'm sure willing to test patches... 
> Thanks in advance, christian. 
> - 


Hi Christian

I don't know why the high load on xtpic except maybe heaps of spurious irq's 
under the hood.

I am working with 2.4.26-rc2 and have noticed a change with the the recent acpi?
update. The recent fix to stop unnecessary ioapic irq routing entries puts the 
following if statement into io_apic.c, io_apic_set_pci_routing()

	/*
	 * IRQs < 16 are already in the irq_2_pin[] map
	 */
	if (irq >= 16)
		add_pin_to_irq(irq, ioapic, pin);

which prevents my io-apic patch from using that function to reprogram the
io-apic pin on irq0 from pin2 to pin0. 

As a quick fix you could drop the "if (irq >= 16)".
I don't know what harm if any that would do other than create unwanted
irq mapping entries as in the past.

As a better solution to work with the new code I have created a function to
change the pin an irq comes into the io-apic on and also re-initialise the 
io-apic to deal with the change. 

Here is the function for 2.4.26-rc2.

/*
 * reroute irq to different pin clearing old and enabling new
 */
static void __init replace_IO_APIC_pin_at_irq(unsigned int irq,
						int oldapic, int oldpin,
						int newapic, int newpin)
{
	struct IO_APIC_route_entry entry;
	unsigned long flags;
	/*
	 * read oldapic entry
	 */
	spin_lock_irqsave(&ioapic_lock, flags);
	*(((int*)&entry) + 0) = io_apic_read(oldapic, 0x10 + 2 * oldpin);
	*(((int*)&entry) + 1) = io_apic_read(oldapic, 0x11 + 2 * oldpin);
	spin_unlock_irqrestore(&ioapic_lock, flags);
	/*
	 * Check delivery_mode to be sure we're not clearing an SMI pin
	 */
	if (entry.delivery_mode == dest_SMI)
		return;
	/*
	 * clear oldpin on oldapic
	 */
	clear_IO_APIC_pin(oldapic, oldpin);
	/*
	 * reroute irq to newpin on newapic
	 */
	replace_pin_at_irq(irq, oldapic, oldpin, newapic, newpin);
	/*
	 * Enable newpin on newapic
	*/
	spin_lock_irqsave(&ioapic_lock, flags);
	io_apic_write(newapic, 0x10 + 2*newpin, *(((int *)&entry) + 0));
	io_apic_write(newapic, 0x11 + 2*newpin, *(((int *)&entry) + 1));
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

I am now using this instead of the io_apic_set_pci_routing().
My modified check_timer() to work with it is as follows.

/*
 * This code may look a bit paranoid, but it's supposed to cooperate with
 * a wide range of boards and BIOS bugs.  Fortunately only the timer IRQ
 * is so screwy.  Thanks to Brian Perkins for testing/hacking this beast
 * fanatically on his truly buggy board.
 */
static inline void check_timer(void)
{
	extern int timer_ack;
	int pin1, pin2;
	int vector, i;

	/*
	 * get/set the timer IRQ vector:
	 */
	disable_8259A_irq(0);
	vector = assign_irq_vector(0);
	set_intr_gate(vector, interrupt[0]);

	/*
	 * Subtle, code in do_timer_interrupt() expects an AEOI
	 * mode for the 8259A whenever interrupts are routed
	 * through I/O APICs.  Also IRQ0 has to be enabled in
	 * the 8259A which implies the virtual wire has to be
	 * disabled in the local APIC.
	 */
	apic_write_around(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
	init_8259A(1);
	timer_ack = 1;
	enable_8259A_irq(0);

	pin1 = find_isa_irq_pin(0, mp_INT);
	pin2 = find_isa_irq_pin(0, mp_ExtINT);

	printk(KERN_INFO "..TIMER: vector=0x%02X pin1=%d pin2=%d\n", vector, pin1, pin2);

	if (pin1 != -1) {
		for(i=0;i<2;i++) {
			/*
			 * Ok, does IRQ0 through the IOAPIC work?
			 */
			unmask_IO_APIC_irq(0);
			if (timer_irq_works()) {
				if (nmi_watchdog == NMI_IO_APIC) {
					disable_8259A_irq(0);
					setup_nmi();
					enable_8259A_irq(0);
					check_nmi_watchdog();
				}
				printk(KERN_INFO "..TIMER: works OK on IO-APIC irq0\n" );
				return;
			}
			mask_IO_APIC_irq(0);
			if(!i) { /* try INTIN0 if INTIN2 failed */
				printk(KERN_ERR "..MP-BIOS bug: 8254 timer not connected to IO-APIC INTIN%d\n",pin1);
				printk(KERN_INFO "..TIMER: Check if 8254 timer connected to IO-APIC INTIN0? ...\n");
				replace_IO_APIC_pin_at_irq(0, 0, pin1, 0, 0);
				timer_ack=0;
				disable_8259A_irq(0);
			} else { /* restore settings */
				clear_IO_APIC_pin(0, 0);
				printk(KERN_ERR "..TIMER: 8254 timer not connected to IO-APIC INTIN0\n");
				timer_ack=1;
				enable_8259A_irq(0);
			}
		}
	}

	printk(KERN_INFO "...trying to set up timer (IRQ0) through the 8259A ... ");
	if (pin2 != -1) {
		printk("\n..... (found pin %d) ...", pin2);
		/*
		 * legacy devices should be connected to IO APIC #0
		 */
		setup_ExtINT_IRQ0_pin(pin2, vector);
		if (timer_irq_works()) {
			printk("works.\n");
			if (pin1 != -1)
				replace_pin_at_irq(0, 0, pin1, 0, pin2);
			else
				add_pin_to_irq(0, 0, pin2);
			if (nmi_watchdog == NMI_IO_APIC) {
				setup_nmi();
				check_nmi_watchdog();
			}
			return;
		}
		/*
		 * Cleanup, just in case ...
		 */
		clear_IO_APIC_pin(0, pin2);
	}
	printk(" failed.\n");

	if (nmi_watchdog) {
		printk(KERN_WARNING "timer doesn't work through the IO-APIC - disabling NMI Watchdog!\n");
		nmi_watchdog = 0;
	}

	printk(KERN_INFO "...trying to set up timer as Virtual Wire IRQ...");

	disable_8259A_irq(0);
	irq_desc[0].handler = &lapic_irq_type;
	apic_write_around(APIC_LVT0, APIC_DM_FIXED | vector);	/* Fixed mode */
	enable_8259A_irq(0);

	if (timer_irq_works()) {
		printk(" works.\n");
		return;
	}
	apic_write_around(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | vector);
	printk(" failed.\n");

	printk(KERN_INFO "...trying to set up timer as ExtINT IRQ...");

	init_8259A(0);
	make_8259A_irq(0);
	apic_write_around(APIC_LVT0, APIC_DM_EXTINT);

	unlock_ExtINT_logic();

	if (timer_irq_works()) {
		printk(" works.\n");
		return;
	}
	printk(" failed :(.\n");
	panic("IO-APIC + timer doesn't work! pester mingo@xxxxxxxxxx");
}

This version loops twice on the "pin1" attempt, firstly trying the bios assigned
pin, then trying pin0 with no timer acks and the 8259 xtpic disabled.

I have not as yet downloaded 2.6.5xxx
From memory this 2.4.26-rc2 code should be very similar to the (2.6.5-linus)
but a bit different to the -mm series. For the -mm series I think you can drop
the "timer_ack=" lines from my changes as it still has Maciej Rozycki's 8259 
ack patch? The timer ack should already have been correctly set to off by it's
checking if the apic is an integrated one.

Here are the changes as a diff on the io_apic.c in 2.4.26-rc2

--- io_apic.c.orig	2004-04-08 15:56:53.000000000 +1000
+++ io_apic.c	2004-04-10 02:33:02.000000000 +1000
@@ -197,10 +197,48 @@ static void clear_IO_APIC (void)
 		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
 			clear_IO_APIC_pin(apic, pin);
 }

 /*
+ * reroute irq to different pin clearing old and enabling new
+ */
+static void __init replace_IO_APIC_pin_at_irq(unsigned int irq,
+						int oldapic, int oldpin,
+						int newapic, int newpin)
+{
+	struct IO_APIC_route_entry entry;
+	unsigned long flags;
+	/*
+	 * read oldapic entry
+	 */
+	spin_lock_irqsave(&ioapic_lock, flags);
+	*(((int*)&entry) + 0) = io_apic_read(oldapic, 0x10 + 2 * oldpin);
+	*(((int*)&entry) + 1) = io_apic_read(oldapic, 0x11 + 2 * oldpin);
+	spin_unlock_irqrestore(&ioapic_lock, flags);
+	/*
+	 * Check delivery_mode to be sure we're not clearing an SMI pin
+	 */
+	if (entry.delivery_mode == dest_SMI)
+		return;
+	/*
+	 * clear oldpin on oldapic
+	 */
+	clear_IO_APIC_pin(oldapic, oldpin);
+	/*
+	 * reroute irq to newpin on newapic
+	 */
+	replace_pin_at_irq(irq, oldapic, oldpin, newapic, newpin);
+	/*
+	 * Enable newpin on newapic
+	*/
+	spin_lock_irqsave(&ioapic_lock, flags);
+	io_apic_write(newapic, 0x10 + 2*newpin, *(((int *)&entry) + 0));
+	io_apic_write(newapic, 0x11 + 2*newpin, *(((int *)&entry) + 1));
+	spin_unlock_irqrestore(&ioapic_lock, flags);
+}
+
+/*
  * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
  * specific CPU-side IRQs.
  */

 #define MAX_PIRQS 8
@@ -1582,11 +1620,11 @@ static inline void unlock_ExtINT_logic(v
  */
 static inline void check_timer(void)
 {
 	extern int timer_ack;
 	int pin1, pin2;
-	int vector;
+	int vector, i;

 	/*
 	 * get/set the timer IRQ vector:
 	 */
 	disable_8259A_irq(0);
@@ -1609,25 +1647,39 @@ static inline void check_timer(void)
 	pin2 = find_isa_irq_pin(0, mp_ExtINT);

 	printk(KERN_INFO "..TIMER: vector=0x%02X pin1=%d pin2=%d\n", vector, pin1, pin2);

 	if (pin1 != -1) {
-		/*
-		 * Ok, does IRQ0 through the IOAPIC work?
-		 */
-		unmask_IO_APIC_irq(0);
-		if (timer_irq_works()) {
-			if (nmi_watchdog == NMI_IO_APIC) {
+		for(i=0;i<2;i++) {
+			/*
+			 * Ok, does IRQ0 through the IOAPIC work?
+			 */
+			unmask_IO_APIC_irq(0);
+			if (timer_irq_works()) {
+				if (nmi_watchdog == NMI_IO_APIC) {
+					disable_8259A_irq(0);
+					setup_nmi();
+					enable_8259A_irq(0);
+					check_nmi_watchdog();
+				}
+				printk(KERN_INFO "..TIMER: works OK on IO-APIC irq0\n" );
+				return;
+			}
+			mask_IO_APIC_irq(0);
+			if(!i) { /* try INTIN0 if INTIN2 failed */
+				printk(KERN_ERR "..MP-BIOS bug: 8254 timer not connected to IO-APIC INTIN%d\n",pin1);
+				printk(KERN_INFO "..TIMER: Check if 8254 timer connected to IO-APIC INTIN0? ...\n");
+				replace_IO_APIC_pin_at_irq(0, 0, pin1, 0, 0);
+				timer_ack=0;
 				disable_8259A_irq(0);
-				setup_nmi();
+			} else { /* restore settings */
+				clear_IO_APIC_pin(0, 0);
+				printk(KERN_ERR "..TIMER: 8254 timer not connected to IO-APIC INTIN0\n");
+				timer_ack=1;
 				enable_8259A_irq(0);
-				check_nmi_watchdog();
 			}
-			return;
 		}
-		clear_IO_APIC_pin(0, pin1);
-		printk(KERN_ERR "..MP-BIOS bug: 8254 timer not connected to IO-APIC\n");
 	}

 	printk(KERN_INFO "...trying to set up timer (IRQ0) through the 8259A ... ");
 	if (pin2 != -1) {
 		printk("\n..... (found pin %d) ...", pin2);

Also attached as tarball if whitespace problems,
Hope this helps, Please cc me on responses,
Ross Dickson

Attachment: nforce2-ioapic-rd-2.4.26-rc2.patch.tgz
Description: application/tgz