[PATCH 2/2] I-pipe: x86 port

From: Philippe Gerum
Date: Fri Jun 17 2005 - 18:34:14 EST


diffstat:
arch/i386/Kconfig | 3
arch/i386/kernel/Makefile | 1
arch/i386/kernel/apic.c | 8
arch/i386/kernel/entry.S | 87 +++-
arch/i386/kernel/i386_ksyms.c | 19
arch/i386/kernel/i8259.c | 31 +
arch/i386/kernel/io_apic.c | 90 ++++
arch/i386/kernel/ipipe.c | 646 +++++++++++++++++++++++++++++++
arch/i386/kernel/process.c | 1
arch/i386/kernel/smp.c | 29 +
arch/i386/kernel/time.c | 5
arch/i386/kernel/traps.c | 8
arch/i386/mm/fault.c | 2
arch/i386/mm/ioremap.c | 2
include/asm-i386/apic.h | 6
include/asm-i386/io_apic.h | 4
include/asm-i386/ipipe.h | 308 ++++++++++++++
include/asm-i386/mach-default/do_timer.h | 5
include/asm-i386/mach-visws/do_timer.h | 5
include/asm-i386/pgalloc.h | 23 +
include/asm-i386/spinlock.h | 4
include/asm-i386/system.h | 31 +
ipipe/x86.c | 577 +++++++++++++++++++++++++++
23 files changed, 1868 insertions(+), 27 deletions(-)


Signed-off-by: Philippe Gerum <rpm@xxxxxxxxxxx>


--- linux-2.6.12-rc6/arch/i386/Kconfig 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/Kconfig 2005-06-15 17:58:21.000000000 +0200
@@ -534,6 +534,8 @@ config PREEMPT_BKL
Say Y here if you are building a kernel for a desktop system.
Say N if you are unsure.

+source "ipipe/Kconfig"
+
config X86_UP_APIC
bool "Local APIC support on uniprocessors"
depends on !SMP && !(X86_VISWS || X86_VOYAGER)
@@ -941,7 +943,6 @@ config SECCOMP

endmenu

-
menu "Power management options (ACPI, APM)"
depends on !X86_VOYAGER

--- linux-2.6.12-rc6/arch/i386/kernel/Makefile 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/Makefile 2005-06-15 17:35:32.000000000 +0200
@@ -11,6 +11,7 @@ obj-y := process.o semaphore.o signal.o

obj-y += cpu/
obj-y += timers/
+obj-$(CONFIG_IPIPE_CORE) += ipipe.o
obj-$(CONFIG_ACPI_BOOT) += acpi/
obj-$(CONFIG_X86_BIOS_REBOOT) += reboot.o
obj-$(CONFIG_MCA) += mca.o
--- linux-2.6.12-rc6/arch/i386/kernel/apic.c 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/apic.c 2005-06-15 17:35:32.000000000 +0200
@@ -62,7 +62,7 @@ void ack_bad_irq(unsigned int irq)
* unexpected vectors occur) that might lock up the APIC
* completely.
*/
- ack_APIC_irq();
+ __ack_APIC_irq();
}

void __init apic_intr_init(void)
@@ -1161,6 +1161,10 @@ inline void smp_local_timer_interrupt(st
fastcall void smp_apic_timer_interrupt(struct pt_regs *regs)
{
int cpu = smp_processor_id();
+#ifdef CONFIG_IPIPE_CORE
+ if (ipipe_running)
+ regs = __ipipe_tick_regs + cpu;
+#endif /* CONFIG_IPIPE_CORE */

/*
* the NMI deadlock-detector uses this.
@@ -1197,7 +1201,7 @@ fastcall void smp_spurious_interrupt(str
*/
v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1));
if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
- ack_APIC_irq();
+ __ack_APIC_irq();

/* see sw-dev-man vol 3, chapter 7.4.13.5 */
printk(KERN_INFO "spurious APIC interrupt on CPU#%d, should never happen.\n",
--- linux-2.6.12-rc6/arch/i386/kernel/entry.S 2005-06-14 13:57:27.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/entry.S 2005-06-15 20:20:06.000000000 +0200
@@ -75,11 +75,27 @@ DF_MASK = 0x00000400
NT_MASK = 0x00004000
VM_MASK = 0x00020000

+#ifdef CONFIG_IPIPE_CORE
+#define CLI call __ipipe_stall_root
+#define STI call __ipipe_unstall_root
+#define FIXUP_ROOT_IF call __ipipe_if_fixup_root ; movl ORIG_EAX(%esp),%eax
+#define EMULATE_ROOT_IRET call __ipipe_unstall_iret_root ; movl EAX(%esp),%eax
+#define TEST_PREEMPTIBLE(regs) call __ipipe_kpreempt_root ; testl %eax,%eax
+#define restore_nocheck .ipipe_unstall_and_restore_nocheck
+#else /* !CONFIG_IPIPE_CORE */
+#define CLI cli
+#define STI sti
+#define FIXUP_ROOT_IF
+#define EMULATE_ROOT_IRET
+#define TEST_PREEMPTIBLE(regs) testl $IF_MASK,EFLAGS(regs)
+#define restore_nmi restore_all
+#endif /* CONFIG_IPIPE_CORE */
+
#ifdef CONFIG_PREEMPT
-#define preempt_stop cli
+#define preempt_stop CLI
#else
#define preempt_stop
-#define resume_kernel restore_nocheck
+#define resume_kernel restore_nocheck
#endif

#define SAVE_ALL \
@@ -141,14 +157,14 @@ ENTRY(ret_from_fork)
ALIGN
ret_from_exception:
preempt_stop
-ret_from_intr:
+ENTRY(ret_from_intr)
GET_THREAD_INFO(%ebp)
movl EFLAGS(%esp), %eax # mix EFLAGS and CS
movb CS(%esp), %al
testl $(VM_MASK | 3), %eax
jz resume_kernel
ENTRY(resume_userspace)
- cli # make sure we don't miss an interrupt
+ CLI # make sure we don't miss an interrupt
# setting need_resched or sigpending
# between sampling and the iret
movl TI_flags(%ebp), %ecx
@@ -159,14 +175,14 @@ ENTRY(resume_userspace)

#ifdef CONFIG_PREEMPT
ENTRY(resume_kernel)
- cli
+ CLI
cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
jnz restore_nocheck
need_resched:
movl TI_flags(%ebp), %ecx # need_resched set ?
testb $_TIF_NEED_RESCHED, %cl
jz restore_all
- testl $IF_MASK,EFLAGS(%esp) # interrupts off (exception path) ?
+ TEST_PREEMPTIBLE(%esp) # interrupts off (exception path) ?
jz restore_all
call preempt_schedule_irq
jmp need_resched
@@ -200,6 +216,7 @@ sysenter_past_esp:

pushl %eax
SAVE_ALL
+ FIXUP_ROOT_IF
GET_THREAD_INFO(%ebp)

/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
@@ -209,11 +226,12 @@ sysenter_past_esp:
jae syscall_badsys
call *sys_call_table(,%eax,4)
movl %eax,EAX(%esp)
- cli
+ CLI
movl TI_flags(%ebp), %ecx
testw $_TIF_ALLWORK_MASK, %cx
jne syscall_exit_work
/* if something modifies registers it must also disable sysexit */
+ EMULATE_ROOT_IRET
movl EIP(%esp), %edx
movl OLDESP(%esp), %ecx
xorl %ebp,%ebp
@@ -225,6 +243,7 @@ sysenter_past_esp:
ENTRY(system_call)
pushl %eax # save orig_eax
SAVE_ALL
+ FIXUP_ROOT_IF
GET_THREAD_INFO(%ebp)
# system call tracing in operation
/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
@@ -236,7 +255,7 @@ syscall_call:
call *sys_call_table(,%eax,4)
movl %eax,EAX(%esp) # store the return value
syscall_exit:
- cli # make sure we don't miss an interrupt
+ CLI # make sure we don't miss an interrupt
# setting need_resched or sigpending
# between sampling and the iret
movl TI_flags(%ebp), %ecx
@@ -253,7 +272,15 @@ restore_all:
andl $(VM_MASK | (4 << 8) | 3), %eax
cmpl $((4 << 8) | 3), %eax
je ldt_ss # returning to user-space with LDT SS
+#ifdef CONFIG_IPIPE_CORE
+.ipipe_unstall_and_restore_nocheck:
+ call __ipipe_unstall_iret_root
+restore_nmi:
+ # FIXME: we need to check for a return to
+ # user-space on a 16bit stack even in the NMI case
+#else /* !CONFIG_IPIPE_CORE */
restore_nocheck:
+#endif /* CONFIG_IPIPE_CORE */
RESTORE_REGS
addl $4, %esp
1: iret
@@ -301,7 +328,7 @@ work_pending:
jz work_notifysig
work_resched:
call schedule
- cli # make sure we don't miss an interrupt
+ CLI # make sure we don't miss an interrupt
# setting need_resched or sigpending
# between sampling and the iret
movl TI_flags(%ebp), %ecx
@@ -348,7 +375,7 @@ syscall_trace_entry:
syscall_exit_work:
testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
jz work_pending
- sti # could let do_syscall_trace() call
+ STI # could let do_syscall_trace() call
# schedule() instead
movl %esp, %eax
movl $1, %edx
@@ -399,7 +426,11 @@ ENTRY(interrupt)

vector=0
ENTRY(irq_entries_start)
+#ifdef CONFIG_IPIPE_CORE
+.rept 224
+#else
.rept NR_IRQS
+#endif
ALIGN
1: pushl $vector-256
jmp common_interrupt
@@ -424,6 +455,36 @@ ENTRY(name) \
call smp_/**/name; \
jmp ret_from_intr;

+#ifdef CONFIG_IPIPE_CORE
+
+.data
+ENTRY(__ipipe_irq_trampolines)
+.text
+
+vector=0
+ENTRY(__ipipe_irq_entries)
+.rept 224
+ ALIGN
+1: pushl $vector-256
+ jmp __ipipe_irq_common
+.data
+ .long 1b
+.text
+vector=vector+1
+.endr
+
+ ALIGN
+__ipipe_irq_common:
+ SAVE_ALL
+ call __ipipe_handle_irq
+ testl %eax,%eax
+ jnz ret_from_intr
+ RESTORE_REGS
+ addl $4, %esp
+ iret
+
+#endif /* CONFIG_IPIPE_CORE */
+
/* The include is where all of the SMP etc. interrupts come from */
#include "entry_arch.h"

@@ -455,6 +516,7 @@ error_code:
movl %ecx, %es
movl %esp,%eax # pt_regs pointer
call *%edi
+ FIXUP_ROOT_IF
jmp ret_from_exception

ENTRY(coprocessor_error)
@@ -470,6 +532,7 @@ ENTRY(simd_coprocessor_error)
ENTRY(device_not_available)
pushl $-1 # mark this as an int
SAVE_ALL
+ FIXUP_ROOT_IF
movl %cr0, %eax
testl $0x4, %eax # EM (math emulation bit)
jne device_not_available_emulate
@@ -511,6 +574,7 @@ ENTRY(debug)
debug_stack_correct:
pushl $-1 # mark this as an int
SAVE_ALL
+ FIXUP_ROOT_IF
xorl %edx,%edx # error code 0
movl %esp,%eax # pt_regs pointer
call do_debug
@@ -549,7 +613,7 @@ nmi_stack_correct:
xorl %edx,%edx # zero error code
movl %esp,%eax # pt_regs pointer
call do_nmi
- jmp restore_all
+ jmp restore_nmi

nmi_stack_fixup:
FIX_STACK(12,nmi_stack_correct, 1)
@@ -591,6 +655,7 @@ nmi_16bit_stack:
ENTRY(int3)
pushl $-1 # mark this as an int
SAVE_ALL
+ FIXUP_ROOT_IF
xorl %edx,%edx # zero error code
movl %esp,%eax # pt_regs pointer
call do_int3
--- linux-2.6.12-rc6/arch/i386/kernel/i386_ksyms.c 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/i386_ksyms.c 2005-06-17 17:36:33.000000000 +0200
@@ -36,6 +36,25 @@
extern void dump_thread(struct pt_regs *, struct user *);
extern spinlock_t rtc_lock;

+#ifdef CONFIG_IPIPE_CORE
+#ifdef CONFIG_IPIPE_MODULE
+extern int __ipipe_irq_trampolines;
+EXPORT_SYMBOL(__ipipe_irq_trampolines);
+#ifdef CONFIG_X86_LOCAL_APIC
+extern int using_apic_timer;
+EXPORT_SYMBOL(using_apic_timer);
+#endif /* CONFIG_X86_LOCAL_APIC */
+extern struct desc_struct idt_table[];
+EXPORT_SYMBOL(idt_table);
+#ifdef CONFIG_X86_IO_APIC
+EXPORT_SYMBOL(io_apic_irqs);
+EXPORT_SYMBOL(irq_vector);
+#endif /* CONFIG_X86_IO_APIC */
+#endif /* CONFIG_IPIPE_MODULE */
+extern irq_desc_t irq_desc[];
+EXPORT_SYMBOL(irq_desc);
+#endif /* CONFIG_IPIPE_CORE */
+
/* This is definitely a GPL-only symbol */
EXPORT_SYMBOL_GPL(cpu_gdt_table);

--- linux-2.6.12-rc6/arch/i386/kernel/i8259.c 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/i8259.c 2005-06-15 17:35:32.000000000 +0200
@@ -170,6 +170,36 @@ static inline int i8259A_irq_real(unsign
*/
static void mask_and_ack_8259A(unsigned int irq)
{
+#ifdef CONFIG_IPIPE_CORE
+ unsigned int irqmask = 1 << irq;
+
+ spin_lock(&i8259A_lock);
+
+ if (cached_irq_mask & irqmask)
+ goto spurious_8259A_irq;
+
+ if (irq == 0) {
+ /* Fast timer ack -- don't mask
+ (unless supposedly spurious) */
+ outb(0x20,PIC_MASTER_CMD);
+ spin_unlock(&i8259A_lock);
+ return;
+ }
+
+ cached_irq_mask |= irqmask;
+
+handle_real_irq:
+ if (irq & 8) {
+ outb(cached_slave_mask, PIC_SLAVE_IMR);
+ outb(0x60+(irq&7),PIC_SLAVE_CMD);/* 'Specific EOI' to slave */
+ outb(0x60+PIC_CASCADE_IR,PIC_MASTER_CMD); /* 'Specific EOI' to master-IRQ2 */
+ } else {
+ outb(cached_master_mask, PIC_MASTER_IMR);
+ outb(0x60+irq,PIC_MASTER_CMD); /* 'Specific EOI to master */
+ }
+ spin_unlock(&i8259A_lock);
+ return;
+#else /* !CONFIG_IPIPE_CORE */
unsigned int irqmask = 1 << irq;
unsigned long flags;

@@ -206,6 +236,7 @@ handle_real_irq:
}
spin_unlock_irqrestore(&i8259A_lock, flags);
return;
+#endif /* CONFIG_IPIPE_CORE */

spurious_8259A_irq:
/*
--- linux-2.6.12-rc6/arch/i386/kernel/io_apic.c 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/io_apic.c 2005-06-16 10:52:46.000000000 +0200
@@ -1828,6 +1828,26 @@ static unsigned int startup_edge_ioapic_
* interrupt for real. This prevents IRQ storms from unhandled
* devices.
*/
+
+#ifdef CONFIG_IPIPE_CORE
+
+static void ack_edge_ioapic_irq (unsigned irq)
+
+{
+ move_irq(irq);
+ if ((irq_desc[irq].status & (IRQ_PENDING | IRQ_DISABLED)) == (IRQ_PENDING | IRQ_DISABLED))
+ {
+ unsigned long flags;
+ spin_lock_irqsave_hw(&ioapic_lock,flags);
+ __mask_IO_APIC_irq(irq);
+ spin_unlock_irqrestore_hw(&ioapic_lock,flags);
+ }
+
+ __ack_APIC_irq();
+}
+
+#else /* !CONFIG_IPIPE_CORE */
+
static void ack_edge_ioapic_irq(unsigned int irq)
{
move_irq(irq);
@@ -1837,6 +1857,8 @@ static void ack_edge_ioapic_irq(unsigned
ack_APIC_irq();
}

+#endif /* CONFIG_IPIPE_CORE */
+
/*
* Level triggered interrupts can just be masked,
* and shutting down and starting up the interrupt
@@ -1858,6 +1880,70 @@ static unsigned int startup_level_ioapic
return 0; /* don't check for pending */
}

+#ifdef CONFIG_IPIPE_CORE
+
+/* Prevent low priority IRQs grabbed by high priority domains from
+ being delayed, waiting for a high priority interrupt handler
+ running in a low priority domain to complete. */
+
+static unsigned long bugous_edge_triggers;
+
+static void end_level_ioapic_irq (unsigned irq)
+
+{
+ move_irq(irq);
+
+ spin_lock(&ioapic_lock);
+
+ if (test_and_clear_bit(irq,&bugous_edge_triggers))
+ {
+ atomic_inc(&irq_mis_count);
+ __unmask_and_level_IO_APIC_irq(irq);
+ }
+ else
+ __unmask_IO_APIC_irq(irq);
+
+ spin_unlock(&ioapic_lock);
+}
+
+static void mask_and_ack_level_ioapic_irq (unsigned irq)
+
+{
+ unsigned long v;
+ int i;
+
+ i = IO_APIC_VECTOR(irq);
+ v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
+
+ spin_lock(&ioapic_lock);
+
+ if (!(v & (1 << (i & 0x1f))))
+ {
+ set_bit(irq,&bugous_edge_triggers);
+ __mask_and_edge_IO_APIC_irq(irq);
+ }
+ else
+ __mask_IO_APIC_irq(irq);
+
+ spin_unlock(&ioapic_lock);
+
+ __ack_APIC_irq();
+}
+
+#ifdef CONFIG_PCI_MSI
+
+static inline void mask_and_ack_level_ioapic_vector(unsigned int vector)
+
+{
+ int irq = vector_to_irq(vector);
+
+ mask_and_ack_level_ioapic_irq(irq);
+}
+
+#endif /* CONFIG_PCI_MSI */
+
+#else /* !CONFIG_IPIPE_CORE */
+
static void end_level_ioapic_irq (unsigned int irq)
{
unsigned long v;
@@ -1898,6 +1984,8 @@ static void end_level_ioapic_irq (unsign
}
}

+#endif /* CONFIG_IPIPE_CORE */
+
#ifdef CONFIG_PCI_MSI
static unsigned int startup_edge_ioapic_vector(unsigned int vector)
{
@@ -2035,7 +2123,7 @@ static void disable_lapic_irq (unsigned

static void ack_lapic_irq (unsigned int irq)
{
- ack_APIC_irq();
+ __ack_APIC_irq();
}

static void end_lapic_irq (unsigned int i) { /* nothing */ }
--- linux-2.6.12-rc6/arch/i386/kernel/ipipe.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/ipipe.c 2005-06-16 12:38:10.000000000 +0200
@@ -0,0 +1,646 @@
+/* -*- linux-c -*-
+ * linux/arch/i386/kernel/ipipe.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-PIPE core support for x86.
+ */
+
+#include <linux/config.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/slab.h>
+#include <asm/system.h>
+#include <asm/atomic.h>
+#include <asm/hw_irq.h>
+#include <asm/irq.h>
+#include <asm/desc.h>
+#include <asm/io.h>
+#ifdef CONFIG_X86_LOCAL_APIC
+#include <asm/fixmap.h>
+#include <asm/bitops.h>
+#include <asm/mpspec.h>
+#ifdef CONFIG_X86_IO_APIC
+#include <asm/io_apic.h>
+#endif /* CONFIG_X86_IO_APIC */
+#include <asm/apic.h>
+#include <mach_ipi.h>
+#endif /* CONFIG_X86_LOCAL_APIC */
+
+struct pt_regs __ipipe_tick_regs[IPIPE_NR_CPUS];
+
+int __ipipe_tick_irq;
+
+#ifdef CONFIG_SMP
+
+static cpumask_t __ipipe_cpu_sync_map;
+
+static cpumask_t __ipipe_cpu_lock_map;
+
+static raw_spinlock_t __ipipe_cpu_barrier = RAW_SPIN_LOCK_UNLOCKED;
+
+static atomic_t __ipipe_critical_count = ATOMIC_INIT(0);
+
+static void (*__ipipe_cpu_sync) (void);
+
+#endif /* CONFIG_SMP */
+
+#define __ipipe_call_asm_irq_handler(ipd,irq) \
+ __asm__ __volatile__ ("pushfl\n\t" \
+ "push %%cs\n\t" \
+ "call *%1\n" \
+ : /* no output */ \
+ : "a" (irq), "m" ((ipd)->irqs[irq].handler))
+
+#define __ipipe_call_c_root_irq_handler(ipd,irq) \
+ __asm__ __volatile__ ("pushfl\n\t" \
+ "pushl %%cs\n\t" \
+ "pushl $1f\n\t" \
+ "pushl $-1\n\t" \
+ "pushl %%es\n\t" \
+ "pushl %%ds\n\t" \
+ "pushl %%eax\n\t" \
+ "pushl %%ebp\n\t" \
+ "pushl %%edi\n\t" \
+ "pushl %%esi\n\t" \
+ "pushl %%edx\n\t" \
+ "pushl %%ecx\n\t" \
+ "pushl %%ebx\n\t" \
+ "pushl %%eax\n\t" \
+ "call *%1\n\t" \
+ "addl $4,%%esp\n\t" \
+ "jmp ret_from_intr\n\t" \
+ "1:\n" \
+ : /* no output */ \
+ : "a" (irq), "m" ((ipd)->irqs[irq].handler))
+
+/* Do _not_ forcibly re-enable hw IRQs in the following trampoline
+ used for non-root domains; unlike Linux handlers, non-root domain
+ handlers are fully in control of the hw masking state. */
+
+#define __ipipe_call_c_irq_handler(ipd,irq) \
+ __asm__ __volatile__ ("pushl %%ebp\n\t" \
+ "pushl %%edi\n\t" \
+ "pushl %%esi\n\t" \
+ "pushl %%edx\n\t" \
+ "pushl %%ecx\n\t" \
+ "pushl %%ebx\n\t" \
+ "pushl %%eax\n\t" \
+ "call *%1\n\t" \
+ "addl $4,%%esp\n\t" \
+ "popl %%ebx\n\t" \
+ "popl %%ecx\n\t" \
+ "popl %%edx\n\t" \
+ "popl %%esi\n\t" \
+ "popl %%edi\n\t" \
+ "popl %%ebp\n" \
+ : /* no output */ \
+ : "a" (irq), "m" ((ipd)->irqs[irq].handler))
+
+static __inline__ unsigned long flnz(unsigned long word)
+{
+ __asm__("bsrl %1, %0":"=r"(word)
+ : "r"(word));
+ return word;
+}
+
+int __ipipe_ack_system_irq(unsigned irq)
+{
+#ifdef CONFIG_X86_LOCAL_APIC
+ __ack_APIC_irq();
+#endif /* CONFIG_X86_LOCAL_APIC */
+ return 1;
+}
+
+#ifdef CONFIG_SMP
+
+/* Always called with hw interrupts off. */
+
+static void __ipipe_do_critical_sync(unsigned irq)
+{
+ ipipe_declare_cpuid;
+
+ ipipe_load_cpuid();
+
+ cpu_set(cpuid, __ipipe_cpu_sync_map);
+
+ /* Now we are in sync with the lock requestor running on another
+ CPU. Enter a spinning wait until he releases the global
+ lock. */
+ spin_lock_hw(&__ipipe_cpu_barrier);
+
+ /* Got it. Now get out. */
+
+ if (__ipipe_cpu_sync)
+ /* Call the sync routine if any. */
+ __ipipe_cpu_sync();
+
+ spin_unlock_hw(&__ipipe_cpu_barrier);
+
+ cpu_clear(cpuid, __ipipe_cpu_sync_map);
+}
+
+#endif /* CONFIG_SMP */
+
+/* ipipe_critical_enter() -- Grab the superlock excluding all CPUs
+ but the current one from a critical section. This lock is used when
+ we must enforce a global critical section for a single CPU in a
+ possibly SMP system whichever context the CPUs are running. */
+
+unsigned long ipipe_critical_enter(void (*syncfn) (void))
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+
+#ifdef CONFIG_SMP
+ if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a UP box... */
+ ipipe_declare_cpuid;
+ cpumask_t lock_map;
+
+ ipipe_load_cpuid();
+
+ if (!cpu_test_and_set(cpuid, __ipipe_cpu_lock_map)) {
+ while (cpu_test_and_set
+ (BITS_PER_LONG - 1, __ipipe_cpu_lock_map)) {
+ int n = 0;
+ do {
+ cpu_relax();
+ } while (++n < cpuid);
+ }
+
+ spin_lock_hw(&__ipipe_cpu_barrier);
+
+ __ipipe_cpu_sync = syncfn;
+
+ /* Send the sync IPI to all processors but the current one. */
+ send_IPI_allbutself(IPIPE_CRITICAL_VECTOR);
+
+ cpus_andnot(lock_map, cpu_online_map,
+ __ipipe_cpu_lock_map);
+
+ while (!cpus_equal(__ipipe_cpu_sync_map, lock_map))
+ cpu_relax();
+ }
+
+ atomic_inc(&__ipipe_critical_count);
+ }
+#endif /* CONFIG_SMP */
+
+ return flags;
+}
+
+/* ipipe_critical_exit() -- Release the superlock. */
+
+void ipipe_critical_exit(unsigned long flags)
+{
+#ifdef CONFIG_SMP
+ if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a UP box... */
+ ipipe_declare_cpuid;
+
+ ipipe_load_cpuid();
+
+ if (atomic_dec_and_test(&__ipipe_critical_count)) {
+ spin_unlock_hw(&__ipipe_cpu_barrier);
+
+ while (!cpus_empty(__ipipe_cpu_sync_map))
+ cpu_relax();
+
+ cpu_clear(cpuid, __ipipe_cpu_lock_map);
+ cpu_clear(BITS_PER_LONG - 1, __ipipe_cpu_lock_map);
+ }
+ }
+#endif /* CONFIG_SMP */
+
+ local_irq_restore_hw(flags);
+}
+
+void __ipipe_init_stage(struct ipipe_domain *ipd)
+{
+ int cpuid, n;
+
+ for (cpuid = 0; cpuid < IPIPE_NR_CPUS; cpuid++) {
+ ipd->cpudata[cpuid].irq_pending_hi = 0;
+
+ for (n = 0; n < IPIPE_IRQ_IWORDS; n++)
+ ipd->cpudata[cpuid].irq_pending_lo[n] = 0;
+
+ for (n = 0; n < IPIPE_NR_IRQS; n++)
+ ipd->cpudata[cpuid].irq_hits[n] = 0;
+ }
+
+ for (n = 0; n < IPIPE_NR_IRQS; n++) {
+ ipd->irqs[n].acknowledge = NULL;
+ ipd->irqs[n].handler = NULL;
+ ipd->irqs[n].control = IPIPE_PASS_MASK; /* Pass but don't handle */
+ }
+
+#ifdef CONFIG_SMP
+ ipd->irqs[IPIPE_CRITICAL_IPI].acknowledge = &__ipipe_ack_system_irq;
+ ipd->irqs[IPIPE_CRITICAL_IPI].handler = &__ipipe_do_critical_sync;
+ /* Immediately handle in the current domain but *never* pass */
+ ipd->irqs[IPIPE_CRITICAL_IPI].control =
+ IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK | IPIPE_SYSTEM_MASK;
+#endif /* CONFIG_SMP */
+}
+
+/* __ipipe_sync_stage() -- Flush the pending IRQs for the current
+ domain (and processor). This routine flushes the interrupt log
+ (see "Optimistic interrupt protection" from D. Stodolsky et al. for
+ more on the deferred interrupt scheme). Every interrupt that
+ occurred while the pipeline was stalled gets played. WARNING:
+ callers on SMP boxen should always check for CPU migration on
+ return of this routine. One can control the kind of interrupts
+ which are going to be sync'ed using the syncmask
+ parameter. IPIPE_IRQMASK_ANY plays them all, IPIPE_IRQMASK_VIRT
+ plays virtual interrupts only. This routine must be called with hw
+ interrupts off. */
+
+void fastcall __ipipe_sync_stage(unsigned long syncmask)
+{
+ unsigned long mask, submask;
+ struct ipcpudata *cpudata;
+ struct ipipe_domain *ipd;
+ int level, rank, sync;
+ ipipe_declare_cpuid;
+ unsigned irq;
+
+ ipipe_load_cpuid();
+ ipd = ipipe_percpu_domain[cpuid];
+ cpudata = &ipd->cpudata[cpuid];
+
+ sync = __test_and_set_bit(IPIPE_SYNC_FLAG, &cpudata->status);
+
+ /* The policy here is to keep the dispatching code interrupt-free
+ by stalling the current stage. If the upper domain handler
+ (which we call) wants to re-enable interrupts while in a safe
+ portion of the code (e.g. SA_INTERRUPT flag unset for Linux's
+ sigaction()), it will have to unstall (then stall again before
+ returning to us!) the stage when it sees fit. */
+
+ while ((mask = (cpudata->irq_pending_hi & syncmask)) != 0) {
+ /* Give a slight priority advantage to high-numbered IRQs
+ like the virtual ones. */
+ level = flnz(mask);
+ __clear_bit(level, &cpudata->irq_pending_hi);
+
+ while ((submask = cpudata->irq_pending_lo[level]) != 0) {
+ rank = flnz(submask);
+ irq = (level << IPIPE_IRQ_ISHIFT) + rank;
+
+ if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) {
+ __clear_bit(rank,
+ &cpudata->irq_pending_lo[level]);
+ continue;
+ }
+
+ if (--cpudata->irq_hits[irq] == 0)
+ __clear_bit(rank,
+ &cpudata->irq_pending_lo[level]);
+
+ __set_bit(IPIPE_STALL_FLAG, &cpudata->status);
+
+ if (ipd == ipipe_root_domain) {
+ /* Make sure to re-enable hw interrupts to reduce
+ preemption latency by higher priority domains when
+ calling the Linux handlers in the next two
+ trampolines. Additionally, this ensures that the
+ forged interrupt frame will allow the final check
+ for a rescheduling opportunity in ret_from_intr. */
+
+ local_irq_enable_hw();
+
+ if (!test_bit
+ (IPIPE_CALLASM_FLAG,
+ &ipd->irqs[irq].control)) {
+ irq_enter();
+ __ipipe_call_c_root_irq_handler(ipd,irq);
+ irq_exit();
+ } else
+ __ipipe_call_asm_irq_handler(ipd, irq);
+ } else
+ __ipipe_call_c_irq_handler(ipd, irq);
+
+ local_irq_disable_hw();
+
+#ifdef CONFIG_SMP
+ {
+ int _cpuid = ipipe_processor_id();
+
+ if (_cpuid != cpuid) { /* Handle CPU migration. */
+ /* We expect any domain to clear the SYNC bit each
+ time it switches in a new task, so that preemptions
+ and/or CPU migrations (in the SMP case) over the
+ ISR do not lock out the log syncer for some
+ indefinite amount of time. In the Linux case,
+ schedule() handles this (see kernel/sched.c). For
+ this reason, we don't bother clearing it here for
+ the source CPU in the migration handling case,
+ since it must have scheduled another task in by
+ now. */
+ cpuid = _cpuid;
+ cpudata = &ipd->cpudata[cpuid];
+ __set_bit(IPIPE_SYNC_FLAG,
+ &cpudata->status);
+ }
+ }
+#endif /* CONFIG_SMP */
+
+ __clear_bit(IPIPE_STALL_FLAG, &cpudata->status);
+ }
+ }
+
+ if (!sync)
+ __clear_bit(IPIPE_SYNC_FLAG, &cpudata->status);
+}
+
+/* __ipipe_walk_pipeline(): Plays interrupts pending in the log. Must
+ be called with local hw interrupts disabled. */
+
+static inline void __ipipe_walk_pipeline(struct list_head *pos, int cpuid)
+{
+ struct ipipe_domain *this_domain = ipipe_percpu_domain[cpuid];
+
+ while (pos != &__ipipe_pipeline) {
+ struct ipipe_domain *next_domain =
+ list_entry(pos, struct ipipe_domain, p_link);
+
+ if (test_bit
+ (IPIPE_STALL_FLAG, &next_domain->cpudata[cpuid].status))
+ break; /* Stalled stage -- do not go further. */
+
+ if (next_domain->cpudata[cpuid].irq_pending_hi != 0) {
+
+ if (next_domain == this_domain)
+ __ipipe_sync_stage(IPIPE_IRQMASK_ANY);
+ else {
+ __ipipe_switch_to(this_domain, next_domain,
+ cpuid);
+
+ ipipe_load_cpuid(); /* Processor might have changed. */
+
+ if (this_domain->cpudata[cpuid].
+ irq_pending_hi != 0
+ && !test_bit(IPIPE_STALL_FLAG,
+ &this_domain->cpudata[cpuid].
+ status)
+ && !test_bit(IPIPE_SYNC_FLAG,
+ &this_domain->cpudata[cpuid].
+ status))
+ __ipipe_sync_stage(IPIPE_IRQMASK_ANY);
+ }
+
+ break;
+ } else if (next_domain == this_domain)
+ break;
+
+ pos = next_domain->p_link.next;
+ }
+}
+
+/* __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
+ interrupt protection log is maintained here for each domain. Hw
+ interrupts are off on entry. */
+
+int __ipipe_handle_irq(struct pt_regs regs)
+{
+ int m_ack = regs.orig_eax >= 0, s_ack;
+ unsigned irq = regs.orig_eax & 0xff;
+ struct list_head *head, *pos;
+ struct ipipe_domain *this_domain;
+ ipipe_declare_cpuid;
+
+ ipipe_load_cpuid();
+
+ this_domain = ipipe_percpu_domain[cpuid];
+
+ s_ack = m_ack;
+
+ if (test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control))
+ head = &this_domain->p_link;
+ else
+ head = __ipipe_pipeline.next;
+
+ /* Ack the interrupt. */
+
+ pos = head;
+
+ while (pos != &__ipipe_pipeline) {
+ struct ipipe_domain *next_domain =
+ list_entry(pos, struct ipipe_domain, p_link);
+
+ /* For each domain handling the incoming IRQ, mark it as
+ pending in its log. */
+
+ if (test_bit
+ (IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) {
+ /* Domains that handle this IRQ are polled for
+ acknowledging it by decreasing priority order. The
+ interrupt must be made pending _first_ in the domain's
+ status flags before the PIC is unlocked. */
+
+ next_domain->cpudata[cpuid].irq_hits[irq]++;
+ __ipipe_set_irq_bit(next_domain, cpuid, irq);
+
+ /* Always get the first master acknowledge available. Once
+ we've got it, allow slave acknowledge handlers to run
+ (until one of them stops us). */
+
+ if (!m_ack)
+ m_ack = next_domain->irqs[irq].acknowledge(irq);
+ else if (test_bit
+ (IPIPE_SHARED_FLAG,
+ &next_domain->irqs[irq].control) && !s_ack)
+ s_ack = next_domain->irqs[irq].acknowledge(irq);
+ }
+
+ /* If the domain does not want the IRQ to be passed down the
+ interrupt pipe, exit the loop now. */
+
+ if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
+ break;
+
+ pos = next_domain->p_link.next;
+ }
+
+ if (irq == __ipipe_tick_irq) {
+ __ipipe_tick_regs[cpuid].eflags = regs.eflags;
+ __ipipe_tick_regs[cpuid].eip = regs.eip;
+ __ipipe_tick_regs[cpuid].xcs = regs.xcs;
+#if defined(CONFIG_SMP) && defined(CONFIG_FRAME_POINTER)
+ /* Linux profiling code needs this. */
+ __ipipe_tick_regs[cpuid].ebp = regs.ebp;
+#endif /* CONFIG_SMP && CONFIG_FRAME_POINTER */
+ }
+
+ /* Now walk the pipeline, yielding control to the highest
+ priority domain that has pending interrupt(s) or
+ immediately to the current domain if the interrupt has been
+ marked as 'sticky'. This search does not go beyond the
+ current domain in the pipeline. */
+
+ __ipipe_walk_pipeline(head, cpuid);
+
+ ipipe_load_cpuid();
+
+ if (ipipe_percpu_domain[cpuid] != ipipe_root_domain ||
+ test_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status))
+ return 0;
+
+#ifdef CONFIG_SMP
+ /* Prevent a spurious rescheduling from being triggered on
+ preemptible kernels along the way out through ret_from_intr. */
+ __set_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status);
+#endif /* CONFIG_SMP */
+
+ return 1;
+}
+
+/* ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
+ just like if it has been actually received from a hw source. Also
+ works for virtual interrupts. */
+
+int fastcall ipipe_trigger_irq(unsigned irq)
+{
+ struct pt_regs regs;
+ unsigned long flags;
+
+ if (irq >= IPIPE_NR_IRQS ||
+ (ipipe_virtual_irq_p(irq) &&
+ !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
+ return -EINVAL;
+
+ local_irq_save_hw(flags);
+
+ regs.orig_eax = irq; /* Won't be acked */
+ regs.xcs = __KERNEL_CS;
+ regs.eflags = flags;
+
+ __ipipe_handle_irq(regs);
+
+ local_irq_restore_hw(flags);
+
+ return 1;
+}
+
+static inline void __fixup_if(struct pt_regs *regs)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ ipipe_get_cpu(flags);
+
+ if (ipipe_percpu_domain[cpuid] == ipipe_root_domain) {
+ /* Have the saved hw state look like the domain stall bit, so
+ that __ipipe_unstall_iret_root() restores the proper
+ pipeline state for the root stage upon exit. */
+
+ if (test_bit
+ (IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status))
+ regs->eflags &= ~X86_EFLAGS_IF;
+ else
+ regs->eflags |= X86_EFLAGS_IF;
+ }
+
+ ipipe_put_cpu(flags);
+}
+
+asmlinkage void __ipipe_if_fixup_root(struct pt_regs regs)
+{
+ if (ipipe_running)
+ __fixup_if(&regs);
+}
+
+/* Check the interrupt flag to make sure the existing preemption
+ opportunity upon in-kernel resumption could be exploited. If
+ pipelining is active, the stall bit of the root domain is checked,
+ otherwise, the EFLAGS register from the stacked interrupt frame is
+ tested. In case a rescheduling could take place in pipelined mode,
+ the root stage is stalled before the hw interrupts are
+ re-enabled. This routine must be called with hw interrupts off. */
+
+asmlinkage int __ipipe_kpreempt_root(struct pt_regs regs)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ if (!ipipe_running)
+ return !!(regs.eflags & X86_EFLAGS_IF);
+
+ ipipe_get_cpu(flags);
+
+ if (test_bit
+ (IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status)) {
+ ipipe_put_cpu(flags);
+ return 0; /* Root stage is stalled: rescheduling denied. */
+ }
+
+ __ipipe_stall_root();
+ local_irq_enable_hw();
+
+ return 1; /* Ok, may reschedule now. */
+}
+
+asmlinkage void __ipipe_unstall_iret_root(struct pt_regs regs)
+{
+ ipipe_declare_cpuid;
+
+ if (!ipipe_running)
+ return;
+
+ /* Emulate IRET's handling of the interrupt flag. */
+
+ local_irq_disable_hw();
+
+ ipipe_load_cpuid();
+
+ /* Restore the software state as it used to be on kernel
+ entry. CAUTION: NMIs must *not* return through this
+ emulation. */
+
+ if (!(regs.eflags & X86_EFLAGS_IF)) {
+ __set_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status);
+ regs.eflags |= X86_EFLAGS_IF;
+ } else {
+ __clear_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status);
+
+ /* Only sync virtual IRQs here, so that we don't recurse
+ indefinitely in case of an external interrupt flood. */
+
+ if ((ipipe_root_domain->cpudata[cpuid].
+ irq_pending_hi & IPIPE_IRQMASK_VIRT) != 0)
+ __ipipe_sync_stage(IPIPE_IRQMASK_VIRT);
+ }
+}
+
+EXPORT_SYMBOL(__ipipe_init_stage);
+EXPORT_SYMBOL(__ipipe_sync_stage);
+EXPORT_SYMBOL(__ipipe_ack_system_irq);
+EXPORT_SYMBOL(__ipipe_handle_irq);
+EXPORT_SYMBOL(__ipipe_tick_irq);
+EXPORT_SYMBOL(ipipe_critical_enter);
+EXPORT_SYMBOL(ipipe_critical_exit);
+EXPORT_SYMBOL(ipipe_trigger_irq);
--- linux-2.6.12-rc6/arch/i386/kernel/process.c 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/process.c 2005-06-16 17:36:34.000000000 +0200
@@ -161,6 +161,7 @@ void cpu_idle (void)
idle = default_idle;

__get_cpu_var(irq_stat).idle_timestamp = jiffies;
+ ipipe_suspend_domain();
idle();
}
schedule();
--- linux-2.6.12-rc6/arch/i386/kernel/smp.c 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/smp.c 2005-06-15 17:35:32.000000000 +0200
@@ -131,6 +131,9 @@ void __send_IPI_shortcut(unsigned int sh
* to the APIC.
*/
unsigned int cfg;
+ unsigned long flags;
+
+ local_irq_save_hw_cond(flags);

/*
* Wait for idle.
@@ -146,6 +149,8 @@ void __send_IPI_shortcut(unsigned int sh
* Send the IPI. The write to APIC_ICR fires this off.
*/
apic_write_around(APIC_ICR, cfg);
+
+ local_irq_restore_hw_cond(flags);
}

void fastcall send_IPI_self(int vector)
@@ -162,7 +167,7 @@ void send_IPI_mask_bitmask(cpumask_t cpu
unsigned long cfg;
unsigned long flags;

- local_irq_save(flags);
+ local_irq_save_hw(flags);

/*
* Wait for idle.
@@ -185,7 +190,7 @@ void send_IPI_mask_bitmask(cpumask_t cpu
*/
apic_write_around(APIC_ICR, cfg);

- local_irq_restore(flags);
+ local_irq_restore_hw(flags);
}

void send_IPI_mask_sequence(cpumask_t mask, int vector)
@@ -199,7 +204,7 @@ void send_IPI_mask_sequence(cpumask_t ma
* should be modified to do 1 message per cluster ID - mbligh
*/

- local_irq_save(flags);
+ local_irq_save_hw(flags);

for (query_cpu = 0; query_cpu < NR_CPUS; ++query_cpu) {
if (cpu_isset(query_cpu, mask)) {
@@ -226,7 +231,7 @@ void send_IPI_mask_sequence(cpumask_t ma
apic_write_around(APIC_ICR, cfg);
}
}
- local_irq_restore(flags);
+ local_irq_restore_hw(flags);
}

#include <mach_ipi.h> /* must come after the send_IPI functions above for inlining */
@@ -310,7 +315,9 @@ static inline void leave_mm (unsigned lo

fastcall void smp_invalidate_interrupt(struct pt_regs *regs)
{
- unsigned long cpu;
+ unsigned long cpu, flags;
+
+ local_irq_save_hw_cond(flags);

cpu = get_cpu();

@@ -340,6 +347,7 @@ fastcall void smp_invalidate_interrupt(s
smp_mb__after_clear_bit();
out:
put_cpu_no_resched();
+ local_irq_restore_hw_cond(flags);
}

static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
@@ -400,14 +408,19 @@ void flush_tlb_current_task(void)
{
struct mm_struct *mm = current->mm;
cpumask_t cpu_mask;
+ unsigned long flags;

preempt_disable();
+ local_irq_save_hw_cond(flags);
+
cpu_mask = mm->cpu_vm_mask;
cpu_clear(smp_processor_id(), cpu_mask);

local_flush_tlb();
if (!cpus_empty(cpu_mask))
flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
+
+ local_irq_restore_hw_cond(flags);
preempt_enable();
}

@@ -435,8 +448,11 @@ void flush_tlb_page(struct vm_area_struc
{
struct mm_struct *mm = vma->vm_mm;
cpumask_t cpu_mask;
+ unsigned long flags;

preempt_disable();
+ local_irq_save_hw_cond(flags);
+
cpu_mask = mm->cpu_vm_mask;
cpu_clear(smp_processor_id(), cpu_mask);

@@ -447,6 +463,8 @@ void flush_tlb_page(struct vm_area_struc
leave_mm(smp_processor_id());
}

+ local_irq_restore_hw_cond(flags);
+
if (!cpus_empty(cpu_mask))
flush_tlb_others(cpu_mask, mm, va);

@@ -609,4 +627,3 @@ fastcall void smp_call_function_interrup
atomic_inc(&call_data->finished);
}
}
-
--- linux-2.6.12-rc6/arch/i386/kernel/time.c 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/time.c 2005-06-15 17:35:32.000000000 +0200
@@ -258,11 +258,12 @@ static inline void do_timer_interrupt(in
* This will also deassert NMI lines for the watchdog if run
* on an 82489DX-based system.
*/
- spin_lock(&i8259A_lock);
+ unsigned long flags;
+ spin_lock_irqsave_hw_cond(&i8259A_lock,flags);
outb(0x0c, PIC_MASTER_OCW3);
/* Ack the IRQ; AEOI will end it automatically. */
inb(PIC_MASTER_POLL);
- spin_unlock(&i8259A_lock);
+ spin_unlock_irqrestore_hw_cond(&i8259A_lock,flags);
}
#endif

--- linux-2.6.12-rc6/arch/i386/kernel/traps.c 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/kernel/traps.c 2005-06-16 17:36:30.000000000 +0200
@@ -226,6 +226,11 @@ void show_registers(struct pt_regs *regs
regs->esi, regs->edi, regs->ebp, esp);
printk("ds: %04x es: %04x ss: %04x\n",
regs->xds & 0xffff, regs->xes & 0xffff, ss);
+#ifdef CONFIG_IPIPE_CORE
+ if (ipipe_current_domain != ipipe_root_domain)
+ printk("I-pipe domain %s",ipipe_current_domain->name);
+ else
+#endif /* CONFIG_IPIPE_CORE */
printk("Process %s (pid: %d, threadinfo=%p task=%p)",
current->comm, current->pid, current_thread_info(), current);
/*
@@ -946,12 +951,15 @@ asmlinkage void math_state_restore(struc
{
struct thread_info *thread = current_thread_info();
struct task_struct *tsk = thread->task;
+ unsigned long flags;

+ local_irq_save_hw_cond(flags);
clts(); /* Allow maths ops (or we recurse) */
if (!tsk_used_math(tsk))
init_fpu(tsk);
restore_fpu(tsk);
thread->status |= TS_USEDFPU; /* So we fnsave on switch_to() */
+ local_irq_restore_hw_cond(flags);
}

#ifndef CONFIG_MATH_EMULATION
--- linux-2.6.12-rc6/arch/i386/mm/fault.c 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/mm/fault.c 2005-06-15 17:35:32.000000000 +0200
@@ -224,6 +224,8 @@ fastcall void do_page_fault(struct pt_re
/* get the address */
__asm__("movl %%cr2,%0":"=r" (address));

+ local_irq_enable_hw_cond();
+
if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
SIGSEGV) == NOTIFY_STOP)
return;
--- linux-2.6.12-rc6/arch/i386/mm/ioremap.c 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/arch/i386/mm/ioremap.c 2005-06-15 18:28:03.000000000 +0200
@@ -16,6 +16,7 @@
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#include <asm/pgtable.h>
+#include <asm/pgalloc.h>

#define ISA_START_ADDRESS 0xa0000
#define ISA_END_ADDRESS 0x100000
@@ -92,6 +93,7 @@ static int ioremap_page_range(unsigned l
err = ioremap_pud_range(pgd, addr, next, phys_addr+addr, flags);
if (err)
break;
+ set_pgdir(addr, *pgd);
} while (pgd++, addr = next, addr != end);
spin_unlock(&init_mm.page_table_lock);
flush_tlb_all();
--- linux-2.6.12-rc6/include/asm-i386/apic.h 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/include/asm-i386/apic.h 2005-06-15 17:35:32.000000000 +0200
@@ -69,7 +69,13 @@ int get_physical_broadcast(void);
# define apic_write_around(x,y) apic_write_atomic((x),(y))
#endif

+#ifdef CONFIG_IPIPE_CORE
+#define ack_APIC_irq() do { if (!ipipe_running) __ack_APIC_irq(); } while(0)
+static inline void __ack_APIC_irq(void)
+#else /* !CONFIG_IPIPE_CORE */
+#define __ack_APIC_irq() ack_APIC_irq()
static inline void ack_APIC_irq(void)
+#endif /* CONFIG_IPIPE_CORE */
{
/*
* ack_APIC_irq() actually gets compiled as a single instruction:
--- linux-2.6.12-rc6/include/asm-i386/io_apic.h 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/include/asm-i386/io_apic.h 2005-06-15 17:35:32.000000000 +0200
@@ -16,7 +16,9 @@
#ifdef CONFIG_PCI_MSI
static inline int use_pci_vector(void) {return 1;}
static inline void disable_edge_ioapic_vector(unsigned int vector) { }
+#ifndef CONFIG_IPIPE_CORE
static inline void mask_and_ack_level_ioapic_vector(unsigned int vector) { }
+#endif /* !CONFIG_IPIPE_CORE */
static inline void end_edge_ioapic_vector (unsigned int vector) { }
#define startup_level_ioapic startup_level_ioapic_vector
#define shutdown_level_ioapic mask_IO_APIC_vector
@@ -35,7 +37,9 @@ static inline void end_edge_ioapic_vecto
#else
static inline int use_pci_vector(void) {return 0;}
static inline void disable_edge_ioapic_irq(unsigned int irq) { }
+#ifndef CONFIG_IPIPE_CORE
static inline void mask_and_ack_level_ioapic_irq(unsigned int irq) { }
+#endif /* !CONFIG_IPIPE_CORE */
static inline void end_edge_ioapic_irq (unsigned int irq) { }
#define startup_level_ioapic startup_level_ioapic_irq
#define shutdown_level_ioapic mask_IO_APIC_irq
--- linux-2.6.12-rc6/include/asm-i386/ipipe.h 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.12-rc6-ipipe-0.5/include/asm-i386/ipipe.h 2005-06-17 20:26:33.000000000 +0200
@@ -0,0 +1,308 @@
+/* -*- linux-c -*-
+ * include/asm-i386/ipipe.h
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __I386_IPIPE_H
+#define __I386_IPIPE_H
+
+#include <irq_vectors.h>
+#include <asm/ptrace.h>
+#include <linux/cpumask.h>
+#include <linux/list.h>
+#include <linux/threads.h>
+
+#define IPIPE_ARCH_STRING "0.5/x86"
+#define IPIPE_MAJOR_NUMBER 0
+#define IPIPE_MINOR_NUMBER 5
+
+#ifdef CONFIG_SMP
+
+#include <asm/fixmap.h>
+#include <asm/mpspec.h>
+#include <mach_apicdef.h>
+#include <linux/thread_info.h>
+
+#define IPIPE_NR_CPUS NR_CPUS
+#define IPIPE_CRITICAL_VECTOR 0xf9 /* Used by ipipe_critical_enter/exit() */
+#define IPIPE_CRITICAL_IPI (IPIPE_CRITICAL_VECTOR - FIRST_EXTERNAL_VECTOR)
+
+#define ipipe_processor_id() (current_thread_info()->cpu)
+
+#define ipipe_declare_cpuid int cpuid
+#define ipipe_load_cpuid() do { \
+ (cpuid) = ipipe_processor_id(); \
+ } while(0)
+#define ipipe_lock_cpu(flags) do { \
+ local_irq_save_hw(flags); \
+ (cpuid) = ipipe_processor_id(); \
+ } while(0)
+#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags)
+#define ipipe_get_cpu(flags) ipipe_lock_cpu(flags)
+#define ipipe_put_cpu(flags) ipipe_unlock_cpu(flags)
+#define ipipe_current_domain (ipipe_percpu_domain[ipipe_processor_id()])
+
+#else /* !CONFIG_SMP */
+
+#define IPIPE_NR_CPUS 1
+#define ipipe_processor_id() 0
+#define ipipe_declare_cpuid const int cpuid = 0
+#define ipipe_load_cpuid() /* nop */
+#define ipipe_lock_cpu(flags) local_irq_save_hw(flags)
+#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags)
+#define ipipe_get_cpu(flags) do { flags = flags; } while(0)
+#define ipipe_put_cpu(flags) /* nop */
+#define ipipe_current_domain (ipipe_percpu_domain[0])
+
+#endif /* CONFIG_SMP */
+
+#define prepare_arch_switch(rq, next) do { } while (0)
+#define finish_arch_switch(rq, prev) do { \
+ __clear_bit(IPIPE_SYNC_FLAG,&ipipe_root_domain->cpudata[task_cpu(prev)].status); \
+ spin_unlock_irq(&(rq)->lock); \
+} while(0)
+#define task_running(rq, p) ((rq)->curr == (p))
+
+struct ipipe_sysinfo {
+
+ int ncpus; /* Number of CPUs on board */
+ u64 cpufreq; /* CPU frequency (in Hz) */
+
+ /* Arch-dependent block */
+
+ struct {
+ unsigned tmirq; /* Timer tick IRQ */
+ u64 tmfreq; /* Timer frequency */
+ } archdep;
+};
+
+#ifdef CONFIG_X86_LOCAL_APIC
+/* We must cover the whole IRQ space to map the local timer interrupt
+ (#207). */
+#ifdef CONFIG_PCI_MSI
+#define IPIPE_NR_XIRQS NR_IRQS
+#else /* CONFIG_PCI_MSI */
+#define IPIPE_NR_XIRQS 224
+#endif /* CONFIG_PCI_MSI */
+#else /* !CONFIG_X86_LOCAL_APIC */
+#define IPIPE_NR_XIRQS NR_IRQS
+#endif /* CONFIG_X86_LOCAL_APIC */
+
+/* Number of virtual IRQs */
+#define IPIPE_NR_VIRQS BITS_PER_LONG
+/* First virtual IRQ # */
+#define IPIPE_VIRQ_BASE (((IPIPE_NR_XIRQS + BITS_PER_LONG - 1) / BITS_PER_LONG) * BITS_PER_LONG)
+/* Total number of IRQ slots */
+#define IPIPE_NR_IRQS (IPIPE_VIRQ_BASE + IPIPE_NR_VIRQS)
+/* Number of indirect words needed to map the whole IRQ space. */
+#define IPIPE_IRQ_IWORDS ((IPIPE_NR_IRQS + BITS_PER_LONG - 1) / BITS_PER_LONG)
+#define IPIPE_IRQ_IMASK (BITS_PER_LONG - 1)
+#define IPIPE_IRQ_ISHIFT 5 /* 2^5 for 32bits arch. */
+
+#define IPIPE_IRQMASK_ANY (~0L)
+#define IPIPE_IRQMASK_VIRT (IPIPE_IRQMASK_ANY << (IPIPE_VIRQ_BASE / BITS_PER_LONG))
+
+struct ipipe_domain {
+
+ struct list_head p_link; /* Link in pipeline */
+
+ struct ipcpudata {
+ volatile unsigned long status;
+ volatile unsigned long irq_pending_hi;
+ volatile unsigned long irq_pending_lo[IPIPE_IRQ_IWORDS];
+ volatile unsigned irq_hits[IPIPE_NR_IRQS];
+ } cpudata[IPIPE_NR_CPUS];
+
+ struct {
+ int (*acknowledge) (unsigned irq);
+ void (*handler) (unsigned irq);
+ unsigned long control;
+ } irqs[IPIPE_NR_IRQS];
+
+ unsigned long flags;
+ unsigned domid;
+ const char *name;
+ int priority;
+ void *pdd;
+};
+
+/* The following macros must be used hw interrupts off. */
+
+#define __ipipe_set_irq_bit(ipd,cpuid,irq) \
+do { \
+ if (!test_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) { \
+ __set_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \
+ __set_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[cpuid].irq_pending_hi); \
+ } \
+} while(0)
+
+#define __ipipe_clear_pend(ipd,cpuid,irq) \
+do { \
+ __clear_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \
+ if ((ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT] == 0) \
+ __clear_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[cpuid].irq_pending_hi); \
+} while(0)
+
+#define __ipipe_lock_irq(ipd,cpuid,irq) \
+do { \
+ if (!test_and_set_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) \
+ __ipipe_clear_pend(ipd,cpuid,irq); \
+} while(0)
+
+#define __ipipe_unlock_irq(ipd,irq) \
+do { \
+ int __cpuid, __nr_cpus = num_online_cpus(); \
+ if (test_and_clear_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) \
+ for (__cpuid = 0; __cpuid < __nr_cpus; __cpuid++) \
+ if ((ipd)->cpudata[__cpuid].irq_hits[irq] > 0) { /* We need atomic ops next. */ \
+ set_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[__cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \
+ set_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[__cpuid].irq_pending_hi); \
+ } \
+} while(0)
+
+#define __ipipe_clear_irq(ipd,irq) \
+do { \
+ int __cpuid, __nr_cpus = num_online_cpus(); \
+ clear_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control); \
+ for (__cpuid = 0; __cpuid < __nr_cpus; __cpuid++) { \
+ (ipd)->cpudata[__cpuid].irq_hits[irq] = 0; \
+ __ipipe_clear_pend(ipd,__cpuid,irq); \
+ } \
+} while(0)
+
+#define ipipe_virtual_irq_p(irq) ((irq) >= IPIPE_VIRQ_BASE && \
+ (irq) < IPIPE_NR_IRQS)
+
+#define ipipe_hw_save_flags_and_sti(x) __asm__ __volatile__("pushfl ; popl %0 ; sti":"=g" (x): /* no input */ :"memory")
+#define local_irq_disable_hw() __asm__ __volatile__("cli": : :"memory")
+#define local_irq_enable_hw() __asm__ __volatile__("sti": : :"memory")
+#define local_irq_save_hw(x) __asm__ __volatile__("pushfl ; popl %0 ; cli":"=g" (x): /* no input */ :"memory")
+#define local_irq_restore_hw(x) __asm__ __volatile__("pushl %0 ; popfl": /* no output */ :"g" (x):"memory", "cc")
+#define local_save_flags_hw(x) __asm__ __volatile__("pushfl ; popl %0":"=g" (x): /* no input */)
+#define local_test_iflag_hw(x) ((x) & (1<<9))
+#define irqs_disabled_hw() \
+({ \
+ unsigned long flags; \
+ local_save_flags_hw(flags); \
+ !local_test_iflag_hw(flags); \
+})
+
+#define ipipe_read_tsc(t) __asm__ __volatile__("rdtsc" : "=A" (t))
+#define ipipe_cpu_freq() ({ unsigned long long __freq = cpu_has_tsc?(1000LL * cpu_khz):CLOCK_TICK_RATE; __freq; })
+
+#ifdef CONFIG_IPIPE_PREEMPT_RT
+/* We are over a combo I-pipe+PREEMPT_RT _patched_ kernel, but
+ CONFIG_PREEMPT_RT is not necessarily enabled; use the raw spinlock
+ support for the I-pipe. */
+#define spin_lock_hw(x) __raw_spin_lock(x)
+#define spin_unlock_hw(x) __raw_spin_unlock(x)
+#define spin_trylock_hw(x) __raw_spin_trylock(x)
+#define write_lock_hw(x) __raw_write_lock(x)
+#define write_unlock_hw(x) __raw_write_unlock(x)
+#define write_trylock_hw(x) __raw_write_trylock(x)
+#define read_lock_hw(x) __raw_read_lock(x)
+#define read_unlock_hw(x) __raw_read_unlock(x)
+#else /* !CONFIG_IPIPE_PREEMPT_RT */
+#define spin_lock_hw(x) _spin_lock(x)
+#define spin_unlock_hw(x) _spin_unlock(x)
+#define spin_trylock_hw(x) _spin_trylock(x)
+#define write_lock_hw(x) _write_lock(x)
+#define write_unlock_hw(x) _write_unlock(x)
+#define write_trylock_hw(x) _write_trylock(x)
+#define read_lock_hw(x) _read_lock(x)
+#define read_unlock_hw(x) _read_unlock(x)
+#define raw_spinlock_t spinlock_t
+#define RAW_SPIN_LOCK_UNLOCKED SPIN_LOCK_UNLOCKED
+#define raw_rwlock_t rwlock_t
+#define RAW_RW_LOCK_UNLOCKED RW_LOCK_UNLOCKED
+#endif /* CONFIG_IPIPE_PREEMPT_RT */
+
+#define spin_lock_irqsave_hw(x,flags) \
+do { \
+ local_irq_save_hw(flags); \
+ spin_lock_hw(x); \
+} while (0)
+
+#define spin_unlock_irqrestore_hw(x,flags) \
+do { \
+ spin_unlock_hw(x); \
+ local_irq_restore_hw(flags); \
+} while (0)
+
+#define spin_lock_irq_hw(x) \
+do { \
+ local_irq_disable_hw(); \
+ spin_lock_hw(x); \
+} while (0)
+
+#define spin_unlock_irq_hw(x) \
+do { \
+ spin_unlock_hw(x); \
+ local_irq_enable_hw(); \
+} while (0)
+
+#define read_lock_irqsave_hw(lock, flags) \
+do { \
+ local_irq_save_hw(flags); \
+ read_lock_hw(lock); \
+} while (0)
+
+#define read_unlock_irqrestore_hw(lock, flags) \
+do { \
+ read_unlock_hw(lock); \
+ local_irq_restore_hw(flags); \
+} while (0)
+
+#define write_lock_irqsave_hw(lock, flags) \
+do { \
+ local_irq_save_hw(flags); \
+ write_lock_hw(lock); \
+} while (0)
+
+#define write_unlock_irqrestore_hw(lock, flags) \
+do { \
+ write_unlock_hw(lock); \
+ local_irq_restore_hw(flags); \
+} while (0)
+
+/* Private interface -- Internal use only */
+
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd);
+
+#define __ipipe_check_platform() do { } while(0)
+
+#define __ipipe_init_platform() do { } while(0)
+
+void __ipipe_enable_pipeline(void);
+
+void __ipipe_disable_pipeline(void);
+
+void __ipipe_init_stage(struct ipipe_domain *ipd);
+
+void fastcall __ipipe_sync_stage(unsigned long syncmask);
+
+int __ipipe_ack_system_irq(unsigned irq);
+
+int __ipipe_handle_irq(struct pt_regs regs);
+
+extern struct pt_regs __ipipe_tick_regs[];
+
+extern int __ipipe_tick_irq;
+
+#endif /* !__I386_IPIPE_H */
--- linux-2.6.12-rc6/include/asm-i386/mach-default/do_timer.h 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/include/asm-i386/mach-default/do_timer.h 2005-06-15 17:35:32.000000000 +0200
@@ -49,14 +49,15 @@ static inline void do_timer_interrupt_ho
static inline int do_timer_overflow(int count)
{
int i;
+ unsigned long flags;

- spin_lock(&i8259A_lock);
+ spin_lock_irqsave_hw_cond(&i8259A_lock, flags);
/*
* This is tricky when I/O APICs are used;
* see do_timer_interrupt().
*/
i = inb(0x20);
- spin_unlock(&i8259A_lock);
+ spin_unlock_irqrestore_hw_cond(&i8259A_lock, flags);

/* assumption about timer being IRQ0 */
if (i & 0x01) {
--- linux-2.6.12-rc6/include/asm-i386/mach-visws/do_timer.h 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/include/asm-i386/mach-visws/do_timer.h 2005-06-15 17:35:32.000000000 +0200
@@ -28,14 +28,15 @@ static inline void do_timer_interrupt_ho
static inline int do_timer_overflow(int count)
{
int i;
+ unsigned long flags;

- spin_lock(&i8259A_lock);
+ spin_lock_irqsave_hw_cond(&i8259A_lock, flags);
/*
* This is tricky when I/O APICs are used;
* see do_timer_interrupt().
*/
i = inb(0x20);
- spin_unlock(&i8259A_lock);
+ spin_unlock_irqrestore_hw_cond(&i8259A_lock, flags);

/* assumption about timer being IRQ0 */
if (i & 0x01) {
--- linux-2.6.12-rc6/include/asm-i386/pgalloc.h 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/include/asm-i386/pgalloc.h 2005-06-15 18:32:05.000000000 +0200
@@ -47,4 +47,27 @@ static inline void pte_free(struct page

#define check_pgt_cache() do { } while (0)

+static inline void set_pgdir(unsigned long address, pgd_t entry)
+{
+#ifdef CONFIG_IPIPE_CORE
+ struct task_struct * p;
+ struct page *page;
+ pgd_t *pgd;
+
+ read_lock(&tasklist_lock);
+
+ for_each_process(p) {
+ if(p->mm)
+ *pgd_offset(p->mm,address) = entry;
+ }
+
+ read_unlock(&tasklist_lock);
+
+ for (page = pgd_list; page; page = (struct page *)page->index) {
+ pgd = (pgd_t *)page_address(page);
+ pgd[address >> PGDIR_SHIFT] = entry;
+ }
+#endif /* CONFIG_IPIPE_CORE */
+}
+
#endif /* _I386_PGALLOC_H */
--- linux-2.6.12-rc6/include/asm-i386/spinlock.h 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/include/asm-i386/spinlock.h 2005-06-15 17:35:32.000000000 +0200
@@ -57,6 +57,9 @@ typedef struct {
"jmp 1b\n" \
"3:\n\t"

+#ifdef CONFIG_IPIPE_CORE
+#define spin_lock_string_flags spin_lock_string
+#else /* !CONFIG_IPIPE_CORE */
#define spin_lock_string_flags \
"\n1:\t" \
"lock ; decb %0\n\t" \
@@ -72,6 +75,7 @@ typedef struct {
"cli\n\t" \
"jmp 1b\n" \
"4:\n\t"
+#endif /* CONFIG_IPIPE_CORE */

/*
* This works. Despite all the confusion.
--- linux-2.6.12-rc6/include/asm-i386/system.h 2005-06-06 17:22:29.000000000 +0200
+++ linux-2.6.12-rc6-ipipe-0.5/include/asm-i386/system.h 2005-06-16 17:37:34.000000000 +0200
@@ -441,6 +441,35 @@ struct alt_instr {
#define set_wmb(var, value) do { var = value; wmb(); } while (0)

/* interrupt control.. */
+#ifdef CONFIG_IPIPE_CORE
+
+#include <linux/linkage.h>
+
+void __ipipe_stall_root(void);
+
+void __ipipe_unstall_root(void);
+
+unsigned long __ipipe_test_root(void);
+
+unsigned long __ipipe_test_and_stall_root(void);
+
+void fastcall __ipipe_restore_root(unsigned long flags);
+
+#define local_save_flags(x) ((x) = __ipipe_test_root())
+#define local_irq_save(x) ((x) = __ipipe_test_and_stall_root())
+#define local_irq_restore(x) __ipipe_restore_root(x)
+#define local_irq_disable() __ipipe_stall_root()
+#define local_irq_enable() __ipipe_unstall_root()
+
+#define irqs_disabled() __ipipe_test_root()
+
+#define safe_halt() do { \
+ __ipipe_unstall_root(); \
+ __asm__ __volatile__("sti; hlt": : :"memory"); \
+} while(0)
+
+#else /* !CONFIG_IPIPE_CORE */
+
#define local_save_flags(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushfl ; popl %0":"=g" (x): /* no input */); } while (0)
#define local_irq_restore(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushl %0 ; popfl": /* no output */ :"g" (x):"memory", "cc"); } while (0)
#define local_irq_disable() __asm__ __volatile__("cli": : :"memory")
@@ -458,6 +487,8 @@ struct alt_instr {
/* For spinlocks etc */
#define local_irq_save(x) __asm__ __volatile__("pushfl ; popl %0 ; cli":"=g" (x): /* no input */ :"memory")

+#endif /* CONFIG_IPIPE_CORE */
+
/*
* disable hlt during certain critical i/o operations
*/
--- linux-2.6.12-rc6/ipipe/x86.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.12-rc6-ipipe-0.5/ipipe/x86.c 2005-06-17 12:13:30.000000000 +0200
@@ -0,0 +1,577 @@
+/* -*- linux-c -*-
+ * linux/ipipe/x86.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-PIPE support for x86.
+ */
+
+#include <linux/config.h>
+#include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/irq.h>
+#include <linux/slab.h>
+#include <asm/system.h>
+#include <asm/atomic.h>
+#include <asm/hw_irq.h>
+#include <asm/irq.h>
+#include <asm/desc.h>
+#include <asm/io.h>
+#ifdef CONFIG_X86_LOCAL_APIC
+#include <asm/fixmap.h>
+#include <asm/bitops.h>
+#include <asm/mpspec.h>
+#ifdef CONFIG_X86_IO_APIC
+#include <asm/io_apic.h>
+#endif /* CONFIG_X86_IO_APIC */
+#include <asm/apic.h>
+#include <mach_ipi.h>
+#endif /* CONFIG_X86_LOCAL_APIC */
+
+extern struct desc_struct idt_table[];
+
+extern void (*__ipipe_irq_trampolines[]) (void); /* in entry.S */
+
+static void (*__ipipe_std_vector_table[256]) (void);
+
+static struct hw_interrupt_type __ipipe_std_irq_dtype[NR_IRQS];
+
+/* Lifted from arch/i386/kernel/traps.c */
+#define __ipipe_set_gate(gate_addr,type,dpl,addr) \
+do { \
+ int __d0, __d1; \
+ __asm__ __volatile__ ("movw %%dx,%%ax\n\t" \
+ "movw %4,%%dx\n\t" \
+ "movl %%eax,%0\n\t" \
+ "movl %%edx,%1" \
+ :"=m" (*((long *) (gate_addr))), \
+ "=m" (*(1+(long *) (gate_addr))), "=&a" (__d0), "=&d" (__d1) \
+ :"i" ((short) (0x8000+(dpl<<13)+(type<<8))), \
+ "3" ((char *) (addr)),"2" (__KERNEL_CS << 16)); \
+} while (0)
+
+#define __ipipe_get_gate_addr(v) \
+ ((void *)((idt_table[v].b & 0xffff0000)|(idt_table[v].a & 0xffff)))
+
+#define __ipipe_set_irq_gate(vector,addr) \
+__ipipe_set_gate(idt_table+vector,14,0,addr)
+
+#define __ipipe_set_trap_gate(vector,addr) \
+__ipipe_set_gate(idt_table+vector,15,0,addr)
+
+#define __ipipe_set_sys_gate(vector,addr) \
+__ipipe_set_gate(idt_table+vector,15,3,addr)
+
+static int __ipipe_ack_common_irq(unsigned irq)
+{
+ preempt_disable();
+ __ipipe_std_irq_dtype[irq].ack(irq);
+ preempt_enable_no_resched();
+
+ return 1;
+}
+
+static unsigned __ipipe_override_irq_startup(unsigned irq)
+{
+ unsigned long adflags, hwflags;
+ ipipe_declare_cpuid;
+ unsigned s;
+
+ ipipe_lock_cpu(hwflags);
+ adflags = ipipe_test_and_stall_pipeline();
+ preempt_disable();
+ __ipipe_unlock_irq(ipipe_percpu_domain[cpuid], irq);
+ s = __ipipe_std_irq_dtype[irq].startup(irq);
+ preempt_enable_no_resched();
+ ipipe_restore_pipeline_nosync(ipipe_percpu_domain[cpuid], adflags,
+ cpuid);
+ ipipe_unlock_cpu(hwflags);
+
+ return s;
+}
+
+static void __ipipe_override_irq_shutdown(unsigned irq)
+{
+ unsigned long adflags, hwflags;
+ ipipe_declare_cpuid;
+
+ ipipe_lock_cpu(hwflags);
+ adflags = ipipe_test_and_stall_pipeline();
+ preempt_disable();
+ __ipipe_std_irq_dtype[irq].shutdown(irq);
+ __ipipe_clear_irq(ipipe_percpu_domain[cpuid], irq);
+ preempt_enable_no_resched();
+ ipipe_restore_pipeline_nosync(ipipe_percpu_domain[cpuid], adflags,
+ cpuid);
+ ipipe_unlock_cpu(hwflags);
+}
+
+static void __ipipe_override_irq_enable(unsigned irq)
+{
+ unsigned long adflags, hwflags;
+ ipipe_declare_cpuid;
+
+ ipipe_lock_cpu(hwflags);
+ adflags = ipipe_test_and_stall_pipeline();
+ preempt_disable();
+ __ipipe_unlock_irq(ipipe_percpu_domain[cpuid], irq);
+ __ipipe_std_irq_dtype[irq].enable(irq);
+ preempt_enable_no_resched();
+ ipipe_restore_pipeline_nosync(ipipe_percpu_domain[cpuid], adflags,
+ cpuid);
+ ipipe_unlock_cpu(hwflags);
+}
+
+static void __ipipe_override_irq_disable(unsigned irq)
+{
+ unsigned long adflags, hwflags;
+ ipipe_declare_cpuid;
+
+ ipipe_lock_cpu(hwflags);
+ adflags = ipipe_test_and_stall_pipeline();
+ preempt_disable();
+ __ipipe_std_irq_dtype[irq].disable(irq);
+ __ipipe_lock_irq(ipipe_percpu_domain[cpuid], cpuid, irq);
+ preempt_enable_no_resched();
+ ipipe_restore_pipeline_nosync(ipipe_percpu_domain[cpuid], adflags,
+ cpuid);
+ ipipe_unlock_cpu(hwflags);
+}
+
+static void __ipipe_override_irq_ack(unsigned irq)
+{
+ if (!ipipe_running)
+ __ipipe_std_irq_dtype[irq].ack(irq);
+}
+
+static void __ipipe_override_irq_end(unsigned irq)
+{
+ unsigned long adflags, hwflags;
+ ipipe_declare_cpuid;
+
+ ipipe_lock_cpu(hwflags);
+ adflags = ipipe_test_and_stall_pipeline();
+ preempt_disable();
+
+ if (!(irq_desc[irq].status & (IRQ_DISABLED | IRQ_INPROGRESS)))
+ __ipipe_unlock_irq(ipipe_percpu_domain[cpuid], irq);
+
+ __ipipe_std_irq_dtype[irq].end(irq);
+
+ preempt_enable_no_resched();
+ ipipe_restore_pipeline_nosync(ipipe_percpu_domain[cpuid], adflags,
+ cpuid);
+ ipipe_unlock_cpu(hwflags);
+}
+
+static void __ipipe_override_irq_affinity(unsigned irq, cpumask_t mask)
+{
+ unsigned long adflags, hwflags;
+ ipipe_declare_cpuid;
+
+ ipipe_lock_cpu(hwflags);
+ adflags = ipipe_test_and_stall_pipeline();
+ preempt_disable();
+ __ipipe_std_irq_dtype[irq].set_affinity(irq, mask);
+ preempt_enable_no_resched();
+ ipipe_restore_pipeline_nosync(ipipe_percpu_domain[cpuid], adflags,
+ cpuid);
+ ipipe_unlock_cpu(hwflags);
+}
+
+/* __ipipe_enable_pipeline() -- Take over the interrupt control from
+ the root domain (i.e. Linux). After this routine has returned, all
+ interrupts go through the pipeline. */
+
+void __ipipe_enable_pipeline(void)
+{
+ unsigned vector, irq, rirq;
+ unsigned long flags;
+
+ /* Collect the original vector table. */
+
+ for (vector = 0; vector < 256; vector++)
+ __ipipe_std_vector_table[vector] =
+ __ipipe_get_gate_addr(vector);
+
+#ifdef CONFIG_SMP
+
+ /* This vector must be set up prior to call
+ ipipe_critical_enter(). */
+
+ __ipipe_set_irq_gate(IPIPE_CRITICAL_VECTOR,
+ __ipipe_irq_trampolines[IPIPE_CRITICAL_IPI]);
+
+#endif /* CONFIG_SMP */
+
+ flags = ipipe_critical_enter(NULL);
+
+ /* First, grab the ISA and IO-APIC interrupts. */
+
+ for (irq = 0;
+ irq < NR_IRQS && irq + FIRST_EXTERNAL_VECTOR < FIRST_SYSTEM_VECTOR;
+ irq++) {
+ rirq = irq;
+
+#ifdef CONFIG_X86_IO_APIC
+ if (IO_APIC_IRQ(irq)) {
+ vector = IO_APIC_VECTOR(irq);
+
+ if (vector == 0)
+ continue;
+
+#ifdef CONFIG_PCI_MSI
+ /* Account specifically for MSI routing. */
+ if (!platform_legacy_irq(irq))
+ rirq = vector;
+#endif /* CONFIG_PCI_MSI */
+ } else
+#endif /* CONFIG_X86_IO_APIC */
+ {
+ vector = irq + FIRST_EXTERNAL_VECTOR;
+
+ if (vector == SYSCALL_VECTOR)
+ continue;
+ }
+
+ /* Fails for IPIPE_CRITICAL_IPI but that's ok. */
+
+ ipipe_virtualize_irq(rirq,
+ (void (*)(unsigned))
+ __ipipe_std_vector_table[vector],
+ &__ipipe_ack_common_irq,
+ IPIPE_CALLASM_MASK | IPIPE_HANDLE_MASK |
+ IPIPE_PASS_MASK);
+
+ __ipipe_set_irq_gate(vector, __ipipe_irq_trampolines[rirq]);
+ }
+
+ /* Interpose on the IRQ control routines so we can make them
+ atomic using hw masking and prevent the interrupt log from
+ being untimely flushed. Since we don't want to be too smart
+ about what's going on into irq.c and we want to change only
+ some of the controller members, let's be dumb and interpose the
+ rough way. */
+
+ for (irq = 0; irq < NR_IRQS; irq++)
+ __ipipe_std_irq_dtype[irq] = *irq_desc[irq].handler;
+
+ /* The original controller structs may be shared, so we first
+ save them all before changing any of them. Notice that we
+ don't redirect the ack handler since the relevant
+ XT-PIC/IO-APIC management code is already IPIPE-aware. */
+
+ for (irq = 0; irq < NR_IRQS; irq++) {
+ irq_desc[irq].handler->startup = &__ipipe_override_irq_startup;
+ irq_desc[irq].handler->shutdown =
+ &__ipipe_override_irq_shutdown;
+ irq_desc[irq].handler->enable = &__ipipe_override_irq_enable;
+ irq_desc[irq].handler->disable = &__ipipe_override_irq_disable;
+ irq_desc[irq].handler->ack = &__ipipe_override_irq_ack;
+ irq_desc[irq].handler->end = &__ipipe_override_irq_end;
+
+ if (irq_desc[irq].handler->set_affinity != NULL)
+ irq_desc[irq].handler->set_affinity =
+ &__ipipe_override_irq_affinity;
+ }
+
+#ifdef CONFIG_X86_LOCAL_APIC
+
+ /* Map the APIC system vectors including the unused ones so that
+ client domains can virtualize the corresponding IRQs. */
+
+ for (vector = FIRST_SYSTEM_VECTOR; vector < CALL_FUNCTION_VECTOR;
+ vector++) {
+ ipipe_virtualize_irq(vector - FIRST_EXTERNAL_VECTOR,
+ (void (*)(unsigned))
+ __ipipe_std_vector_table[vector],
+ &__ipipe_ack_system_irq,
+ IPIPE_CALLASM_MASK | IPIPE_HANDLE_MASK |
+ IPIPE_PASS_MASK);
+
+ __ipipe_set_irq_gate(vector,
+ __ipipe_irq_trampolines[vector -
+ FIRST_EXTERNAL_VECTOR]);
+ }
+
+ __ipipe_tick_irq =
+ using_apic_timer ? LOCAL_TIMER_VECTOR - FIRST_EXTERNAL_VECTOR : 0;
+
+#else /* !CONFIG_X86_LOCAL_APIC */
+
+ __ipipe_tick_irq = 0;
+
+#endif /* CONFIG_X86_LOCAL_APIC */
+
+#ifdef CONFIG_SMP
+
+ /* All interrupts must be pipelined, but the spurious one since we
+ don't even want to acknowledge it. */
+
+ for (vector = CALL_FUNCTION_VECTOR; vector < SPURIOUS_APIC_VECTOR;
+ vector++) {
+ ipipe_virtualize_irq(vector - FIRST_EXTERNAL_VECTOR,
+ (void (*)(unsigned))
+ __ipipe_std_vector_table[vector],
+ &__ipipe_ack_system_irq,
+ IPIPE_CALLASM_MASK | IPIPE_HANDLE_MASK |
+ IPIPE_PASS_MASK);
+
+ __ipipe_set_irq_gate(vector,
+ __ipipe_irq_trampolines[vector -
+ FIRST_EXTERNAL_VECTOR]);
+ }
+
+#endif /* CONFIG_SMP */
+
+ ipipe_running = 1;
+
+ ipipe_critical_exit(flags);
+
+ printk(KERN_WARNING "I-pipe: Pipelining started.\n");
+}
+
+/* __ipipe_disable_pipeline() -- Disengage the pipeline. */
+
+void __ipipe_disable_pipeline(void)
+{
+ unsigned vector, irq;
+ unsigned long flags;
+
+ flags = ipipe_critical_enter(NULL);
+
+ /* Restore interrupt controllers. */
+
+ for (irq = 0; irq < NR_IRQS; irq++)
+ *irq_desc[irq].handler = __ipipe_std_irq_dtype[irq];
+
+ /* Restore original IDT settings. */
+
+ for (irq = 0;
+ irq < NR_IRQS && irq + FIRST_EXTERNAL_VECTOR < FIRST_SYSTEM_VECTOR;
+ irq++) {
+#ifdef CONFIG_X86_IO_APIC
+ if (IO_APIC_IRQ(irq)) {
+ vector = IO_APIC_VECTOR(irq);
+
+ if (vector == 0)
+ continue;
+ } else
+#endif /* CONFIG_X86_IO_APIC */
+ {
+ vector = irq + FIRST_EXTERNAL_VECTOR;
+
+ if (vector == SYSCALL_VECTOR)
+ continue;
+ }
+
+ __ipipe_set_irq_gate(vector, __ipipe_std_vector_table[vector]);
+ }
+
+#ifdef CONFIG_X86_LOCAL_APIC
+
+ for (vector = FIRST_SYSTEM_VECTOR; vector < CALL_FUNCTION_VECTOR;
+ vector++)
+ __ipipe_set_irq_gate(vector, __ipipe_std_vector_table[vector]);
+
+#endif /* CONFIG_X86_LOCAL_APIC */
+
+#ifdef CONFIG_SMP
+
+ for (vector = CALL_FUNCTION_VECTOR; vector < SPURIOUS_APIC_VECTOR;
+ vector++)
+ __ipipe_set_irq_gate(vector, __ipipe_std_vector_table[vector]);
+
+ __ipipe_set_irq_gate(IPIPE_CRITICAL_VECTOR,
+ __ipipe_std_vector_table[IPIPE_CRITICAL_VECTOR]);
+
+#endif /* CONFIG_SMP */
+
+ ipipe_running = 0;
+
+ ipipe_critical_exit(flags);
+
+ printk(KERN_WARNING "I-pipe: Pipelining stopped.\n");
+}
+
+/* ipipe_virtualize_irq_from() -- Attach a handler (and optionally a
+ hw acknowledge routine) to an interrupt for the given domain. */
+
+int ipipe_virtualize_irq_from(struct ipipe_domain *ipd,
+ unsigned irq,
+ void (*handler) (unsigned irq),
+ int (*acknowledge) (unsigned irq),
+ unsigned modemask)
+{
+ unsigned long flags;
+ int err;
+
+ if (irq >= IPIPE_NR_IRQS)
+ return -EINVAL;
+
+ if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
+ return -EPERM;
+
+ spin_lock_irqsave_hw(&__ipipe_pipelock, flags);
+
+ if (handler != NULL) {
+
+ if (handler == IPIPE_SAME_HANDLER) {
+ handler = ipd->irqs[irq].handler;
+
+ if (handler == NULL) {
+ err = -EINVAL;
+ goto unlock_and_exit;
+ }
+ } else if ((modemask & IPIPE_EXCLUSIVE_MASK) != 0 &&
+ ipd->irqs[irq].handler != NULL) {
+ err = -EBUSY;
+ goto unlock_and_exit;
+ }
+
+ if ((modemask & (IPIPE_SHARED_MASK | IPIPE_PASS_MASK)) ==
+ IPIPE_SHARED_MASK) {
+ err = -EINVAL;
+ goto unlock_and_exit;
+ }
+
+ if ((modemask & IPIPE_STICKY_MASK) != 0)
+ modemask |= IPIPE_HANDLE_MASK;
+ } else
+ modemask &=
+ ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK |
+ IPIPE_SHARED_MASK);
+
+ if (acknowledge == NULL) {
+ if ((modemask & IPIPE_SHARED_MASK) == 0)
+ /* Acknowledge handler unspecified -- this is ok in
+ non-shared management mode, but we will force the use
+ of the Linux-defined handler instead. */
+ acknowledge = ipipe_root_domain->irqs[irq].acknowledge;
+ else {
+ /* A valid acknowledge handler to be called in shared mode
+ is required when declaring a shared IRQ. */
+ err = -EINVAL;
+ goto unlock_and_exit;
+ }
+ }
+
+ ipd->irqs[irq].handler = handler;
+ ipd->irqs[irq].acknowledge = acknowledge;
+ ipd->irqs[irq].control = modemask;
+
+ if (irq < NR_IRQS &&
+ handler != NULL &&
+ !ipipe_virtual_irq_p(irq) && (modemask & IPIPE_ENABLE_MASK) != 0) {
+ if (ipd != ipipe_current_domain) {
+ /* IRQ enable/disable state is domain-sensitive, so we may
+ not change it for another domain. What is allowed
+ however is forcing some domain to handle an interrupt
+ source, by passing the proper 'ipd' descriptor which
+ thus may be different from ipipe_current_domain. */
+ err = -EPERM;
+ goto unlock_and_exit;
+ }
+
+ irq_desc[irq].handler->enable(irq);
+ }
+
+ err = 0;
+
+ unlock_and_exit:
+
+ spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags);
+
+ return err;
+}
+
+/* ipipe_control_irq() -- Change modes of a pipelined interrupt for
+ * the current domain. */
+
+int ipipe_control_irq(unsigned irq, unsigned clrmask, unsigned setmask)
+{
+ struct ipipe_domain *ipd;
+ unsigned long flags;
+ irq_desc_t *desc;
+
+ if (irq >= IPIPE_NR_IRQS)
+ return -EINVAL;
+
+ ipd = ipipe_current_domain;
+
+ if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
+ return -EPERM;
+
+ if (((setmask | clrmask) & IPIPE_SHARED_MASK) != 0)
+ return -EINVAL;
+
+ desc = irq_desc + irq;
+
+ if (ipd->irqs[irq].handler == NULL)
+ setmask &= ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
+
+ if ((setmask & IPIPE_STICKY_MASK) != 0)
+ setmask |= IPIPE_HANDLE_MASK;
+
+ if ((clrmask & (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK)) != 0) /* If one goes, both go. */
+ clrmask |= (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
+
+ spin_lock_irqsave_hw(&__ipipe_pipelock, flags);
+
+ ipd->irqs[irq].control &= ~clrmask;
+ ipd->irqs[irq].control |= setmask;
+
+ if ((setmask & IPIPE_ENABLE_MASK) != 0)
+ desc->handler->enable(irq);
+ else if ((clrmask & IPIPE_ENABLE_MASK) != 0)
+ desc->handler->disable(irq);
+
+ spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags);
+
+ return 0;
+}
+
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd)
+{
+ ipipe_unstall_pipeline_from(ipd);
+
+#ifdef CONFIG_SMP
+ {
+ int nr_cpus = num_online_cpus(), _cpuid;
+
+ for (_cpuid = 0; _cpuid < nr_cpus; _cpuid++)
+ while (ipd->cpudata[_cpuid].irq_pending_hi != 0)
+ cpu_relax();
+ }
+#endif /* CONFIG_SMP */
+}
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
+{
+ info->ncpus = num_online_cpus();
+ info->cpufreq = ipipe_cpu_freq();
+ info->archdep.tmirq = __ipipe_tick_irq;
+#ifdef CONFIG_X86_TSC
+ info->archdep.tmfreq = ipipe_cpu_freq();
+#else /* !CONFIG_X86_TSC */
+ info->archdep.tmfreq = CLOCK_TICK_RATE;
+#endif /* CONFIG_X86_TSC */
+
+ return 0;
+}
--

Philippe.

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/