Re: Add prefetch switch stack hook in scheduler function
From: Ingo Molnar
Date: Thu Jul 28 2005 - 04:11:51 EST
* Ingo Molnar <mingo@xxxxxxx> wrote:
> [...] If yes then we want to have something like:
>
> prefetch(kernel_stack(next));
>
> to make it more generic. By default kernel_stack(next) could be
> next->thread_info (to make sure we prefetch something real). On e.g.
> x86/x64, kernel_stack(next) should be something like next->thread.esp.
i.e. like the patch below. Boot-tested on x86. x86, x64 and ia64 have a
real kernel_stack() implementation, the other architectures all return
'next'. (I've also cleaned up a couple of other things in the
prefetch-next area, see the changelog below.)
Ken, would this patch generate a sufficient amount of prefetching on
ia64?
Ingo
-----
For architecture like ia64, the switch stack structure is fairly large
(currently 528 bytes). For context switch intensive application, we
found that significant amount of cache misses occurs in switch_to()
function. The following patch adds a hook in the schedule() function to
prefetch switch stack structure as soon as 'next' task is determined.
This allows maximum overlap in prefetch cache lines for that structure.
Signed-off-by: Ken Chen <kenneth.w.chen@xxxxxxxxx>
modifications:
- add a generic kernel_stack() function instead of a hook
- prefetch the next task's thread_info and kernel stack as well
- dont try to prefetch 'next' itself - we've already touched it
so the cacheline is present.
- do the prefetching as early as possible
currently covered architectures: ia64, x86, x64. (Other architectures
will have to replace the default 'return task' in their kernel_stack()
implementations.)
Signed-off-by: Ingo Molnar <mingo@xxxxxxx>
include/asm-alpha/mmu_context.h | 9 +++++++++
include/asm-arm/mmu_context.h | 9 +++++++++
include/asm-arm26/mmu_context.h | 9 +++++++++
include/asm-cris/mmu_context.h | 9 +++++++++
include/asm-frv/mmu_context.h | 9 +++++++++
include/asm-h8300/mmu_context.h | 9 +++++++++
include/asm-i386/mmu_context.h | 5 +++++
include/asm-ia64/mmu_context.h | 9 +++++++++
include/asm-m32r/mmu_context.h | 9 +++++++++
include/asm-m68k/mmu_context.h | 9 +++++++++
include/asm-m68knommu/mmu_context.h | 9 +++++++++
include/asm-mips/mmu_context.h | 9 +++++++++
include/asm-parisc/mmu_context.h | 10 ++++++++++
include/asm-ppc/mmu_context.h | 9 +++++++++
include/asm-ppc64/mmu_context.h | 9 +++++++++
include/asm-s390/mmu_context.h | 9 +++++++++
include/asm-sh/mmu_context.h | 9 +++++++++
include/asm-sh64/mmu_context.h | 9 +++++++++
include/asm-sparc/mmu_context.h | 9 +++++++++
include/asm-sparc64/mmu_context.h | 9 +++++++++
include/asm-um/mmu_context.h | 9 +++++++++
include/asm-v850/mmu_context.h | 9 +++++++++
include/asm-x86_64/mmu_context.h | 9 +++++++++
include/asm-xtensa/mmu_context.h | 9 +++++++++
kernel/sched.c | 9 ++++++++-
25 files changed, 221 insertions(+), 1 deletion(-)
Index: linux/include/asm-alpha/mmu_context.h
===================================================================
--- linux.orig/include/asm-alpha/mmu_context.h
+++ linux/include/asm-alpha/mmu_context.h
@@ -258,4 +258,13 @@ enter_lazy_tlb(struct mm_struct *mm, str
#undef __MMU_EXTERN_INLINE
#endif
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* __ALPHA_MMU_CONTEXT_H */
Index: linux/include/asm-arm/mmu_context.h
===================================================================
--- linux.orig/include/asm-arm/mmu_context.h
+++ linux/include/asm-arm/mmu_context.h
@@ -93,4 +93,13 @@ switch_mm(struct mm_struct *prev, struct
#define deactivate_mm(tsk,mm) do { } while (0)
#define activate_mm(prev,next) switch_mm(prev, next, NULL)
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif
Index: linux/include/asm-arm26/mmu_context.h
===================================================================
--- linux.orig/include/asm-arm26/mmu_context.h
+++ linux/include/asm-arm26/mmu_context.h
@@ -48,4 +48,13 @@ static inline void activate_mm(struct mm
cpu_switch_mm(next->pgd, next);
}
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif
Index: linux/include/asm-cris/mmu_context.h
===================================================================
--- linux.orig/include/asm-cris/mmu_context.h
+++ linux/include/asm-cris/mmu_context.h
@@ -21,4 +21,13 @@ static inline void enter_lazy_tlb(struct
{
}
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif
Index: linux/include/asm-frv/mmu_context.h
===================================================================
--- linux.orig/include/asm-frv/mmu_context.h
+++ linux/include/asm-frv/mmu_context.h
@@ -47,4 +47,13 @@ do { \
do { \
} while(0)
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif
Index: linux/include/asm-h8300/mmu_context.h
===================================================================
--- linux.orig/include/asm-h8300/mmu_context.h
+++ linux/include/asm-h8300/mmu_context.h
@@ -29,4 +29,13 @@ extern inline void activate_mm(struct mm
{
}
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif
Index: linux/include/asm-i386/mmu_context.h
===================================================================
--- linux.orig/include/asm-i386/mmu_context.h
+++ linux/include/asm-i386/mmu_context.h
@@ -69,4 +69,9 @@ static inline void switch_mm(struct mm_s
#define activate_mm(prev, next) \
switch_mm((prev),(next),NULL)
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return (void *) task->thread.esp;
+}
+
#endif
Index: linux/include/asm-ia64/mmu_context.h
===================================================================
--- linux.orig/include/asm-ia64/mmu_context.h
+++ linux/include/asm-ia64/mmu_context.h
@@ -169,5 +169,14 @@ activate_mm (struct mm_struct *prev, str
#define switch_mm(prev_mm,next_mm,next_task) activate_mm(prev_mm, next_mm)
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return (void *) task->thread.ksp;
+}
+
# endif /* ! __ASSEMBLY__ */
#endif /* _ASM_IA64_MMU_CONTEXT_H */
Index: linux/include/asm-m32r/mmu_context.h
===================================================================
--- linux.orig/include/asm-m32r/mmu_context.h
+++ linux/include/asm-m32r/mmu_context.h
@@ -167,4 +167,13 @@ static inline void switch_mm(struct mm_s
#endif /* __KERNEL__ */
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* _ASM_M32R_MMU_CONTEXT_H */
Index: linux/include/asm-m68k/mmu_context.h
===================================================================
--- linux.orig/include/asm-m68k/mmu_context.h
+++ linux/include/asm-m68k/mmu_context.h
@@ -150,5 +150,14 @@ static inline void activate_mm(struct mm
activate_context(next_mm);
}
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif
#endif
Index: linux/include/asm-m68knommu/mmu_context.h
===================================================================
--- linux.orig/include/asm-m68knommu/mmu_context.h
+++ linux/include/asm-m68knommu/mmu_context.h
@@ -30,4 +30,13 @@ extern inline void activate_mm(struct mm
{
}
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif
Index: linux/include/asm-mips/mmu_context.h
===================================================================
--- linux.orig/include/asm-mips/mmu_context.h
+++ linux/include/asm-mips/mmu_context.h
@@ -193,4 +193,13 @@ drop_mmu_context(struct mm_struct *mm, u
local_irq_restore(flags);
}
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* _ASM_MMU_CONTEXT_H */
Index: linux/include/asm-parisc/mmu_context.h
===================================================================
--- linux.orig/include/asm-parisc/mmu_context.h
+++ linux/include/asm-parisc/mmu_context.h
@@ -70,4 +70,14 @@ static inline void activate_mm(struct mm
switch_mm(prev,next,current);
}
+
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif
Index: linux/include/asm-ppc/mmu_context.h
===================================================================
--- linux.orig/include/asm-ppc/mmu_context.h
+++ linux/include/asm-ppc/mmu_context.h
@@ -195,5 +195,14 @@ static inline void switch_mm(struct mm_s
extern void mmu_context_init(void);
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* __PPC_MMU_CONTEXT_H */
#endif /* __KERNEL__ */
Index: linux/include/asm-ppc64/mmu_context.h
===================================================================
--- linux.orig/include/asm-ppc64/mmu_context.h
+++ linux/include/asm-ppc64/mmu_context.h
@@ -84,4 +84,13 @@ static inline void activate_mm(struct mm
local_irq_restore(flags);
}
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* __PPC64_MMU_CONTEXT_H */
Index: linux/include/asm-s390/mmu_context.h
===================================================================
--- linux.orig/include/asm-s390/mmu_context.h
+++ linux/include/asm-s390/mmu_context.h
@@ -51,4 +51,13 @@ extern inline void activate_mm(struct mm
set_fs(current->thread.mm_segment);
}
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif
Index: linux/include/asm-sh/mmu_context.h
===================================================================
--- linux.orig/include/asm-sh/mmu_context.h
+++ linux/include/asm-sh/mmu_context.h
@@ -202,5 +202,14 @@ static inline void disable_mmu(void)
#define disable_mmu() do { BUG(); } while (0)
#endif
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* __KERNEL__ */
#endif /* __ASM_SH_MMU_CONTEXT_H */
Index: linux/include/asm-sh64/mmu_context.h
===================================================================
--- linux.orig/include/asm-sh64/mmu_context.h
+++ linux/include/asm-sh64/mmu_context.h
@@ -206,4 +206,13 @@ enter_lazy_tlb(struct mm_struct *mm, str
#endif /* __ASSEMBLY__ */
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* __ASM_SH64_MMU_CONTEXT_H */
Index: linux/include/asm-sparc/mmu_context.h
===================================================================
--- linux.orig/include/asm-sparc/mmu_context.h
+++ linux/include/asm-sparc/mmu_context.h
@@ -37,4 +37,13 @@ BTFIXUPDEF_CALL(void, switch_mm, struct
#endif /* !(__ASSEMBLY__) */
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* !(__SPARC_MMU_CONTEXT_H) */
Index: linux/include/asm-sparc64/mmu_context.h
===================================================================
--- linux.orig/include/asm-sparc64/mmu_context.h
+++ linux/include/asm-sparc64/mmu_context.h
@@ -142,4 +142,13 @@ static inline void activate_mm(struct mm
#endif /* !(__ASSEMBLY__) */
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* !(__SPARC64_MMU_CONTEXT_H) */
Index: linux/include/asm-um/mmu_context.h
===================================================================
--- linux.orig/include/asm-um/mmu_context.h
+++ linux/include/asm-um/mmu_context.h
@@ -66,6 +66,15 @@ static inline void destroy_context(struc
CHOOSE_MODE((void) 0, destroy_context_skas(mm));
}
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif
/*
Index: linux/include/asm-v850/mmu_context.h
===================================================================
--- linux.orig/include/asm-v850/mmu_context.h
+++ linux/include/asm-v850/mmu_context.h
@@ -8,4 +8,13 @@
#define activate_mm(prev,next) ((void)0)
#define enter_lazy_tlb(mm,tsk) ((void)0)
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* __V850_MMU_CONTEXT_H__ */
Index: linux/include/asm-x86_64/mmu_context.h
===================================================================
--- linux.orig/include/asm-x86_64/mmu_context.h
+++ linux/include/asm-x86_64/mmu_context.h
@@ -76,4 +76,13 @@ static inline void switch_mm(struct mm_s
switch_mm((prev),(next),NULL)
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return (void *) task->thread.rsp;
+}
+
#endif
Index: linux/include/asm-xtensa/mmu_context.h
===================================================================
--- linux.orig/include/asm-xtensa/mmu_context.h
+++ linux/include/asm-xtensa/mmu_context.h
@@ -327,4 +327,13 @@ static inline void enter_lazy_tlb(struct
}
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+ return task;
+}
+
#endif /* _XTENSA_MMU_CONTEXT_H */
Index: linux/kernel/sched.c
===================================================================
--- linux.orig/kernel/sched.c
+++ linux/kernel/sched.c
@@ -2864,6 +2864,13 @@ go_idle:
queue = array->queue + idx;
next = list_entry(queue->next, task_t, run_list);
+ /*
+ * Cache-prefetch crutial memory areas of the next task,
+ * its thread_info and its kernel stack:
+ */
+ prefetch(next->thread_info);
+ prefetch(kernel_stack(next));
+
if (!rt_task(next) && next->activated > 0) {
unsigned long long delta = now - next->timestamp;
if (unlikely((long long)(now - next->timestamp) < 0))
@@ -2886,7 +2893,7 @@ go_idle:
switch_tasks:
if (next == rq->idle)
schedstat_inc(rq, sched_goidle);
- prefetch(next);
+
clear_tsk_need_resched(prev);
rcu_qsctr_inc(task_cpu(prev));
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/