[PATCH] PER_CPU [3/4] - PER_CPU-cpu_tlbstate

From: Shai Fultheim
Date: Fri Jul 09 2004 - 05:01:10 EST

[SECOND SUBBMITAL - Thanks for all the comments]
Andrew,

Please find below one out of collection of patched that move NR_CPU array
variables to the per-cpu area. Please consider applying, any comment will
highly appreciated.

Patches (altogether) tested using make allmodconfig, and defconfig, and
booted my system very nicely.

1/4. PER_CPU-cpu_gdt_table
2/4. PER_CPU-init_tss
3/4. PER_CPU-cpu_tlbstate
4/4. PER_CPU-irq_stat

PER_CPU-cpu_tlbstate:
arch/i386/kernel/smp.c | 12 ++++++------
arch/i386/mach-voyager/voyager_smp.c | 12 ++++++------
include/asm-i386/mmu_context.h | 12 ++++++------
include/asm-i386/tlbflush.h | 2 +-
4 files changed, 19 insertions(+), 19 deletions(-)

Signed-off-by: Martin Hicks <mort@xxxxxxxxxxxxxxxxxx>
Signed-off-by: Shai Fultheim <shai@xxxxxxxxxxxx>

=================================================================================
# This is a BitKeeper generated diff -Nru style patch.
#
# ChangeSet
# 2004/07/09 01:15:31-07:00 shai@compile.(none)
# tlbflush.h, mmu_context.h, voyager_smp.c, smp.c:
# PER_CPU-cpu_tlbstate
# Convert cpu_tlbstate into a per_cpu variable.
#
# Signed-off-by: Martin Hicks <mort@xxxxxxxxxxxxxxxxxx>
# Signed-off-by: Shai Fultheim <shai@xxxxxxxxxxxx>
#
# include/asm-i386/tlbflush.h
# 2004/07/09 01:14:52-07:00 shai@compile.(none) +1 -1
# PER_CPU-cpu_tlbstate
#
# include/asm-i386/mmu_context.h
# 2004/07/09 01:14:52-07:00 shai@compile.(none) +6 -6
# PER_CPU-cpu_tlbstate
#
# arch/i386/mach-voyager/voyager_smp.c
# 2004/07/09 01:14:52-07:00 shai@compile.(none) +6 -6
# PER_CPU-cpu_tlbstate
#
# arch/i386/kernel/smp.c
# 2004/07/09 01:14:52-07:00 shai@compile.(none) +6 -6
# PER_CPU-cpu_tlbstate
#
diff -Nru a/arch/i386/kernel/smp.c b/arch/i386/kernel/smp.c
--- a/arch/i386/kernel/smp.c 2004-07-09 01:34:17 -07:00
+++ b/arch/i386/kernel/smp.c 2004-07-09 01:34:17 -07:00
@@ -104,7 +104,7 @@
* about nothing of note with C stepping upwards.
*/

-struct tlb_state cpu_tlbstate[NR_CPUS] __cacheline_aligned = {[0 ... NR_CPUS-1] = { &init_mm, 0, }};
+DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate) ____cacheline_aligned = { &init_mm, 0, };

/*
* the following functions deal with sending IPIs between CPUs.
@@ -255,9 +255,9 @@
*/
static inline void leave_mm (unsigned long cpu)
{
- if (cpu_tlbstate[cpu].state == TLBSTATE_OK)
+ if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
BUG();
- cpu_clear(cpu, cpu_tlbstate[cpu].active_mm->cpu_vm_mask);
+ cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask);
load_cr3(swapper_pg_dir);
}

@@ -324,8 +324,8 @@
* BUG();
*/

- if (flush_mm == cpu_tlbstate[cpu].active_mm) {
- if (cpu_tlbstate[cpu].state == TLBSTATE_OK) {
+ if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) {
+ if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) {
if (flush_va == FLUSH_ALL)
local_flush_tlb();
else
@@ -457,7 +457,7 @@
unsigned long cpu = smp_processor_id();

__flush_tlb_all();
- if (cpu_tlbstate[cpu].state == TLBSTATE_LAZY)
+ if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY)
leave_mm(cpu);
}

diff -Nru a/arch/i386/mach-voyager/voyager_smp.c b/arch/i386/mach-voyager/voyager_smp.c
--- a/arch/i386/mach-voyager/voyager_smp.c 2004-07-09 01:34:17 -07:00
+++ b/arch/i386/mach-voyager/voyager_smp.c 2004-07-09 01:34:17 -07:00
@@ -35,7 +35,7 @@
int reboot_smp = 0;

/* TLB state -- visible externally, indexed physically */
-struct tlb_state cpu_tlbstate[NR_CPUS] __cacheline_aligned = {[0 ... NR_CPUS-1] = { &init_mm, 0 }};
+DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate) ____cacheline_aligned = { &init_mm, 0 };

/* CPU IRQ affinity -- set to all ones initially */
static unsigned long cpu_irq_affinity[NR_CPUS] __cacheline_aligned = { [0 ... NR_CPUS-1] = ~0UL };
@@ -860,9 +860,9 @@
static inline void
leave_mm (unsigned long cpu)
{
- if (cpu_tlbstate[cpu].state == TLBSTATE_OK)
+ if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
BUG();
- cpu_clear(cpu, cpu_tlbstate[cpu].active_mm->cpu_vm_mask);
+ cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask);
load_cr3(swapper_pg_dir);
}

@@ -883,8 +883,8 @@
smp_processor_id()));
*/

- if (flush_mm == cpu_tlbstate[cpu].active_mm) {
- if (cpu_tlbstate[cpu].state == TLBSTATE_OK) {
+ if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) {
+ if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) {
if (flush_va == FLUSH_ALL)
local_flush_tlb();
else
@@ -1218,7 +1218,7 @@
unsigned long cpu = smp_processor_id();

__flush_tlb_all();
- if (cpu_tlbstate[cpu].state == TLBSTATE_LAZY)
+ if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY)
leave_mm(cpu);
}

diff -Nru a/include/asm-i386/mmu_context.h b/include/asm-i386/mmu_context.h
--- a/include/asm-i386/mmu_context.h 2004-07-09 01:34:17 -07:00
+++ b/include/asm-i386/mmu_context.h 2004-07-09 01:34:17 -07:00
@@ -18,8 +18,8 @@
{
#ifdef CONFIG_SMP
unsigned cpu = smp_processor_id();
- if (cpu_tlbstate[cpu].state == TLBSTATE_OK)
- cpu_tlbstate[cpu].state = TLBSTATE_LAZY;
+ if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
+ per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_LAZY;
#endif
}

@@ -33,8 +33,8 @@
/* stop flush ipis for the previous mm */
cpu_clear(cpu, prev->cpu_vm_mask);
#ifdef CONFIG_SMP
- cpu_tlbstate[cpu].state = TLBSTATE_OK;
- cpu_tlbstate[cpu].active_mm = next;
+ per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_OK;
+ per_cpu(cpu_tlbstate, cpu).active_mm = next;
#endif
cpu_set(cpu, next->cpu_vm_mask);

@@ -49,8 +49,8 @@
}
#ifdef CONFIG_SMP
else {
- cpu_tlbstate[cpu].state = TLBSTATE_OK;
- BUG_ON(cpu_tlbstate[cpu].active_mm != next);
+ per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_OK;
+ BUG_ON(per_cpu(cpu_tlbstate, cpu).active_mm != next);

if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
/* We were in lazy tlb mode and leave_mm disabled
diff -Nru a/include/asm-i386/tlbflush.h b/include/asm-i386/tlbflush.h
--- a/include/asm-i386/tlbflush.h 2004-07-09 01:34:17 -07:00
+++ b/include/asm-i386/tlbflush.h 2004-07-09 01:34:17 -07:00
@@ -131,7 +131,7 @@
int state;
char __cacheline_padding[L1_CACHE_BYTES-8];
};
-extern struct tlb_state cpu_tlbstate[NR_CPUS];
+DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);


#endif
=================================================================================
 
-----------------
Shai Fultheim
Scalex86.org


Attachment: rev-1.1821.patch
Description: Binary data