powerpc: Replace __get_cpu_var uses
From: Christoph Lameter
Date: Fri May 16 2014 - 12:03:41 EST
Got my hands on a power8 system and was able to make sure that this builds
fine. Dont have kvm working yet so the kernel was not tested. Applies
against Linus' tree.
__get_cpu_var() is used for multiple purposes in the kernel source. One of
them is address calculation via the form &__get_cpu_var(x). This calculates
the address for the instance of the percpu variable of the current processor
based on an offset.
Other use cases are for storing and retrieving data from the current
processors percpu area. __get_cpu_var() can be used as an lvalue when
writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store
and retrieve operations could use a segment prefix (or global register on
other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a
percpu area and use optimized assembly code to read and write per cpu
variables.
This patch converts __get_cpu_var into either an explicit address
calculation using this_cpu_ptr() or into a use of this_cpu operations that
use the offset. Thereby address calculations are avoided and less registers
are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so
the macro is removed too.
The patch set includes passes over all arches as well. Once these operations
are used throughout then specialized macros can be defined in non -x86
arches as well in order to optimize per cpu access by f.e. using a global
register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);
2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);
3. Retrieve the content of the current processors instance of a per cpu
variable.
DEFINE_PER_CPU(int, y);
int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y);
struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y)
__get_cpu_var(y) = x;
Converts to
__this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y);
__get_cpu_var(y)++
Converts to
__this_cpu_inc(y)
Reviewed-by: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Signed-off-by: Christoph Lameter <cl@xxxxxxxxx>
Index: linux/arch/powerpc/include/asm/cputime.h
===================================================================
--- linux.orig/arch/powerpc/include/asm/cputime.h
+++ linux/arch/powerpc/include/asm/cputime.h
@@ -56,10 +56,10 @@ static inline unsigned long cputime_to_j
static inline cputime_t cputime_to_scaled(const cputime_t ct)
{
if (cpu_has_feature(CPU_FTR_SPURR) &&
- __get_cpu_var(cputime_last_delta))
+ __this_cpu_read(cputime_last_delta))
return (__force u64) ct *
- __get_cpu_var(cputime_scaled_last_delta) /
- __get_cpu_var(cputime_last_delta);
+ __this_cpu_read(cputime_scaled_last_delta) /
+ __this_cpu_read(cputime_last_delta);
return ct;
}
Index: linux/arch/powerpc/include/asm/hardirq.h
===================================================================
--- linux.orig/arch/powerpc/include/asm/hardirq.h
+++ linux/arch/powerpc/include/asm/hardirq.h
@@ -19,8 +19,11 @@ typedef struct {
DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
#define __ARCH_IRQ_STAT
+#define __ARCH_SET_SOFTIRQ_PENDING
-#define local_softirq_pending() __get_cpu_var(irq_stat).__softirq_pending
+#define local_softirq_pending() __this_cpu_read(irq_stat.__softirq_pending)
+#define set_softirq_pending(x) __this_cpu_write(irq_stat._softirq_pending, (x))
+#define or_softirq_pending(x) __this_cpu_or(irq_stat._softirq_pending, (x))
static inline void ack_bad_irq(unsigned int irq)
{
Index: linux/arch/powerpc/include/asm/tlbflush.h
===================================================================
--- linux.orig/arch/powerpc/include/asm/tlbflush.h
+++ linux/arch/powerpc/include/asm/tlbflush.h
@@ -107,14 +107,14 @@ extern void __flush_tlb_pending(struct p
static inline void arch_enter_lazy_mmu_mode(void)
{
- struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
+ struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
batch->active = 1;
}
static inline void arch_leave_lazy_mmu_mode(void)
{
- struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
+ struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
if (batch->index)
__flush_tlb_pending(batch);
Index: linux/arch/powerpc/include/asm/xics.h
===================================================================
--- linux.orig/arch/powerpc/include/asm/xics.h
+++ linux/arch/powerpc/include/asm/xics.h
@@ -97,7 +97,7 @@ DECLARE_PER_CPU(struct xics_cppr, xics_c
static inline void xics_push_cppr(unsigned int vec)
{
- struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr);
+ struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr);
if (WARN_ON(os_cppr->index >= MAX_NUM_PRIORITIES - 1))
return;
@@ -110,7 +110,7 @@ static inline void xics_push_cppr(unsign
static inline unsigned char xics_pop_cppr(void)
{
- struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr);
+ struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr);
if (WARN_ON(os_cppr->index < 1))
return LOWEST_PRIORITY;
@@ -120,7 +120,7 @@ static inline unsigned char xics_pop_cpp
static inline void xics_set_base_cppr(unsigned char cppr)
{
- struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr);
+ struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr);
/* we only really want to set the priority when there's
* just one cppr value on the stack
@@ -132,7 +132,7 @@ static inline void xics_set_base_cppr(un
static inline unsigned char xics_cppr_top(void)
{
- struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr);
+ struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr);
return os_cppr->stack[os_cppr->index];
}
Index: linux/arch/powerpc/kernel/dbell.c
===================================================================
--- linux.orig/arch/powerpc/kernel/dbell.c
+++ linux/arch/powerpc/kernel/dbell.c
@@ -41,7 +41,7 @@ void doorbell_exception(struct pt_regs *
may_hard_irq_enable();
- __get_cpu_var(irq_stat).doorbell_irqs++;
+ __this_cpu_inc(irq_stat.doorbell_irqs);
smp_ipi_demux();
Index: linux/arch/powerpc/kernel/hw_breakpoint.c
===================================================================
--- linux.orig/arch/powerpc/kernel/hw_breakpoint.c
+++ linux/arch/powerpc/kernel/hw_breakpoint.c
@@ -63,7 +63,7 @@ int hw_breakpoint_slots(int type)
int arch_install_hw_breakpoint(struct perf_event *bp)
{
struct arch_hw_breakpoint *info = counter_arch_bp(bp);
- struct perf_event **slot = &__get_cpu_var(bp_per_reg);
+ struct perf_event **slot = this_cpu_ptr(&bp_per_reg);
*slot = bp;
@@ -88,7 +88,7 @@ int arch_install_hw_breakpoint(struct pe
*/
void arch_uninstall_hw_breakpoint(struct perf_event *bp)
{
- struct perf_event **slot = &__get_cpu_var(bp_per_reg);
+ struct perf_event **slot = this_cpu_ptr(&bp_per_reg);
if (*slot != bp) {
WARN_ONCE(1, "Can't find the breakpoint");
@@ -226,7 +226,7 @@ int __kprobes hw_breakpoint_handler(stru
*/
rcu_read_lock();
- bp = __get_cpu_var(bp_per_reg);
+ bp = __this_cpu_read(bp_per_reg);
if (!bp)
goto out;
info = counter_arch_bp(bp);
Index: linux/arch/powerpc/kernel/irq.c
===================================================================
--- linux.orig/arch/powerpc/kernel/irq.c
+++ linux/arch/powerpc/kernel/irq.c
@@ -114,7 +114,7 @@ static inline notrace void set_soft_enab
static inline notrace int decrementer_check_overflow(void)
{
u64 now = get_tb_or_rtc();
- u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
+ u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
return now >= *next_tb;
}
@@ -485,7 +485,7 @@ void __do_irq(struct pt_regs *regs)
/* And finally process it */
if (unlikely(irq == NO_IRQ))
- __get_cpu_var(irq_stat).spurious_irqs++;
+ __this_cpu_inc(irq_stat.spurious_irqs);
else
generic_handle_irq(irq);
Index: linux/arch/powerpc/kernel/kprobes.c
===================================================================
--- linux.orig/arch/powerpc/kernel/kprobes.c
+++ linux/arch/powerpc/kernel/kprobes.c
@@ -118,7 +118,7 @@ static void __kprobes save_previous_kpro
static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
{
- __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
+ __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
kcb->kprobe_status = kcb->prev_kprobe.status;
kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
}
@@ -126,7 +126,7 @@ static void __kprobes restore_previous_k
static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb)
{
- __get_cpu_var(current_kprobe) = p;
+ __this_cpu_write(current_kprobe, p);
kcb->kprobe_saved_msr = regs->msr;
}
@@ -191,7 +191,7 @@ static int __kprobes kprobe_handler(stru
ret = 1;
goto no_kprobe;
}
- p = __get_cpu_var(current_kprobe);
+ p = __this_cpu_read(current_kprobe);
if (p->break_handler && p->break_handler(p, regs)) {
goto ss_probe;
}
Index: linux/arch/powerpc/kernel/process.c
===================================================================
--- linux.orig/arch/powerpc/kernel/process.c
+++ linux/arch/powerpc/kernel/process.c
@@ -497,7 +497,7 @@ static inline int set_dawr(struct arch_h
int set_breakpoint(struct arch_hw_breakpoint *brk)
{
- __get_cpu_var(current_brk) = *brk;
+ __this_cpu_write(current_brk, *brk);
if (cpu_has_feature(CPU_FTR_DAWR))
return set_dawr(brk);
@@ -833,7 +833,7 @@ struct task_struct *__switch_to(struct t
* schedule DABR
*/
#ifndef CONFIG_HAVE_HW_BREAKPOINT
- if (unlikely(!hw_brk_match(&__get_cpu_var(current_brk), &new->thread.hw_brk)))
+ if (unlikely(!hw_brk_match(this_cpu_ptr(¤t_brk), &new->thread.hw_brk)))
set_breakpoint(&new->thread.hw_brk);
#endif /* CONFIG_HAVE_HW_BREAKPOINT */
#endif
@@ -847,7 +847,7 @@ struct task_struct *__switch_to(struct t
* Collect processor utilization data per process
*/
if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
- struct cpu_usage *cu = &__get_cpu_var(cpu_usage_array);
+ struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
long unsigned start_tb, current_tb;
start_tb = old_thread->start_tb;
cu->current_tb = current_tb = mfspr(SPRN_PURR);
@@ -857,7 +857,7 @@ struct task_struct *__switch_to(struct t
#endif /* CONFIG_PPC64 */
#ifdef CONFIG_PPC_BOOK3S_64
- batch = &__get_cpu_var(ppc64_tlb_batch);
+ batch = this_cpu_ptr(&ppc64_tlb_batch);
if (batch->active) {
current_thread_info()->local_flags |= _TLF_LAZY_MMU;
if (batch->index)
@@ -880,7 +880,7 @@ struct task_struct *__switch_to(struct t
#ifdef CONFIG_PPC_BOOK3S_64
if (current_thread_info()->local_flags & _TLF_LAZY_MMU) {
current_thread_info()->local_flags &= ~_TLF_LAZY_MMU;
- batch = &__get_cpu_var(ppc64_tlb_batch);
+ batch = this_cpu_ptr(&ppc64_tlb_batch);
batch->active = 1;
}
#endif /* CONFIG_PPC_BOOK3S_64 */
Index: linux/arch/powerpc/kernel/smp.c
===================================================================
--- linux.orig/arch/powerpc/kernel/smp.c
+++ linux/arch/powerpc/kernel/smp.c
@@ -241,7 +241,7 @@ void smp_muxed_ipi_message_pass(int cpu,
irqreturn_t smp_ipi_demux(void)
{
- struct cpu_messages *info = &__get_cpu_var(ipi_message);
+ struct cpu_messages *info = this_cpu_ptr(&ipi_message);
unsigned int all;
mb(); /* order any irq clear */
@@ -431,9 +431,9 @@ void generic_mach_cpu_die(void)
idle_task_exit();
cpu = smp_processor_id();
printk(KERN_DEBUG "CPU%d offline\n", cpu);
- __get_cpu_var(cpu_state) = CPU_DEAD;
+ __this_cpu_write(cpu_state, CPU_DEAD);
smp_wmb();
- while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
+ while (__this_cpu_read(cpu_state) != CPU_UP_PREPARE)
cpu_relax();
}
Index: linux/arch/powerpc/kernel/sysfs.c
===================================================================
--- linux.orig/arch/powerpc/kernel/sysfs.c
+++ linux/arch/powerpc/kernel/sysfs.c
@@ -394,10 +394,10 @@ void ppc_enable_pmcs(void)
ppc_set_pmu_inuse(1);
/* Only need to enable them once */
- if (__get_cpu_var(pmcs_enabled))
+ if (__this_cpu_read(pmcs_enabled))
return;
- __get_cpu_var(pmcs_enabled) = 1;
+ __this_cpu_write(pmcs_enabled, 1);
if (ppc_md.enable_pmcs)
ppc_md.enable_pmcs();
Index: linux/arch/powerpc/kernel/time.c
===================================================================
--- linux.orig/arch/powerpc/kernel/time.c
+++ linux/arch/powerpc/kernel/time.c
@@ -458,9 +458,9 @@ static inline void clear_irq_work_pendin
DEFINE_PER_CPU(u8, irq_work_pending);
-#define set_irq_work_pending_flag() __get_cpu_var(irq_work_pending) = 1
-#define test_irq_work_pending() __get_cpu_var(irq_work_pending)
-#define clear_irq_work_pending() __get_cpu_var(irq_work_pending) = 0
+#define set_irq_work_pending_flag() __this_cpu_write(irq_work_pending, 1)
+#define test_irq_work_pending() __this_cpu_read(irq_work_pending)
+#define clear_irq_work_pending() __this_cpu_write(irq_work_pending, 0)
#endif /* 32 vs 64 bit */
@@ -482,8 +482,8 @@ void arch_irq_work_raise(void)
void __timer_interrupt(void)
{
struct pt_regs *regs = get_irq_regs();
- u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
- struct clock_event_device *evt = &__get_cpu_var(decrementers);
+ u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
+ struct clock_event_device *evt = this_cpu_ptr(&decrementers);
u64 now;
trace_timer_interrupt_entry(regs);
@@ -498,7 +498,7 @@ void __timer_interrupt(void)
*next_tb = ~(u64)0;
if (evt->event_handler)
evt->event_handler(evt);
- __get_cpu_var(irq_stat).timer_irqs_event++;
+ __this_cpu_inc(irq_stat.timer_irqs_event);
} else {
now = *next_tb - now;
if (now <= DECREMENTER_MAX)
@@ -506,13 +506,13 @@ void __timer_interrupt(void)
/* We may have raced with new irq work */
if (test_irq_work_pending())
set_dec(1);
- __get_cpu_var(irq_stat).timer_irqs_others++;
+ __this_cpu_inc(irq_stat.timer_irqs_others);
}
#ifdef CONFIG_PPC64
/* collect purr register values often, for accurate calculations */
if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
- struct cpu_usage *cu = &__get_cpu_var(cpu_usage_array);
+ struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
cu->current_tb = mfspr(SPRN_PURR);
}
#endif
@@ -527,7 +527,7 @@ void __timer_interrupt(void)
void timer_interrupt(struct pt_regs * regs)
{
struct pt_regs *old_regs;
- u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
+ u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
/* Ensure a positive value is written to the decrementer, or else
* some CPUs will continue to take decrementer exceptions.
@@ -816,7 +816,7 @@ static int decrementer_set_next_event(un
/* Don't adjust the decrementer if some irq work is pending */
if (test_irq_work_pending())
return 0;
- __get_cpu_var(decrementers_next_tb) = get_tb_or_rtc() + evt;
+ __this_cpu_write(decrementers_next_tb, get_tb_or_rtc() + evt);
set_dec(evt);
/* We may have raced with new irq work */
@@ -836,7 +836,7 @@ static void decrementer_set_mode(enum cl
/* Interrupt handler for the timer broadcast IPI */
void tick_broadcast_ipi_handler(void)
{
- u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
+ u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
*next_tb = get_tb_or_rtc();
__timer_interrupt();
Index: linux/arch/powerpc/kernel/traps.c
===================================================================
--- linux.orig/arch/powerpc/kernel/traps.c
+++ linux/arch/powerpc/kernel/traps.c
@@ -688,7 +688,7 @@ void machine_check_exception(struct pt_r
enum ctx_state prev_state = exception_enter();
int recover = 0;
- __get_cpu_var(irq_stat).mce_exceptions++;
+ __this_cpu_inc(irq_stat.mce_exceptions);
/* See if any machine dependent calls. In theory, we would want
* to call the CPU first, and call the ppc_md. one if the CPU
@@ -1493,7 +1493,7 @@ void vsx_unavailable_tm(struct pt_regs *
void performance_monitor_exception(struct pt_regs *regs)
{
- __get_cpu_var(irq_stat).pmu_irqs++;
+ __this_cpu_inc(irq_stat.pmu_irqs);
perf_irq(regs);
}
Index: linux/arch/powerpc/kvm/e500.c
===================================================================
--- linux.orig/arch/powerpc/kvm/e500.c
+++ linux/arch/powerpc/kvm/e500.c
@@ -76,11 +76,11 @@ static inline int local_sid_setup_one(st
unsigned long sid;
int ret = -1;
- sid = ++(__get_cpu_var(pcpu_last_used_sid));
+ sid = __this_cpu_inc_return(pcpu_last_used_sid);
if (sid < NUM_TIDS) {
- __get_cpu_var(pcpu_sids).entry[sid] = entry;
+ __this_cpu_write(pcpu_sids)entry[sid], entry);
entry->val = sid;
- entry->pentry = &__get_cpu_var(pcpu_sids).entry[sid];
+ entry->pentry = this_cpu_ptr(&pcpu_sids.entry[sid]);
ret = sid;
}
@@ -108,8 +108,8 @@ static inline int local_sid_setup_one(st
static inline int local_sid_lookup(struct id *entry)
{
if (entry && entry->val != 0 &&
- __get_cpu_var(pcpu_sids).entry[entry->val] == entry &&
- entry->pentry == &__get_cpu_var(pcpu_sids).entry[entry->val])
+ __this_cpu_read(pcpu_sids.entry[entry->val]) == entry &&
+ entry->pentry == this_cpu_ptr(&pcpu_sids.entry[entry->val]))
return entry->val;
return -1;
}
@@ -117,8 +117,8 @@ static inline int local_sid_lookup(struc
/* Invalidate all id mappings on local core -- call with preempt disabled */
static inline void local_sid_destroy_all(void)
{
- __get_cpu_var(pcpu_last_used_sid) = 0;
- memset(&__get_cpu_var(pcpu_sids), 0, sizeof(__get_cpu_var(pcpu_sids)));
+ __this_cpu_write(pcpu_last_used_sid, 0);
+ memset(this_cpu_ptr(&pcpu_sids), 0, sizeof(pcpu_sids));
}
static void *kvmppc_e500_id_table_alloc(struct kvmppc_vcpu_e500 *vcpu_e500)
Index: linux/arch/powerpc/kvm/e500mc.c
===================================================================
--- linux.orig/arch/powerpc/kvm/e500mc.c
+++ linux/arch/powerpc/kvm/e500mc.c
@@ -141,9 +141,9 @@ static void kvmppc_core_vcpu_load_e500mc
mtspr(SPRN_GESR, vcpu->arch.shared->esr);
if (vcpu->arch.oldpir != mfspr(SPRN_PIR) ||
- __get_cpu_var(last_vcpu_on_cpu) != vcpu) {
+ __this_cpu_read(last_vcpu_on_cpu) != vcpu) {
kvmppc_e500_tlbil_all(vcpu_e500);
- __get_cpu_var(last_vcpu_on_cpu) = vcpu;
+ __this_cpu_read(last_vcpu_on_cpu) = vcpu;
}
kvmppc_load_guest_fp(vcpu);
Index: linux/arch/powerpc/mm/hash_native_64.c
===================================================================
--- linux.orig/arch/powerpc/mm/hash_native_64.c
+++ linux/arch/powerpc/mm/hash_native_64.c
@@ -643,7 +643,7 @@ static void native_flush_hash_range(unsi
unsigned long want_v;
unsigned long flags;
real_pte_t pte;
- struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
+ struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
unsigned long psize = batch->psize;
int ssize = batch->ssize;
int i;
Index: linux/arch/powerpc/mm/hash_utils_64.c
===================================================================
--- linux.orig/arch/powerpc/mm/hash_utils_64.c
+++ linux/arch/powerpc/mm/hash_utils_64.c
@@ -1298,7 +1298,7 @@ void flush_hash_range(unsigned long numb
else {
int i;
struct ppc64_tlb_batch *batch =
- &__get_cpu_var(ppc64_tlb_batch);
+ this_cpu_ptr(&ppc64_tlb_batch);
for (i = 0; i < number; i++)
flush_hash_page(batch->vpn[i], batch->pte[i],
Index: linux/arch/powerpc/mm/hugetlbpage-book3e.c
===================================================================
--- linux.orig/arch/powerpc/mm/hugetlbpage-book3e.c
+++ linux/arch/powerpc/mm/hugetlbpage-book3e.c
@@ -33,13 +33,13 @@ static inline int tlb1_next(void)
ncams = mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY;
- index = __get_cpu_var(next_tlbcam_idx);
+ index = this_cpu_read(next_tlbcam_idx);
/* Just round-robin the entries and wrap when we hit the end */
if (unlikely(index == ncams - 1))
- __get_cpu_var(next_tlbcam_idx) = tlbcam_index;
+ __this_cpu_write(next_tlbcam_idx, tlbcam_index);
else
- __get_cpu_var(next_tlbcam_idx)++;
+ __this_cpu_inc(next_tlbcam_idx);
return index;
}
Index: linux/arch/powerpc/mm/hugetlbpage.c
===================================================================
--- linux.orig/arch/powerpc/mm/hugetlbpage.c
+++ linux/arch/powerpc/mm/hugetlbpage.c
@@ -472,7 +472,7 @@ static void hugepd_free(struct mmu_gathe
{
struct hugepd_freelist **batchp;
- batchp = &get_cpu_var(hugepd_freelist_cur);
+ batchp = this_cpu_ptr(&hugepd_freelist_cur);
if (atomic_read(&tlb->mm->mm_users) < 2 ||
cpumask_equal(mm_cpumask(tlb->mm),
Index: linux/arch/powerpc/mm/stab.c
===================================================================
--- linux.orig/arch/powerpc/mm/stab.c
+++ linux/arch/powerpc/mm/stab.c
@@ -133,12 +133,12 @@ static int __ste_allocate(unsigned long
stab_entry = make_ste(get_paca()->stab_addr, GET_ESID(ea), vsid);
if (!is_kernel_addr(ea)) {
- offset = __get_cpu_var(stab_cache_ptr);
+ offset = __this_cpu_read(stab_cache_ptr);
if (offset < NR_STAB_CACHE_ENTRIES)
- __get_cpu_var(stab_cache[offset++]) = stab_entry;
+ __this_cpu_write(stab_cache[offset++], stab_entry);
else
offset = NR_STAB_CACHE_ENTRIES+1;
- __get_cpu_var(stab_cache_ptr) = offset;
+ __this_cpu_write(stab_cache_ptr, offset);
/* Order update */
asm volatile("sync":::"memory");
@@ -177,12 +177,12 @@ void switch_stab(struct task_struct *tsk
*/
hard_irq_disable();
- offset = __get_cpu_var(stab_cache_ptr);
+ offset = __this_cpu_read(stab_cache_ptr);
if (offset <= NR_STAB_CACHE_ENTRIES) {
int i;
for (i = 0; i < offset; i++) {
- ste = stab + __get_cpu_var(stab_cache[i]);
+ ste = stab + __this_cpu_read(stab_cache[i]);
ste->esid_data = 0; /* invalidate entry */
}
} else {
@@ -206,7 +206,7 @@ void switch_stab(struct task_struct *tsk
asm volatile("sync; slbia; sync":::"memory");
- __get_cpu_var(stab_cache_ptr) = 0;
+ __this_cpu_write(stab_cache_ptr, 0);
/* Now preload some entries for the new task */
if (test_tsk_thread_flag(tsk, TIF_32BIT))
Index: linux/arch/powerpc/perf/core-book3s.c
===================================================================
--- linux.orig/arch/powerpc/perf/core-book3s.c
+++ linux/arch/powerpc/perf/core-book3s.c
@@ -334,7 +334,7 @@ static void power_pmu_bhrb_reset(void)
static void power_pmu_bhrb_enable(struct perf_event *event)
{
- struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+ struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
if (!ppmu->bhrb_nr)
return;
@@ -349,7 +349,7 @@ static void power_pmu_bhrb_enable(struct
static void power_pmu_bhrb_disable(struct perf_event *event)
{
- struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+ struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
if (!ppmu->bhrb_nr)
return;
@@ -1106,7 +1106,7 @@ static void power_pmu_disable(struct pmu
if (!ppmu)
return;
local_irq_save(flags);
- cpuhw = &__get_cpu_var(cpu_hw_events);
+ cpuhw = this_cpu_ptr(&cpu_hw_events);
if (!cpuhw->disabled) {
/*
@@ -1173,7 +1173,7 @@ static void power_pmu_enable(struct pmu
return;
local_irq_save(flags);
- cpuhw = &__get_cpu_var(cpu_hw_events);
+ cpuhw = this_cpu_ptr(&cpu_hw_events);
if (!cpuhw->disabled)
goto out;
@@ -1360,7 +1360,7 @@ static int power_pmu_add(struct perf_eve
* Add the event to the list (if there is room)
* and check whether the total set is still feasible.
*/
- cpuhw = &__get_cpu_var(cpu_hw_events);
+ cpuhw = this_cpu_ptr(&cpu_hw_events);
n0 = cpuhw->n_events;
if (n0 >= ppmu->n_counter)
goto out;
@@ -1426,7 +1426,7 @@ static void power_pmu_del(struct perf_ev
power_pmu_read(event);
- cpuhw = &__get_cpu_var(cpu_hw_events);
+ cpuhw = this_cpu_ptr(&cpu_hw_events);
for (i = 0; i < cpuhw->n_events; ++i) {
if (event == cpuhw->event[i]) {
while (++i < cpuhw->n_events) {
@@ -1532,7 +1532,7 @@ static void power_pmu_stop(struct perf_e
*/
void power_pmu_start_txn(struct pmu *pmu)
{
- struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+ struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
perf_pmu_disable(pmu);
cpuhw->group_flag |= PERF_EVENT_TXN;
@@ -1546,7 +1546,7 @@ void power_pmu_start_txn(struct pmu *pmu
*/
void power_pmu_cancel_txn(struct pmu *pmu)
{
- struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+ struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
cpuhw->group_flag &= ~PERF_EVENT_TXN;
perf_pmu_enable(pmu);
@@ -1564,7 +1564,7 @@ int power_pmu_commit_txn(struct pmu *pmu
if (!ppmu)
return -EAGAIN;
- cpuhw = &__get_cpu_var(cpu_hw_events);
+ cpuhw = this_cpu_ptr(&cpu_hw_events);
n = cpuhw->n_events;
if (check_excludes(cpuhw->event, cpuhw->flags, 0, n))
return -EAGAIN;
@@ -1921,7 +1921,7 @@ static void record_and_restart(struct pe
if (event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK) {
struct cpu_hw_events *cpuhw;
- cpuhw = &__get_cpu_var(cpu_hw_events);
+ cpuhw = this_cpu_ptr(&cpu_hw_events);
power_pmu_bhrb_read(cpuhw);
data.br_stack = &cpuhw->bhrb_stack;
}
@@ -1994,7 +1994,7 @@ static bool pmc_overflow(unsigned long v
static void perf_event_interrupt(struct pt_regs *regs)
{
int i, j;
- struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+ struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
struct perf_event *event;
unsigned long val[8];
int found, active;
Index: linux/arch/powerpc/perf/core-fsl-emb.c
===================================================================
--- linux.orig/arch/powerpc/perf/core-fsl-emb.c
+++ linux/arch/powerpc/perf/core-fsl-emb.c
@@ -210,7 +210,7 @@ static void fsl_emb_pmu_disable(struct p
unsigned long flags;
local_irq_save(flags);
- cpuhw = &__get_cpu_var(cpu_hw_events);
+ cpuhw = this_cpu_ptr(&cpu_hw_events);
if (!cpuhw->disabled) {
cpuhw->disabled = 1;
@@ -249,7 +249,7 @@ static void fsl_emb_pmu_enable(struct pm
unsigned long flags;
local_irq_save(flags);
- cpuhw = &__get_cpu_var(cpu_hw_events);
+ cpuhw = this_cpu_ptr(&cpu_hw_events);
if (!cpuhw->disabled)
goto out;
@@ -653,7 +653,7 @@ static void record_and_restart(struct pe
static void perf_event_interrupt(struct pt_regs *regs)
{
int i;
- struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+ struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
struct perf_event *event;
unsigned long val;
int found = 0;
Index: linux/arch/powerpc/platforms/cell/interrupt.c
===================================================================
--- linux.orig/arch/powerpc/platforms/cell/interrupt.c
+++ linux/arch/powerpc/platforms/cell/interrupt.c
@@ -82,7 +82,7 @@ static void iic_unmask(struct irq_data *
static void iic_eoi(struct irq_data *d)
{
- struct iic *iic = &__get_cpu_var(cpu_iic);
+ struct iic *iic = this_cpu_ptr(&cpu_iic);
out_be64(&iic->regs->prio, iic->eoi_stack[--iic->eoi_ptr]);
BUG_ON(iic->eoi_ptr < 0);
}
@@ -148,7 +148,7 @@ static unsigned int iic_get_irq(void)
struct iic *iic;
unsigned int virq;
- iic = &__get_cpu_var(cpu_iic);
+ iic = this_cpu_ptr(&cpu_iic);
*(unsigned long *) &pending =
in_be64((u64 __iomem *) &iic->regs->pending_destr);
if (!(pending.flags & CBE_IIC_IRQ_VALID))
@@ -163,7 +163,7 @@ static unsigned int iic_get_irq(void)
void iic_setup_cpu(void)
{
- out_be64(&__get_cpu_var(cpu_iic).regs->prio, 0xff);
+ out_be64(this_cpu_ptr(&cpu_iic.regs->prio), 0xff);
}
u8 iic_get_target_id(int cpu)
Index: linux/arch/powerpc/platforms/ps3/interrupt.c
===================================================================
--- linux.orig/arch/powerpc/platforms/ps3/interrupt.c
+++ linux/arch/powerpc/platforms/ps3/interrupt.c
@@ -711,7 +711,7 @@ void __init ps3_register_ipi_irq(unsigne
static unsigned int ps3_get_irq(void)
{
- struct ps3_private *pd = &__get_cpu_var(ps3_private);
+ struct ps3_private *pd = this_cpu_ptr(&ps3_private);
u64 x = (pd->bmp.status & pd->bmp.mask);
unsigned int plug;
Index: linux/arch/powerpc/platforms/pseries/dtl.c
===================================================================
--- linux.orig/arch/powerpc/platforms/pseries/dtl.c
+++ linux/arch/powerpc/platforms/pseries/dtl.c
@@ -74,7 +74,7 @@ static atomic_t dtl_count;
*/
static void consume_dtle(struct dtl_entry *dtle, u64 index)
{
- struct dtl_ring *dtlr = &__get_cpu_var(dtl_rings);
+ struct dtl_ring *dtlr = this_cpu_ptr(&dtl_rings);
struct dtl_entry *wp = dtlr->write_ptr;
struct lppaca *vpa = local_paca->lppaca_ptr;
Index: linux/arch/powerpc/platforms/pseries/hvCall_inst.c
===================================================================
--- linux.orig/arch/powerpc/platforms/pseries/hvCall_inst.c
+++ linux/arch/powerpc/platforms/pseries/hvCall_inst.c
@@ -109,7 +109,7 @@ static void probe_hcall_entry(void *igno
if (opcode > MAX_HCALL_OPCODE)
return;
- h = &__get_cpu_var(hcall_stats)[opcode / 4];
+ h = this_cpu_ptr(&hcall_stats[opcode / 4]);
h->tb_start = mftb();
h->purr_start = mfspr(SPRN_PURR);
}
@@ -122,7 +122,7 @@ static void probe_hcall_exit(void *ignor
if (opcode > MAX_HCALL_OPCODE)
return;
- h = &__get_cpu_var(hcall_stats)[opcode / 4];
+ h = this_cpu_ptr(&hcall_stats[opcode / 4]);
h->num_calls++;
h->tb_total += mftb() - h->tb_start;
h->purr_total += mfspr(SPRN_PURR) - h->purr_start;
Index: linux/arch/powerpc/platforms/pseries/iommu.c
===================================================================
--- linux.orig/arch/powerpc/platforms/pseries/iommu.c
+++ linux/arch/powerpc/platforms/pseries/iommu.c
@@ -200,7 +200,7 @@ static int tce_buildmulti_pSeriesLP(stru
local_irq_save(flags); /* to protect tcep and the page behind it */
- tcep = __get_cpu_var(tce_page);
+ tcep = __this_cpu_read(tce_page);
/* This is safe to do since interrupts are off when we're called
* from iommu_alloc{,_sg}()
@@ -213,7 +213,7 @@ static int tce_buildmulti_pSeriesLP(stru
return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
direction, attrs);
}
- __get_cpu_var(tce_page) = tcep;
+ __this_cpu_write(tce_page, tcep);
}
rpn = __pa(uaddr) >> TCE_SHIFT;
@@ -399,7 +399,7 @@ static int tce_setrange_multi_pSeriesLP(
long l, limit;
local_irq_disable(); /* to protect tcep and the page behind it */
- tcep = __get_cpu_var(tce_page);
+ tcep = __this_cpu_read(tce_page);
if (!tcep) {
tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
@@ -407,7 +407,7 @@ static int tce_setrange_multi_pSeriesLP(
local_irq_enable();
return -ENOMEM;
}
- __get_cpu_var(tce_page) = tcep;
+ __this_cpu_write(tce_page, tcep);
}
proto_tce = TCE_PCI_READ | TCE_PCI_WRITE;
Index: linux/arch/powerpc/platforms/pseries/lpar.c
===================================================================
--- linux.orig/arch/powerpc/platforms/pseries/lpar.c
+++ linux/arch/powerpc/platforms/pseries/lpar.c
@@ -514,7 +514,7 @@ static void pSeries_lpar_flush_hash_rang
unsigned long vpn;
unsigned long i, pix, rc;
unsigned long flags = 0;
- struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
+ struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
unsigned long param[9];
unsigned long hash, index, shift, hidx, slot;
@@ -689,7 +689,7 @@ void __trace_hcall_entry(unsigned long o
local_irq_save(flags);
- depth = &__get_cpu_var(hcall_trace_depth);
+ depth = this_cpu_ptr(&hcall_trace_depth);
if (*depth)
goto out;
@@ -714,7 +714,7 @@ void __trace_hcall_exit(long opcode, uns
local_irq_save(flags);
- depth = &__get_cpu_var(hcall_trace_depth);
+ depth = this_cpu_ptr(&hcall_trace_depth);
if (*depth)
goto out;
Index: linux/arch/powerpc/platforms/pseries/ras.c
===================================================================
--- linux.orig/arch/powerpc/platforms/pseries/ras.c
+++ linux/arch/powerpc/platforms/pseries/ras.c
@@ -302,8 +302,8 @@ static struct rtas_error_log *fwnmi_get_
/* If it isn't an extended log we can use the per cpu 64bit buffer */
h = (struct rtas_error_log *)&savep[1];
if (!rtas_error_extended(h)) {
- memcpy(&__get_cpu_var(mce_data_buf), h, sizeof(__u64));
- errhdr = (struct rtas_error_log *)&__get_cpu_var(mce_data_buf);
+ memcpy(this_cpu_ptr(&mce_data_buf), h, sizeof(__u64));
+ errhdr = (struct rtas_error_log *)this_cpu_ptr(&mce_data_buf);
} else {
int len, error_log_length;
Index: linux/arch/powerpc/sysdev/xics/xics-common.c
===================================================================
--- linux.orig/arch/powerpc/sysdev/xics/xics-common.c
+++ linux/arch/powerpc/sysdev/xics/xics-common.c
@@ -155,7 +155,7 @@ int __init xics_smp_probe(void)
void xics_teardown_cpu(void)
{
- struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr);
+ struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr);
/*
* we have to reset the cppr index to 0 because we're
Index: linux/arch/powerpc/kernel/iommu.c
===================================================================
--- linux.orig/arch/powerpc/kernel/iommu.c
+++ linux/arch/powerpc/kernel/iommu.c
@@ -208,7 +208,7 @@ static unsigned long iommu_range_alloc(s
* We don't need to disable preemption here because any CPU can
* safely use any IOMMU pool.
*/
- pool_nr = __raw_get_cpu_var(iommu_pool_hash) & (tbl->nr_pools - 1);
+ pool_nr = __this_cpu_read(iommu_pool_hash) & (tbl->nr_pools - 1);
if (largealloc)
pool = &(tbl->large_pool);
Index: linux/arch/powerpc/kernel/kgdb.c
===================================================================
--- linux.orig/arch/powerpc/kernel/kgdb.c
+++ linux/arch/powerpc/kernel/kgdb.c
@@ -155,7 +155,7 @@ static int kgdb_singlestep(struct pt_reg
{
struct thread_info *thread_info, *exception_thread_info;
struct thread_info *backup_current_thread_info =
- &__get_cpu_var(kgdb_thread_info);
+ this_cpu_ptr(&kgdb_thread_info);
if (user_mode(regs))
return 0;
Index: linux/arch/powerpc/kernel/mce.c
===================================================================
--- linux.orig/arch/powerpc/kernel/mce.c
+++ linux/arch/powerpc/kernel/mce.c
@@ -73,8 +73,8 @@ void save_mce_event(struct pt_regs *regs
uint64_t nip, uint64_t addr)
{
uint64_t srr1;
- int index = __get_cpu_var(mce_nest_count)++;
- struct machine_check_event *mce = &__get_cpu_var(mce_event[index]);
+ int index = __this_cpu_inc_return(mce_nest_count);
+ struct machine_check_event *mce = this_cpu_ptr(&mce_event[index]);
/*
* Return if we don't have enough space to log mce event.
@@ -143,7 +143,7 @@ void save_mce_event(struct pt_regs *regs
*/
int get_mce_event(struct machine_check_event *mce, bool release)
{
- int index = __get_cpu_var(mce_nest_count) - 1;
+ int index = __this_cpu_read(mce_nest_count) - 1;
struct machine_check_event *mc_evt;
int ret = 0;
@@ -153,7 +153,7 @@ int get_mce_event(struct machine_check_e
/* Check if we have MCE info to process. */
if (index < MAX_MC_EVT) {
- mc_evt = &__get_cpu_var(mce_event[index]);
+ mc_evt = this_cpu_ptr(&mce_event[index]);
/* Copy the event structure and release the original */
if (mce)
*mce = *mc_evt;
@@ -163,7 +163,7 @@ int get_mce_event(struct machine_check_e
}
/* Decrement the count to free the slot. */
if (release)
- __get_cpu_var(mce_nest_count)--;
+ __this_cpu_dec(mce_nest_count);
return ret;
}
@@ -184,13 +184,13 @@ void machine_check_queue_event(void)
if (!get_mce_event(&evt, MCE_EVENT_RELEASE))
return;
- index = __get_cpu_var(mce_queue_count)++;
+ index = __this_cpu_inc_return(mce_queue_count);
/* If queue is full, just return for now. */
if (index >= MAX_MC_EVT) {
- __get_cpu_var(mce_queue_count)--;
+ __this_cpu_dec(mce_queue_count);
return;
}
- __get_cpu_var(mce_event_queue[index]) = evt;
+ memcpy(this_cpu_ptr(&mce_event_queue[index]), &evt, sizeof(evt));
/* Queue irq work to process this event later. */
irq_work_queue(&mce_event_process_work);
@@ -208,11 +208,11 @@ static void machine_check_process_queued
* For now just print it to console.
* TODO: log this error event to FSP or nvram.
*/
- while (__get_cpu_var(mce_queue_count) > 0) {
- index = __get_cpu_var(mce_queue_count) - 1;
+ while (__this_cpu_read(mce_queue_count) > 0) {
+ index = __this_cpu_read(mce_queue_count) - 1;
machine_check_print_event_info(
- &__get_cpu_var(mce_event_queue[index]));
- __get_cpu_var(mce_queue_count)--;
+ this_cpu_ptr(&mce_event_queue[index]));
+ __this_cpu_dec(mce_queue_count);
}
}
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/