[PATCH 4/5] mm: convert mm_struct.mm_users from atomic_t to refcount_t

From: Elena Reshetova
Date: Mon Feb 20 2017 - 05:50:11 EST

refcount_t type and corresponding API should be
used instead of atomic_t when the variable is used as
a reference counter. This allows to avoid accidental
refcounter overflows that might lead to use-after-free
situations.

Signed-off-by: Elena Reshetova <elena.reshetova@xxxxxxxxx>
Signed-off-by: Hans Liljestrand <ishkamiel@xxxxxxxxx>
Signed-off-by: Kees Cook <keescook@xxxxxxxxxxxx>
Signed-off-by: David Windsor <dwindsor@xxxxxxxxx>
---
arch/alpha/kernel/smp.c | 6 +++---
arch/arc/mm/tlb.c | 2 +-
arch/blackfin/mach-common/smp.c | 2 +-
arch/ia64/include/asm/tlbflush.h | 2 +-
arch/ia64/kernel/smp.c | 2 +-
arch/ia64/sn/kernel/sn2/sn2_smp.c | 4 ++--
arch/mips/kernel/process.c | 2 +-
arch/mips/kernel/smp.c | 6 +++---
arch/parisc/include/asm/mmu_context.h | 2 +-
arch/powerpc/mm/hugetlbpage.c | 2 +-
arch/powerpc/mm/icswx.c | 4 ++--
arch/sh/kernel/smp.c | 6 +++---
arch/sparc/kernel/smp_64.c | 6 +++---
arch/sparc/mm/srmmu.c | 2 +-
arch/um/kernel/tlb.c | 2 +-
arch/x86/kernel/tboot.c | 2 +-
arch/xtensa/kernel/smp.c | 5 +++++
drivers/firmware/efi/arm-runtime.c | 2 +-
fs/coredump.c | 2 +-
fs/proc/base.c | 10 +++++++++-
include/linux/mm_types.h | 3 ++-
include/linux/sched.h | 4 ++--
kernel/events/uprobes.c | 2 +-
kernel/exit.c | 2 +-
kernel/fork.c | 10 +++++-----
kernel/sched/core.c | 2 +-
lib/is_single_threaded.c | 2 +-
mm/debug.c | 2 +-
mm/init-mm.c | 2 +-
mm/khugepaged.c | 2 +-
mm/ksm.c | 2 +-
mm/memory.c | 2 +-
mm/mmu_notifier.c | 4 ++--
mm/mprotect.c | 2 +-
mm/oom_kill.c | 2 +-
mm/swapfile.c | 2 +-
mm/vmacache.c | 2 +-
37 files changed, 66 insertions(+), 52 deletions(-)

diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c
index acb4b14..c4a82f0 100644
--- a/arch/alpha/kernel/smp.c
+++ b/arch/alpha/kernel/smp.c
@@ -653,7 +653,7 @@ flush_tlb_mm(struct mm_struct *mm)

if (mm == current->active_mm) {
flush_tlb_current(mm);
- if (atomic_read(&mm->mm_users) <= 1) {
+ if (refcount_read(&mm->mm_users) <= 1) {
int cpu, this_cpu = smp_processor_id();
for (cpu = 0; cpu < NR_CPUS; cpu++) {
if (!cpu_online(cpu) || cpu == this_cpu)
@@ -702,7 +702,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)

if (mm == current->active_mm) {
flush_tlb_current_page(mm, vma, addr);
- if (atomic_read(&mm->mm_users) <= 1) {
+ if (refcount_read(&mm->mm_users) <= 1) {
int cpu, this_cpu = smp_processor_id();
for (cpu = 0; cpu < NR_CPUS; cpu++) {
if (!cpu_online(cpu) || cpu == this_cpu)
@@ -758,7 +758,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,

if (mm == current->active_mm) {
__load_new_mm_context(mm);
- if (atomic_read(&mm->mm_users) <= 1) {
+ if (refcount_read(&mm->mm_users) <= 1) {
int cpu, this_cpu = smp_processor_id();
for (cpu = 0; cpu < NR_CPUS; cpu++) {
if (!cpu_online(cpu) || cpu == this_cpu)
diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c
index bdb295e..6dbdfe7 100644
--- a/arch/arc/mm/tlb.c
+++ b/arch/arc/mm/tlb.c
@@ -297,7 +297,7 @@ noinline void local_flush_tlb_mm(struct mm_struct *mm)
* Only for fork( ) do we need to move parent to a new MMU ctxt,
* all other cases are NOPs, hence this check.
*/
- if (atomic_read(&mm->mm_users) == 0)
+ if (refcount_read(&mm->mm_users) == 0)
return;

/*
diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c
index a2e6db2..bab73d2b 100644
--- a/arch/blackfin/mach-common/smp.c
+++ b/arch/blackfin/mach-common/smp.c
@@ -422,7 +422,7 @@ void cpu_die(void)
{
(void)cpu_report_death();

- atomic_dec(&init_mm.mm_users);
+ refcount_dec(&init_mm.mm_users);
atomic_dec(&init_mm.mm_count);

local_irq_disable();
diff --git a/arch/ia64/include/asm/tlbflush.h b/arch/ia64/include/asm/tlbflush.h
index 3be25df..650708a 100644
--- a/arch/ia64/include/asm/tlbflush.h
+++ b/arch/ia64/include/asm/tlbflush.h
@@ -56,7 +56,7 @@ flush_tlb_mm (struct mm_struct *mm)
set_bit(mm->context, ia64_ctx.flushmap);
mm->context = 0;

- if (atomic_read(&mm->mm_users) == 0)
+ if (refcount_read(&mm->mm_users) == 0)
return; /* happens as a result of exit_mmap() */

#ifdef CONFIG_SMP
diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c
index 7f706d4..dd7b680 100644
--- a/arch/ia64/kernel/smp.c
+++ b/arch/ia64/kernel/smp.c
@@ -295,7 +295,7 @@ smp_flush_tlb_mm (struct mm_struct *mm)
cpumask_var_t cpus;
preempt_disable();
/* this happens for the common case of a single-threaded fork(): */
- if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
+ if (likely(mm == current->active_mm && refcount_read(&mm->mm_users) == 1))
{
local_finish_flush_tlb_mm(mm);
preempt_enable();
diff --git a/arch/ia64/sn/kernel/sn2/sn2_smp.c b/arch/ia64/sn/kernel/sn2/sn2_smp.c
index c98dc96..1c801b3 100644
--- a/arch/ia64/sn/kernel/sn2/sn2_smp.c
+++ b/arch/ia64/sn/kernel/sn2/sn2_smp.c
@@ -122,7 +122,7 @@ void sn_migrate(struct task_struct *task)
void sn_tlb_migrate_finish(struct mm_struct *mm)
{
/* flush_tlb_mm is inefficient if more than 1 users of mm */
- if (mm == current->mm && mm && atomic_read(&mm->mm_users) == 1)
+ if (mm == current->mm && mm && refcount_read(&mm->mm_users) == 1)
flush_tlb_mm(mm);
}

@@ -204,7 +204,7 @@ sn2_global_tlb_purge(struct mm_struct *mm, unsigned long start,
return;
}

- if (atomic_read(&mm->mm_users) == 1 && mymm) {
+ if (refcount_read(&mm->mm_users) == 1 && mymm) {
flush_tlb_mm(mm);
__this_cpu_inc(ptcstats.change_rid);
preempt_enable();
diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c
index 803e255..33fa000 100644
--- a/arch/mips/kernel/process.c
+++ b/arch/mips/kernel/process.c
@@ -698,7 +698,7 @@ int mips_set_process_fp_mode(struct task_struct *task, unsigned int value)
/* No need to send an IPI for the local CPU */
max_users = (task->mm == current->mm) ? 1 : 0;

- if (atomic_read(&current->mm->mm_users) > max_users)
+ if (refcount_read(&current->mm->mm_users) > max_users)
smp_call_function(prepare_for_fp_mode_switch,
(void *)current->mm, 1);
}
diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c
index 8c60a29..e62aa56 100644
--- a/arch/mips/kernel/smp.c
+++ b/arch/mips/kernel/smp.c
@@ -511,7 +511,7 @@ void flush_tlb_mm(struct mm_struct *mm)
{
preempt_disable();

- if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
+ if ((refcount_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_on_other_tlbs(flush_tlb_mm_ipi, mm);
} else {
unsigned int cpu;
@@ -544,7 +544,7 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l
struct mm_struct *mm = vma->vm_mm;

preempt_disable();
- if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
+ if ((refcount_read(&mm->mm_users) != 1) || (current->mm != mm)) {
struct flush_tlb_data fd = {
.vma = vma,
.addr1 = start,
@@ -598,7 +598,7 @@ static void flush_tlb_page_ipi(void *info)
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
preempt_disable();
- if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
+ if ((refcount_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
struct flush_tlb_data fd = {
.vma = vma,
.addr1 = page,
diff --git a/arch/parisc/include/asm/mmu_context.h b/arch/parisc/include/asm/mmu_context.h
index 59be257..e64f398 100644
--- a/arch/parisc/include/asm/mmu_context.h
+++ b/arch/parisc/include/asm/mmu_context.h
@@ -21,7 +21,7 @@ extern void free_sid(unsigned long);
static inline int
init_new_context(struct task_struct *tsk, struct mm_struct *mm)
{
- BUG_ON(atomic_read(&mm->mm_users) != 1);
+ BUG_ON(refcount_read(&mm->mm_users) != 1);

mm->context = alloc_sid();
return 0;
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 8c3389c..26aef24 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -403,7 +403,7 @@ static void hugepd_free(struct mmu_gather *tlb, void *hugepte)

batchp = &get_cpu_var(hugepd_freelist_cur);

- if (atomic_read(&tlb->mm->mm_users) < 2 ||
+ if (refcount_read(&tlb->mm->mm_users) < 2 ||
cpumask_equal(mm_cpumask(tlb->mm),
cpumask_of(smp_processor_id()))) {
kmem_cache_free(hugepte_cache, hugepte);
diff --git a/arch/powerpc/mm/icswx.c b/arch/powerpc/mm/icswx.c
index 915412e..2406ff8 100644
--- a/arch/powerpc/mm/icswx.c
+++ b/arch/powerpc/mm/icswx.c
@@ -110,7 +110,7 @@ int use_cop(unsigned long acop, struct mm_struct *mm)
* running. We need to send an IPI to force them to pick up any
* change in PID and ACOP.
*/
- if (atomic_read(&mm->mm_users) > 1)
+ if (refcount_read(&mm->mm_users) > 1)
smp_call_function(sync_cop, mm, 1);

out:
@@ -150,7 +150,7 @@ void drop_cop(unsigned long acop, struct mm_struct *mm)
* running. We need to send an IPI to force them to pick up any
* change in PID and ACOP.
*/
- if (atomic_read(&mm->mm_users) > 1)
+ if (refcount_read(&mm->mm_users) > 1)
smp_call_function(sync_cop, mm, 1);

if (free_pid != COP_PID_NONE)
diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c
index edc4769..9d678bb 100644
--- a/arch/sh/kernel/smp.c
+++ b/arch/sh/kernel/smp.c
@@ -363,7 +363,7 @@ void flush_tlb_mm(struct mm_struct *mm)
{
preempt_disable();

- if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
+ if ((refcount_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1);
} else {
int i;
@@ -395,7 +395,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
struct mm_struct *mm = vma->vm_mm;

preempt_disable();
- if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
+ if ((refcount_read(&mm->mm_users) != 1) || (current->mm != mm)) {
struct flush_tlb_data fd;

fd.vma = vma;
@@ -438,7 +438,7 @@ static void flush_tlb_page_ipi(void *info)
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
preempt_disable();
- if ((atomic_read(&vma->vm_mm->mm_users) != 1) ||
+ if ((refcount_read(&vma->vm_mm->mm_users) != 1) ||
(current->mm != vma->vm_mm)) {
struct flush_tlb_data fd;

diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index dcb12d9..bd5e56c 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -1063,7 +1063,7 @@ void smp_flush_tlb_mm(struct mm_struct *mm)
u32 ctx = CTX_HWBITS(mm->context);
int cpu = get_cpu();

- if (atomic_read(&mm->mm_users) == 1) {
+ if (refcount_read(&mm->mm_users) == 1) {
cpumask_copy(mm_cpumask(mm), cpumask_of(cpu));
goto local_flush_and_out;
}
@@ -1101,7 +1101,7 @@ void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long
info.nr = nr;
info.vaddrs = vaddrs;

- if (mm == current->mm && atomic_read(&mm->mm_users) == 1)
+ if (mm == current->mm && refcount_read(&mm->mm_users) == 1)
cpumask_copy(mm_cpumask(mm), cpumask_of(cpu));
else
smp_call_function_many(mm_cpumask(mm), tlb_pending_func,
@@ -1117,7 +1117,7 @@ void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr)
unsigned long context = CTX_HWBITS(mm->context);
int cpu = get_cpu();

- if (mm == current->mm && atomic_read(&mm->mm_users) == 1)
+ if (mm == current->mm && refcount_read(&mm->mm_users) == 1)
cpumask_copy(mm_cpumask(mm), cpumask_of(cpu));
else
smp_cross_call_masked(&xcall_flush_tlb_page,
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index c7f2a52..17941a8 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -1662,7 +1662,7 @@ static void smp_flush_tlb_mm(struct mm_struct *mm)
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
if (!cpumask_empty(&cpu_mask)) {
xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm);
- if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm)
+ if (refcount_read(&mm->mm_users) == 1 && current->active_mm == mm)
cpumask_copy(mm_cpumask(mm),
cpumask_of(smp_processor_id()));
}
diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c
index 3777b82..1da0463 100644
--- a/arch/um/kernel/tlb.c
+++ b/arch/um/kernel/tlb.c
@@ -530,7 +530,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
* Don't bother flushing if this address space is about to be
* destroyed.
*/
- if (atomic_read(&mm->mm_users) == 0)
+ if (refcount_read(&mm->mm_users) == 0)
return;

fix_range(mm, start, end, 0);
diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
index b868fa1..39aaca5 100644
--- a/arch/x86/kernel/tboot.c
+++ b/arch/x86/kernel/tboot.c
@@ -102,7 +102,7 @@ static pgd_t *tboot_pg_dir;
static struct mm_struct tboot_mm = {
.mm_rb = RB_ROOT,
.pgd = swapper_pg_dir,
- .mm_users = ATOMIC_INIT(2),
+ .mm_users = REFCOUNT_INIT(2),
.mm_count = ATOMIC_INIT(1),
.mmap_sem = __RWSEM_INITIALIZER(init_mm.mmap_sem),
.page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
diff --git a/arch/xtensa/kernel/smp.c b/arch/xtensa/kernel/smp.c
index fcea720..c2b3067 100644
--- a/arch/xtensa/kernel/smp.c
+++ b/arch/xtensa/kernel/smp.c
@@ -135,8 +135,13 @@ void secondary_start_kernel(void)

/* All kernel threads share the same mm context. */

+<<<<<<< e5c2d109eae88b7c203d396dfaeb284f95f6ffe5
mmget(mm);
mmgrab(mm);
+=======
+ refcount_inc(&mm->mm_users);
+ atomic_inc(&mm->mm_count);
+>>>>>>> mm: convert mm_struct.mm_users from atomic_t to refcount_t
current->active_mm = mm;
cpumask_set_cpu(cpu, mm_cpumask(mm));
enter_lazy_tlb(mm, current);
diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c
index 349dc3e..23e41f9 100644
--- a/drivers/firmware/efi/arm-runtime.c
+++ b/drivers/firmware/efi/arm-runtime.c
@@ -32,7 +32,7 @@ extern u64 efi_system_table;

static struct mm_struct efi_mm = {
.mm_rb = RB_ROOT,
- .mm_users = ATOMIC_INIT(2),
+ .mm_users = REFCOUNT_INIT(2),
.mm_count = ATOMIC_INIT(1),
.mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem),
.page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock),
diff --git a/fs/coredump.c b/fs/coredump.c
index ae6b056..05e43f1 100644
--- a/fs/coredump.c
+++ b/fs/coredump.c
@@ -347,7 +347,7 @@ static int zap_threads(struct task_struct *tsk, struct mm_struct *mm,
return nr;

tsk->flags |= PF_DUMPCORE;
- if (atomic_read(&mm->mm_users) == nr + 1)
+ if (refcount_read(&mm->mm_users) == nr + 1)
goto done;
/*
* We should find and kill all tasks which use this mm, and we should
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 6e86558..a75dca3 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -921,7 +921,11 @@ static ssize_t environ_read(struct file *file, char __user *buf,
return -ENOMEM;

ret = 0;
+<<<<<<< e5c2d109eae88b7c203d396dfaeb284f95f6ffe5
if (!mmget_not_zero(mm))
+=======
+ if (!refcount_inc_not_zero(&mm->mm_users))
+>>>>>>> mm: convert mm_struct.mm_users from atomic_t to refcount_t
goto free;

down_read(&mm->mmap_sem);
@@ -1062,9 +1066,13 @@ static int __set_oom_adj(struct file *file, int oom_adj, bool legacy)
struct task_struct *p = find_lock_task_mm(task);

if (p) {
- if (atomic_read(&p->mm->mm_users) > 1) {
+ if (refcount_read(&p->mm->mm_users) > 1) {
mm = p->mm;
+<<<<<<< e5c2d109eae88b7c203d396dfaeb284f95f6ffe5
mmgrab(mm);
+=======
+ refcount_inc(&mm->mm_count);
+>>>>>>> mm: convert mm_struct.mm_users from atomic_t to refcount_t
}
task_unlock(p);
}
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 4f6d440..af260d6 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -7,6 +7,7 @@
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/rbtree.h>
+#include <linux/refcount.h>
#include <linux/rwsem.h>
#include <linux/completion.h>
#include <linux/cpumask.h>
@@ -417,7 +418,7 @@ struct mm_struct {
* (which may then free the &struct mm_struct if @mm_count also
* drops to 0).
*/
- atomic_t mm_users;
+ refcount_t mm_users;

/**
* @mm_count: The number of references to &struct mm_struct
diff --git a/include/linux/sched.h b/include/linux/sched.h
index affcd93..c21682c 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2967,12 +2967,12 @@ static inline void mmdrop_async(struct mm_struct *mm)
*/
static inline void mmget(struct mm_struct *mm)
{
- atomic_inc(&mm->mm_users);
+ refcount_inc(&mm->mm_users);
}

static inline bool mmget_not_zero(struct mm_struct *mm)
{
- return atomic_inc_not_zero(&mm->mm_users);
+ return refcount_inc_not_zero(&mm->mm_users);
}

/* mmput gets rid of the mappings and all user-space */
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 95e42f0..a4b33f8 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -1121,7 +1121,7 @@ void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned lon
if (no_uprobe_events() || !valid_vma(vma, false))
return;

- if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
+ if (!refcount_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
return;

if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
diff --git a/kernel/exit.c b/kernel/exit.c
index 8a768a3..261305d 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -426,7 +426,7 @@ void mm_update_next_owner(struct mm_struct *mm)
* candidates. Do not leave the mm pointing to a possibly
* freed task structure.
*/
- if (atomic_read(&mm->mm_users) <= 1) {
+ if (refcount_read(&mm->mm_users) <= 1) {
mm->owner = NULL;
return;
}
diff --git a/kernel/fork.c b/kernel/fork.c
index 0e096fc..60ff801 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -759,7 +759,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
mm->mmap = NULL;
mm->mm_rb = RB_ROOT;
mm->vmacache_seqnum = 0;
- atomic_set(&mm->mm_users, 1);
+ refcount_set(&mm->mm_users, 1);
atomic_set(&mm->mm_count, 1);
init_rwsem(&mm->mmap_sem);
INIT_LIST_HEAD(&mm->mmlist);
@@ -862,7 +862,7 @@ EXPORT_SYMBOL_GPL(__mmdrop);

static inline void __mmput(struct mm_struct *mm)
{
- VM_BUG_ON(atomic_read(&mm->mm_users));
+ VM_BUG_ON(refcount_read(&mm->mm_users));

uprobe_clear_state(mm);
exit_aio(mm);
@@ -889,7 +889,7 @@ void mmput(struct mm_struct *mm)
{
might_sleep();

- if (atomic_dec_and_test(&mm->mm_users))
+ if (refcount_dec_and_test(&mm->mm_users))
__mmput(mm);
}
EXPORT_SYMBOL_GPL(mmput);
@@ -903,7 +903,7 @@ static void mmput_async_fn(struct work_struct *work)

void mmput_async(struct mm_struct *mm)
{
- if (atomic_dec_and_test(&mm->mm_users)) {
+ if (refcount_dec_and_test(&mm->mm_users)) {
INIT_WORK(&mm->async_put_work, mmput_async_fn);
schedule_work(&mm->async_put_work);
}
@@ -1102,7 +1102,7 @@ void mm_release(struct task_struct *tsk, struct mm_struct *mm)
*/
if (tsk->clear_child_tid) {
if (!(tsk->signal->flags & SIGNAL_GROUP_COREDUMP) &&
- atomic_read(&mm->mm_users) > 1) {
+ refcount_read(&mm->mm_users) > 1) {
/*
* We don't check the error code - if userspace has
* not set up a proper pointer then tough luck.
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index c7ded33..324ea09 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2196,7 +2196,7 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
#endif

#ifdef CONFIG_NUMA_BALANCING
- if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
+ if (p->mm && refcount_read(&p->mm->mm_users) == 1) {
p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
p->mm->numa_scan_seq = 0;
}
diff --git a/lib/is_single_threaded.c b/lib/is_single_threaded.c
index 391fd23..295ddcf 100644
--- a/lib/is_single_threaded.c
+++ b/lib/is_single_threaded.c
@@ -25,7 +25,7 @@ bool current_is_single_threaded(void)
if (atomic_read(&task->signal->live) != 1)
return false;

- if (atomic_read(&mm->mm_users) == 1)
+ if (refcount_read(&mm->mm_users) == 1)
return true;

ret = false;
diff --git a/mm/debug.c b/mm/debug.c
index db1cd26..0866505 100644
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -134,7 +134,7 @@ void dump_mm(const struct mm_struct *mm)
mm->get_unmapped_area,
#endif
mm->mmap_base, mm->mmap_legacy_base, mm->highest_vm_end,
- mm->pgd, atomic_read(&mm->mm_users),
+ mm->pgd, refcount_read(&mm->mm_users),
atomic_read(&mm->mm_count),
atomic_long_read((atomic_long_t *)&mm->nr_ptes),
mm_nr_pmds((struct mm_struct *)mm),
diff --git a/mm/init-mm.c b/mm/init-mm.c
index 975e49f..6927a72 100644
--- a/mm/init-mm.c
+++ b/mm/init-mm.c
@@ -17,7 +17,7 @@
struct mm_struct init_mm = {
.mm_rb = RB_ROOT,
.pgd = swapper_pg_dir,
- .mm_users = ATOMIC_INIT(2),
+ .mm_users = REFCOUNT_INIT(2),
.mm_count = ATOMIC_INIT(1),
.mmap_sem = __RWSEM_INITIALIZER(init_mm.mmap_sem),
.page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 34bce5c..e7c11a6 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -391,7 +391,7 @@ static void insert_to_mm_slots_hash(struct mm_struct *mm,

static inline int khugepaged_test_exit(struct mm_struct *mm)
{
- return atomic_read(&mm->mm_users) == 0;
+ return refcount_read(&mm->mm_users) == 0;
}

int __khugepaged_enter(struct mm_struct *mm)
diff --git a/mm/ksm.c b/mm/ksm.c
index 2e129f0..6152465 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -358,7 +358,7 @@ static void insert_to_mm_slots_hash(struct mm_struct *mm,
*/
static inline bool ksm_test_exit(struct mm_struct *mm)
{
- return atomic_read(&mm->mm_users) == 0;
+ return refcount_read(&mm->mm_users) == 0;
}

/*
diff --git a/mm/memory.c b/mm/memory.c
index 0c759ba..cc93a22 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -374,7 +374,7 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table)
* When there's less then two users of this mm there cannot be a
* concurrent page-table walk.
*/
- if (atomic_read(&tlb->mm->mm_users) < 2) {
+ if (refcount_read(&tlb->mm->mm_users) < 2) {
__tlb_remove_table(table);
return;
}
diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
index 32bc9f2..d8c11e1 100644
--- a/mm/mmu_notifier.c
+++ b/mm/mmu_notifier.c
@@ -249,7 +249,7 @@ static int do_mmu_notifier_register(struct mmu_notifier *mn,
struct mmu_notifier_mm *mmu_notifier_mm;
int ret;

- BUG_ON(atomic_read(&mm->mm_users) <= 0);
+ BUG_ON(refcount_read(&mm->mm_users) <= 0);

/*
* Verify that mmu_notifier_init() already run and the global srcu is
@@ -295,7 +295,7 @@ static int do_mmu_notifier_register(struct mmu_notifier *mn,
up_write(&mm->mmap_sem);
kfree(mmu_notifier_mm);
out:
- BUG_ON(atomic_read(&mm->mm_users) <= 0);
+ BUG_ON(refcount_read(&mm->mm_users) <= 0);
return ret;
}

diff --git a/mm/mprotect.c b/mm/mprotect.c
index 77115bb..f482ce9 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -63,7 +63,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,

/* Get target node for single threaded private VMAs */
if (prot_numa && !(vma->vm_flags & VM_SHARED) &&
- atomic_read(&vma->vm_mm->mm_users) == 1)
+ refcount_read(&vma->vm_mm->mm_users) == 1)
target_node = numa_node_id();

arch_enter_lazy_mmu_mode();
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 51c0918..cc0348d 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -774,7 +774,7 @@ static bool task_will_free_mem(struct task_struct *task)
if (test_bit(MMF_OOM_SKIP, &mm->flags))
return false;

- if (atomic_read(&mm->mm_users) <= 1)
+ if (refcount_read(&mm->mm_users) <= 1)
return true;

/*
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 5ac2cb4..be949e4 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1722,7 +1722,7 @@ int try_to_unuse(unsigned int type, bool frontswap,
/*
* Don't hold on to start_mm if it looks like exiting.
*/
- if (atomic_read(&start_mm->mm_users) == 1) {
+ if (refcount_read(&start_mm->mm_users) == 1) {
mmput(start_mm);
start_mm = &init_mm;
mmget(&init_mm);
diff --git a/mm/vmacache.c b/mm/vmacache.c
index 035fdeb..4747ee6 100644
--- a/mm/vmacache.c
+++ b/mm/vmacache.c
@@ -26,7 +26,7 @@ void vmacache_flush_all(struct mm_struct *mm)
* to worry about other threads' seqnum. Current's
* flush will occur upon the next lookup.
*/
- if (atomic_read(&mm->mm_users) == 1)
+ if (refcount_read(&mm->mm_users) == 1)
return;

rcu_read_lock();
--
2.7.4