Re: [PATCH 01/36] mmu_notifier: add event information to address invalidation v7
From: John Hubbard
Date: Fri May 29 2015 - 23:44:44 EST
On Thu, 21 May 2015, j.glisse@xxxxxxxxx wrote:
> From: JÃrÃme Glisse <jglisse@xxxxxxxxxx>
>
> The event information will be useful for new user of mmu_notifier API.
> The event argument differentiate between a vma disappearing, a page
> being write protected or simply a page being unmaped. This allow new
> user to take different path for different event for instance on unmap
> the resource used to track a vma are still valid and should stay around.
> While if the event is saying that a vma is being destroy it means that any
> resources used to track this vma can be free.
>
> Changed since v1:
> - renamed action into event (updated commit message too).
> - simplified the event names and clarified their usage
> also documenting what exceptation the listener can have in
> respect to each event.
>
> Changed since v2:
> - Avoid crazy name.
> - Do not move code that do not need to move.
>
> Changed since v3:
> - Separate hugue page split from mlock/munlock and softdirty.
Do we care about fixing up patch comments? If so:
s/hugue/huge/
>
> Changed since v4:
> - Rebase (no other changes).
>
> Changed since v5:
> - Typo fix.
> - Changed zap_page_range from MMU_MUNMAP to MMU_MIGRATE to reflect the
> fact that the address range is still valid just the page backing it
> are no longer.
>
> Changed since v6:
> - try_to_unmap_one() only invalidate when doing migration.
> - Differentiate fork from other case.
>
> Signed-off-by: JÃrÃme Glisse <jglisse@xxxxxxxxxx>
> Reviewed-by: Rik van Riel <riel@xxxxxxxxxx>
> ---
> drivers/gpu/drm/i915/i915_gem_userptr.c | 3 +-
> drivers/gpu/drm/radeon/radeon_mn.c | 3 +-
> drivers/infiniband/core/umem_odp.c | 9 ++-
> drivers/iommu/amd_iommu_v2.c | 3 +-
> drivers/misc/sgi-gru/grutlbpurge.c | 9 ++-
> drivers/xen/gntdev.c | 9 ++-
> fs/proc/task_mmu.c | 6 +-
> include/linux/mmu_notifier.h | 135 ++++++++++++++++++++++++++------
> kernel/events/uprobes.c | 10 ++-
> mm/huge_memory.c | 39 ++++++---
> mm/hugetlb.c | 23 +++---
> mm/ksm.c | 18 +++--
> mm/madvise.c | 4 +-
> mm/memory.c | 27 ++++---
> mm/migrate.c | 9 ++-
> mm/mmu_notifier.c | 28 ++++---
> mm/mprotect.c | 6 +-
> mm/mremap.c | 6 +-
> mm/rmap.c | 4 +-
> virt/kvm/kvm_main.c | 12 ++-
> 20 files changed, 261 insertions(+), 102 deletions(-)
>
> diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
> index 4039ede..452e9b1 100644
> --- a/drivers/gpu/drm/i915/i915_gem_userptr.c
> +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
> @@ -132,7 +132,8 @@ restart:
> static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
> struct mm_struct *mm,
> unsigned long start,
> - unsigned long end)
> + unsigned long end,
> + enum mmu_event event)
> {
> struct i915_mmu_notifier *mn = container_of(_mn, struct i915_mmu_notifier, mn);
> struct interval_tree_node *it = NULL;
> diff --git a/drivers/gpu/drm/radeon/radeon_mn.c b/drivers/gpu/drm/radeon/radeon_mn.c
> index eef006c..3a9615b 100644
> --- a/drivers/gpu/drm/radeon/radeon_mn.c
> +++ b/drivers/gpu/drm/radeon/radeon_mn.c
> @@ -121,7 +121,8 @@ static void radeon_mn_release(struct mmu_notifier *mn,
> static void radeon_mn_invalidate_range_start(struct mmu_notifier *mn,
> struct mm_struct *mm,
> unsigned long start,
> - unsigned long end)
> + unsigned long end,
> + enum mmu_event event)
> {
> struct radeon_mn *rmn = container_of(mn, struct radeon_mn, mn);
> struct interval_tree_node *it;
> diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
> index 40becdb..6ed69fa 100644
> --- a/drivers/infiniband/core/umem_odp.c
> +++ b/drivers/infiniband/core/umem_odp.c
> @@ -165,7 +165,8 @@ static int invalidate_page_trampoline(struct ib_umem *item, u64 start,
>
> static void ib_umem_notifier_invalidate_page(struct mmu_notifier *mn,
> struct mm_struct *mm,
> - unsigned long address)
> + unsigned long address,
> + enum mmu_event event)
> {
> struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
>
> @@ -192,7 +193,8 @@ static int invalidate_range_start_trampoline(struct ib_umem *item, u64 start,
> static void ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn,
> struct mm_struct *mm,
> unsigned long start,
> - unsigned long end)
> + unsigned long end,
> + enum mmu_event event)
> {
> struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
>
> @@ -217,7 +219,8 @@ static int invalidate_range_end_trampoline(struct ib_umem *item, u64 start,
> static void ib_umem_notifier_invalidate_range_end(struct mmu_notifier *mn,
> struct mm_struct *mm,
> unsigned long start,
> - unsigned long end)
> + unsigned long end,
> + enum mmu_event event)
> {
> struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
>
> diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
> index 3465faf..4aa4de6 100644
> --- a/drivers/iommu/amd_iommu_v2.c
> +++ b/drivers/iommu/amd_iommu_v2.c
> @@ -384,7 +384,8 @@ static int mn_clear_flush_young(struct mmu_notifier *mn,
>
> static void mn_invalidate_page(struct mmu_notifier *mn,
> struct mm_struct *mm,
> - unsigned long address)
> + unsigned long address,
> + enum mmu_event event)
> {
> __mn_flush_page(mn, address);
> }
> diff --git a/drivers/misc/sgi-gru/grutlbpurge.c b/drivers/misc/sgi-gru/grutlbpurge.c
> index 2129274..e67fed1 100644
> --- a/drivers/misc/sgi-gru/grutlbpurge.c
> +++ b/drivers/misc/sgi-gru/grutlbpurge.c
> @@ -221,7 +221,8 @@ void gru_flush_all_tlb(struct gru_state *gru)
> */
> static void gru_invalidate_range_start(struct mmu_notifier *mn,
> struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + enum mmu_event event)
> {
> struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
> ms_notifier);
> @@ -235,7 +236,8 @@ static void gru_invalidate_range_start(struct mmu_notifier *mn,
>
> static void gru_invalidate_range_end(struct mmu_notifier *mn,
> struct mm_struct *mm, unsigned long start,
> - unsigned long end)
> + unsigned long end,
> + enum mmu_event event)
> {
> struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
> ms_notifier);
> @@ -248,7 +250,8 @@ static void gru_invalidate_range_end(struct mmu_notifier *mn,
> }
>
> static void gru_invalidate_page(struct mmu_notifier *mn, struct mm_struct *mm,
> - unsigned long address)
> + unsigned long address,
> + enum mmu_event event)
> {
> struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
> ms_notifier);
> diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
> index 8927485..46bc610 100644
> --- a/drivers/xen/gntdev.c
> +++ b/drivers/xen/gntdev.c
> @@ -467,7 +467,9 @@ static void unmap_if_in_range(struct grant_map *map,
>
> static void mn_invl_range_start(struct mmu_notifier *mn,
> struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event)
> {
> struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn);
> struct grant_map *map;
> @@ -484,9 +486,10 @@ static void mn_invl_range_start(struct mmu_notifier *mn,
>
> static void mn_invl_page(struct mmu_notifier *mn,
> struct mm_struct *mm,
> - unsigned long address)
> + unsigned long address,
> + enum mmu_event event)
> {
> - mn_invl_range_start(mn, mm, address, address + PAGE_SIZE);
> + mn_invl_range_start(mn, mm, address, address + PAGE_SIZE, event);
> }
>
> static void mn_release(struct mmu_notifier *mn,
> diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
> index 6dee68d..58e2390 100644
> --- a/fs/proc/task_mmu.c
> +++ b/fs/proc/task_mmu.c
> @@ -934,11 +934,13 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
> downgrade_write(&mm->mmap_sem);
> break;
> }
> - mmu_notifier_invalidate_range_start(mm, 0, -1);
> + mmu_notifier_invalidate_range_start(mm, 0,
> + -1, MMU_ISDIRTY);
> }
> walk_page_range(0, ~0UL, &clear_refs_walk);
> if (type == CLEAR_REFS_SOFT_DIRTY)
> - mmu_notifier_invalidate_range_end(mm, 0, -1);
> + mmu_notifier_invalidate_range_end(mm, 0,
> + -1, MMU_ISDIRTY);
> flush_tlb_mm(mm);
> up_read(&mm->mmap_sem);
> out_mm:
> diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
> index 61cd67f..8b11b1b 100644
> --- a/include/linux/mmu_notifier.h
> +++ b/include/linux/mmu_notifier.h
> @@ -9,6 +9,70 @@
> struct mmu_notifier;
> struct mmu_notifier_ops;
>
> +/* MMU Events report fine-grained information to the callback routine, allowing
> + * the event listener to make a more informed decision as to what action to
> + * take. The event types are:
> + *
> + * - MMU_FORK when a process is forking and as a results various vma needs to
> + * be write protected to allow for COW.
Make that:
- MMU_FORK: a process is forking. This will lead to vmas getting
write-protected, in order to set up COW.
> + *
> + * - MMU_HSPLIT huge page split, the memory is the same only the page table
> + * structure is updated (level added or removed).
Make that (depending on the name change ideas mentioned later):
- MMU_PAGE_SPLIT: the pages don't move, nor does their content change, but
the page table structure is updated (levels added or removed).
> + *
> + * - MMU_ISDIRTY need to update the dirty bit of the page table so proper
> + * dirty accounting can happen.
> + *
> + * - MMU_MIGRATE: memory is migrating from one page to another, thus all write
> + * access must stop after invalidate_range_start callback returns.
> + * Furthermore, no read access should be allowed either, as a new page can
> + * be remapped with write access before the invalidate_range_end callback
> + * happens and thus any read access to old page might read stale data. There
> + * are several sources for this event, including:
> + *
> + * - A page moving to swap (various reasons, including page reclaim),
> + * - An mremap syscall,
> + * - migration for NUMA reasons,
> + * - balancing the memory pool,
> + * - write fault on COW page,
> + * - and more that are not listed here.
> + *
> + * - MMU_MPROT: memory access protection is changing. Refer to the vma to get
> + * the new access protection. All memory access are still valid until the
> + * invalidate_range_end callback.
> + *
> + * - MMU_MUNLOCK: unlock memory. Content of page table stays the same but
> + * page are unlocked.
> + *
> + * - MMU_MUNMAP: the range is being unmapped (outcome of a munmap syscall or
> + * process destruction). However, access is still allowed, up until the
> + * invalidate_range_free_pages callback. This also implies that secondary
> + * page table can be trimmed, because the address range is no longer valid.
> + *
> + * - MMU_WRITE_BACK: memory is being written back to disk, all write accesses
> + * must stop after invalidate_range_start callback returns. Read access are
> + * still allowed.
> + *
> + * - MMU_WRITE_PROTECT: memory is being write protected (ie should be mapped
> + * read only no matter what the vma memory protection allows). All write
> + * accesses must stop after invalidate_range_start callback returns. Read
> + * access are still allowed.
> + *
> + * If in doubt when adding a new notifier caller, please use MMU_MIGRATE,
> + * because it will always lead to reasonable behavior, but will not allow the
> + * listener a chance to optimize its events.
> + */
> +enum mmu_event {
> + MMU_FORK = 0,
> + MMU_HSPLIT,
Let's rename MMU_HSPLIT to one of the following, take your pick:
MMU_HUGE_PAGE_SPLIT (too long, but you can't possibly misunderstand it)
MMU_PAGE_SPLIT (my favorite: only huge pages are ever split, so it works)
MMU_HUGE_SPLIT (ugly, but still hard to misunderstand)
> + MMU_ISDIRTY,
This MMU_ISDIRTY seems like a problem to me. First of all, it looks
backwards: the only place that invokes it is the clear_refs_write()
routine, for the soft-dirty tracking feature. And in that case, the pages
are *not* being made dirty! Rather, the kernel is actually making the
pages non-writable, in order to be able to trap the subsequent page fault
and figure out if the page is in active use.
So, given that there is only one call site, and that call site should
actually be setting MMU_WRITE_PROTECT instead (I think), let's just delete
MMU_ISDIRTY.
Come to think about it, there is no callback possible for "a page became
dirty", anyway. Because the dirty and accessed bits are actually set by
the hardware, and software is generally unable to know the current state.
So MMU_ISDIRTY just seems inappropriate to me, across the board.
I'll take a look at the corresponding HMM_ISDIRTY, too.
> + MMU_MIGRATE,
> + MMU_MPROT,
The MMU_PROT also looks questionable. Short answer: probably better to
read the protection, and pass either MMU_WRITE_PROTECT, MMU_READ_WRITE
(that's a new item, of course), or MMU_UNMAP.
Here's why: the call site knows the protection, but by the time it filters
down to HMM (in later patches), that information is lost, and HMM ends up
doing (ouch!) another find_vma() call in order to retrieve it--and then
translates it into only three possible things:
// hmm_mmu_mprot_to_etype() sets one of these:
HMM_MUNMAP
HMM_WRITE_PROTECT
HMM_NONE
> + MMU_MUNLOCK,
I think MMU_UNLOCK would be clearer. We already know the scope, so the
extra "M" isn't adding anything.
> + MMU_MUNMAP,
Same thing here: MMU_UNMAP seems better.
> + MMU_WRITE_BACK,
> + MMU_WRITE_PROTECT,
We may have to add MMU_READ_WRITE (and maybe another one, I haven't
bottomed out on that), if you agree with the above approach of
always sending a precise event, instead of "protection changed".
That's all I saw. This is not a complicated patch, even though it's
touching a lot of files, and I think everything else is correct.
thanks,
John Hubbard
> +};
> +
> #ifdef CONFIG_MMU_NOTIFIER
>
> /*
> @@ -82,7 +146,8 @@ struct mmu_notifier_ops {
> void (*change_pte)(struct mmu_notifier *mn,
> struct mm_struct *mm,
> unsigned long address,
> - pte_t pte);
> + pte_t pte,
> + enum mmu_event event);
>
> /*
> * Before this is invoked any secondary MMU is still ok to
> @@ -93,7 +158,8 @@ struct mmu_notifier_ops {
> */
> void (*invalidate_page)(struct mmu_notifier *mn,
> struct mm_struct *mm,
> - unsigned long address);
> + unsigned long address,
> + enum mmu_event event);
>
> /*
> * invalidate_range_start() and invalidate_range_end() must be
> @@ -140,10 +206,14 @@ struct mmu_notifier_ops {
> */
> void (*invalidate_range_start)(struct mmu_notifier *mn,
> struct mm_struct *mm,
> - unsigned long start, unsigned long end);
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event);
> void (*invalidate_range_end)(struct mmu_notifier *mn,
> struct mm_struct *mm,
> - unsigned long start, unsigned long end);
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event);
>
> /*
> * invalidate_range() is either called between
> @@ -206,13 +276,20 @@ extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
> extern int __mmu_notifier_test_young(struct mm_struct *mm,
> unsigned long address);
> extern void __mmu_notifier_change_pte(struct mm_struct *mm,
> - unsigned long address, pte_t pte);
> + unsigned long address,
> + pte_t pte,
> + enum mmu_event event);
> extern void __mmu_notifier_invalidate_page(struct mm_struct *mm,
> - unsigned long address);
> + unsigned long address,
> + enum mmu_event event);
> extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
> - unsigned long start, unsigned long end);
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event);
> extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
> - unsigned long start, unsigned long end);
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event);
> extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
> unsigned long start, unsigned long end);
>
> @@ -240,31 +317,38 @@ static inline int mmu_notifier_test_young(struct mm_struct *mm,
> }
>
> static inline void mmu_notifier_change_pte(struct mm_struct *mm,
> - unsigned long address, pte_t pte)
> + unsigned long address,
> + pte_t pte,
> + enum mmu_event event)
> {
> if (mm_has_notifiers(mm))
> - __mmu_notifier_change_pte(mm, address, pte);
> + __mmu_notifier_change_pte(mm, address, pte, event);
> }
>
> static inline void mmu_notifier_invalidate_page(struct mm_struct *mm,
> - unsigned long address)
> + unsigned long address,
> + enum mmu_event event)
> {
> if (mm_has_notifiers(mm))
> - __mmu_notifier_invalidate_page(mm, address);
> + __mmu_notifier_invalidate_page(mm, address, event);
> }
>
> static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event)
> {
> if (mm_has_notifiers(mm))
> - __mmu_notifier_invalidate_range_start(mm, start, end);
> + __mmu_notifier_invalidate_range_start(mm, start, end, event);
> }
>
> static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event)
> {
> if (mm_has_notifiers(mm))
> - __mmu_notifier_invalidate_range_end(mm, start, end);
> + __mmu_notifier_invalidate_range_end(mm, start, end, event);
> }
>
> static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
> @@ -359,13 +443,13 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
> * old page would remain mapped readonly in the secondary MMUs after the new
> * page is already writable by some CPU through the primary MMU.
> */
> -#define set_pte_at_notify(__mm, __address, __ptep, __pte) \
> +#define set_pte_at_notify(__mm, __address, __ptep, __pte, __event) \
> ({ \
> struct mm_struct *___mm = __mm; \
> unsigned long ___address = __address; \
> pte_t ___pte = __pte; \
> \
> - mmu_notifier_change_pte(___mm, ___address, ___pte); \
> + mmu_notifier_change_pte(___mm, ___address, ___pte, __event); \
> set_pte_at(___mm, ___address, __ptep, ___pte); \
> })
>
> @@ -393,22 +477,29 @@ static inline int mmu_notifier_test_young(struct mm_struct *mm,
> }
>
> static inline void mmu_notifier_change_pte(struct mm_struct *mm,
> - unsigned long address, pte_t pte)
> + unsigned long address,
> + pte_t pte,
> + enum mmu_event event)
> {
> }
>
> static inline void mmu_notifier_invalidate_page(struct mm_struct *mm,
> - unsigned long address)
> + unsigned long address,
> + enum mmu_event event)
> {
> }
>
> static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event)
> {
> }
>
> static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event)
> {
> }
>
> diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
> index cb346f2..802828a 100644
> --- a/kernel/events/uprobes.c
> +++ b/kernel/events/uprobes.c
> @@ -176,7 +176,8 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
> /* For try_to_free_swap() and munlock_vma_page() below */
> lock_page(page);
>
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> err = -EAGAIN;
> ptep = page_check_address(page, mm, addr, &ptl, 0);
> if (!ptep)
> @@ -194,7 +195,9 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
>
> flush_cache_page(vma, addr, pte_pfn(*ptep));
> ptep_clear_flush_notify(vma, addr, ptep);
> - set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
> + set_pte_at_notify(mm, addr, ptep,
> + mk_pte(kpage, vma->vm_page_prot),
> + MMU_MIGRATE);
>
> page_remove_rmap(page);
> if (!page_mapped(page))
> @@ -208,7 +211,8 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
> err = 0;
> unlock:
> mem_cgroup_cancel_charge(kpage, memcg);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> unlock_page(page);
> return err;
> }
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index cb8904c..41c342c 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -1024,7 +1024,8 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
>
> mmun_start = haddr;
> mmun_end = haddr + HPAGE_PMD_SIZE;
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end,
> + MMU_MIGRATE);
>
> ptl = pmd_lock(mm, pmd);
> if (unlikely(!pmd_same(*pmd, orig_pmd)))
> @@ -1058,7 +1059,8 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
> page_remove_rmap(page);
> spin_unlock(ptl);
>
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
>
> ret |= VM_FAULT_WRITE;
> put_page(page);
> @@ -1068,7 +1070,8 @@ out:
>
> out_free_pages:
> spin_unlock(ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> for (i = 0; i < HPAGE_PMD_NR; i++) {
> memcg = (void *)page_private(pages[i]);
> set_page_private(pages[i], 0);
> @@ -1160,7 +1163,8 @@ alloc:
>
> mmun_start = haddr;
> mmun_end = haddr + HPAGE_PMD_SIZE;
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end,
> + MMU_MIGRATE);
>
> spin_lock(ptl);
> if (page)
> @@ -1192,7 +1196,8 @@ alloc:
> }
> spin_unlock(ptl);
> out_mn:
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> out:
> return ret;
> out_unlock:
> @@ -1646,7 +1651,8 @@ static int __split_huge_page_splitting(struct page *page,
> const unsigned long mmun_start = address;
> const unsigned long mmun_end = address + HPAGE_PMD_SIZE;
>
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start,
> + mmun_end, MMU_HSPLIT);
> pmd = page_check_address_pmd(page, mm, address,
> PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG, &ptl);
> if (pmd) {
> @@ -1662,7 +1668,8 @@ static int __split_huge_page_splitting(struct page *page,
> ret = 1;
> spin_unlock(ptl);
> }
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_HSPLIT);
>
> return ret;
> }
> @@ -2526,7 +2533,8 @@ static void collapse_huge_page(struct mm_struct *mm,
>
> mmun_start = address;
> mmun_end = address + HPAGE_PMD_SIZE;
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
> /*
> * After this gup_fast can't run anymore. This also removes
> @@ -2536,7 +2544,8 @@ static void collapse_huge_page(struct mm_struct *mm,
> */
> _pmd = pmdp_collapse_flush(vma, address, pmd);
> spin_unlock(pmd_ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
>
> spin_lock(pte_ptl);
> isolated = __collapse_huge_page_isolate(vma, address, pte);
> @@ -2933,24 +2942,28 @@ void __split_huge_page_pmd(struct vm_area_struct *vma, unsigned long address,
> mmun_start = haddr;
> mmun_end = haddr + HPAGE_PMD_SIZE;
> again:
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> ptl = pmd_lock(mm, pmd);
> if (unlikely(!pmd_trans_huge(*pmd))) {
> spin_unlock(ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> return;
> }
> if (is_huge_zero_pmd(*pmd)) {
> __split_huge_zero_page_pmd(vma, haddr, pmd);
> spin_unlock(ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> return;
> }
> page = pmd_page(*pmd);
> VM_BUG_ON_PAGE(!page_count(page), page);
> get_page(page);
> spin_unlock(ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
>
> split_huge_page(page);
>
> diff --git a/mm/hugetlb.c b/mm/hugetlb.c
> index 54f129d..19da310 100644
> --- a/mm/hugetlb.c
> +++ b/mm/hugetlb.c
> @@ -2670,7 +2670,8 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
> mmun_start = vma->vm_start;
> mmun_end = vma->vm_end;
> if (cow)
> - mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(src, mmun_start,
> + mmun_end, MMU_MIGRATE);
>
> for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
> spinlock_t *src_ptl, *dst_ptl;
> @@ -2724,7 +2725,8 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
> }
>
> if (cow)
> - mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(src, mmun_start,
> + mmun_end, MMU_MIGRATE);
>
> return ret;
> }
> @@ -2750,7 +2752,8 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
> BUG_ON(end & ~huge_page_mask(h));
>
> tlb_start_vma(tlb, vma);
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> address = start;
> again:
> for (; address < end; address += sz) {
> @@ -2824,7 +2827,8 @@ unlock:
> if (address < end && !ref_page)
> goto again;
> }
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> tlb_end_vma(tlb, vma);
> }
>
> @@ -3003,8 +3007,8 @@ retry_avoidcopy:
>
> mmun_start = address & huge_page_mask(h);
> mmun_end = mmun_start + huge_page_size(h);
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> -
> + mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end,
> + MMU_MIGRATE);
> /*
> * Retake the page table lock to check for racing updates
> * before the page tables are altered
> @@ -3025,7 +3029,8 @@ retry_avoidcopy:
> new_page = old_page;
> }
> spin_unlock(ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end,
> + MMU_MIGRATE);
> out_release_all:
> page_cache_release(new_page);
> out_release_old:
> @@ -3493,7 +3498,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
> BUG_ON(address >= end);
> flush_cache_range(vma, address, end);
>
> - mmu_notifier_invalidate_range_start(mm, start, end);
> + mmu_notifier_invalidate_range_start(mm, start, end, MMU_MPROT);
> i_mmap_lock_write(vma->vm_file->f_mapping);
> for (; address < end; address += huge_page_size(h)) {
> spinlock_t *ptl;
> @@ -3543,7 +3548,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
> flush_tlb_range(vma, start, end);
> mmu_notifier_invalidate_range(mm, start, end);
> i_mmap_unlock_write(vma->vm_file->f_mapping);
> - mmu_notifier_invalidate_range_end(mm, start, end);
> + mmu_notifier_invalidate_range_end(mm, start, end, MMU_MPROT);
>
> return pages << h->order;
> }
> diff --git a/mm/ksm.c b/mm/ksm.c
> index bc7be0e..76f167c 100644
> --- a/mm/ksm.c
> +++ b/mm/ksm.c
> @@ -872,7 +872,8 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
>
> mmun_start = addr;
> mmun_end = addr + PAGE_SIZE;
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end,
> + MMU_WRITE_PROTECT);
>
> ptep = page_check_address(page, mm, addr, &ptl, 0);
> if (!ptep)
> @@ -904,7 +905,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
> if (pte_dirty(entry))
> set_page_dirty(page);
> entry = pte_mkclean(pte_wrprotect(entry));
> - set_pte_at_notify(mm, addr, ptep, entry);
> + set_pte_at_notify(mm, addr, ptep, entry, MMU_WRITE_PROTECT);
> }
> *orig_pte = *ptep;
> err = 0;
> @@ -912,7 +913,8 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
> out_unlock:
> pte_unmap_unlock(ptep, ptl);
> out_mn:
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end,
> + MMU_WRITE_PROTECT);
> out:
> return err;
> }
> @@ -948,7 +950,8 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
>
> mmun_start = addr;
> mmun_end = addr + PAGE_SIZE;
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end,
> + MMU_MIGRATE);
>
> ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
> if (!pte_same(*ptep, orig_pte)) {
> @@ -961,7 +964,9 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
>
> flush_cache_page(vma, addr, pte_pfn(*ptep));
> ptep_clear_flush_notify(vma, addr, ptep);
> - set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
> + set_pte_at_notify(mm, addr, ptep,
> + mk_pte(kpage, vma->vm_page_prot),
> + MMU_MIGRATE);
>
> page_remove_rmap(page);
> if (!page_mapped(page))
> @@ -971,7 +976,8 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
> pte_unmap_unlock(ptep, ptl);
> err = 0;
> out_mn:
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end,
> + MMU_MIGRATE);
> out:
> return err;
> }
> diff --git a/mm/madvise.c b/mm/madvise.c
> index 22e8f0c..b90ba3d 100644
> --- a/mm/madvise.c
> +++ b/mm/madvise.c
> @@ -405,9 +405,9 @@ static int madvise_free_single_vma(struct vm_area_struct *vma,
> tlb_gather_mmu(&tlb, mm, start, end);
> update_hiwater_rss(mm);
>
> - mmu_notifier_invalidate_range_start(mm, start, end);
> + mmu_notifier_invalidate_range_start(mm, start, end, MMU_MUNMAP);
> madvise_free_page_range(&tlb, vma, start, end);
> - mmu_notifier_invalidate_range_end(mm, start, end);
> + mmu_notifier_invalidate_range_end(mm, start, end, MMU_MUNMAP);
> tlb_finish_mmu(&tlb, start, end);
>
> return 0;
> diff --git a/mm/memory.c b/mm/memory.c
> index d1fa0c1..9300fad 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -1048,7 +1048,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
> mmun_end = end;
> if (is_cow)
> mmu_notifier_invalidate_range_start(src_mm, mmun_start,
> - mmun_end);
> + mmun_end, MMU_FORK);
>
> ret = 0;
> dst_pgd = pgd_offset(dst_mm, addr);
> @@ -1065,7 +1065,8 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
> } while (dst_pgd++, src_pgd++, addr = next, addr != end);
>
> if (is_cow)
> - mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(src_mm, mmun_start,
> + mmun_end, MMU_FORK);
> return ret;
> }
>
> @@ -1335,10 +1336,12 @@ void unmap_vmas(struct mmu_gather *tlb,
> {
> struct mm_struct *mm = vma->vm_mm;
>
> - mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
> + mmu_notifier_invalidate_range_start(mm, start_addr,
> + end_addr, MMU_MUNMAP);
> for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
> unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
> - mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
> + mmu_notifier_invalidate_range_end(mm, start_addr,
> + end_addr, MMU_MUNMAP);
> }
>
> /**
> @@ -1360,10 +1363,10 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start,
> lru_add_drain();
> tlb_gather_mmu(&tlb, mm, start, end);
> update_hiwater_rss(mm);
> - mmu_notifier_invalidate_range_start(mm, start, end);
> + mmu_notifier_invalidate_range_start(mm, start, end, MMU_MIGRATE);
> for ( ; vma && vma->vm_start < end; vma = vma->vm_next)
> unmap_single_vma(&tlb, vma, start, end, details);
> - mmu_notifier_invalidate_range_end(mm, start, end);
> + mmu_notifier_invalidate_range_end(mm, start, end, MMU_MIGRATE);
> tlb_finish_mmu(&tlb, start, end);
> }
>
> @@ -1386,9 +1389,9 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr
> lru_add_drain();
> tlb_gather_mmu(&tlb, mm, address, end);
> update_hiwater_rss(mm);
> - mmu_notifier_invalidate_range_start(mm, address, end);
> + mmu_notifier_invalidate_range_start(mm, address, end, MMU_MUNMAP);
> unmap_single_vma(&tlb, vma, address, end, details);
> - mmu_notifier_invalidate_range_end(mm, address, end);
> + mmu_notifier_invalidate_range_end(mm, address, end, MMU_MUNMAP);
> tlb_finish_mmu(&tlb, address, end);
> }
>
> @@ -2086,7 +2089,8 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
> if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg))
> goto oom_free_new;
>
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
>
> /*
> * Re-check the pte - we dropped the lock
> @@ -2119,7 +2123,7 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
> * mmu page tables (such as kvm shadow page tables), we want the
> * new page to be mapped directly into the secondary page table.
> */
> - set_pte_at_notify(mm, address, page_table, entry);
> + set_pte_at_notify(mm, address, page_table, entry, MMU_MIGRATE);
> update_mmu_cache(vma, address, page_table);
> if (old_page) {
> /*
> @@ -2158,7 +2162,8 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
> page_cache_release(new_page);
>
> pte_unmap_unlock(page_table, ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> if (old_page) {
> /*
> * Don't let another task, with possibly unlocked vma,
> diff --git a/mm/migrate.c b/mm/migrate.c
> index 236ee25..ad9a55a 100644
> --- a/mm/migrate.c
> +++ b/mm/migrate.c
> @@ -1759,12 +1759,14 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
> WARN_ON(PageLRU(new_page));
>
> /* Recheck the target PMD */
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
> ptl = pmd_lock(mm, pmd);
> if (unlikely(!pmd_same(*pmd, entry) || page_count(page) != 2)) {
> fail_putback:
> spin_unlock(ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
>
> /* Reverse changes made by migrate_page_copy() */
> if (TestClearPageActive(new_page))
> @@ -1818,7 +1820,8 @@ fail_putback:
> page_remove_rmap(page);
>
> spin_unlock(ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
>
> /* Take an "isolate" reference and put new page on the LRU. */
> get_page(new_page);
> diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
> index 3b9b3d0..e51ea02 100644
> --- a/mm/mmu_notifier.c
> +++ b/mm/mmu_notifier.c
> @@ -142,8 +142,10 @@ int __mmu_notifier_test_young(struct mm_struct *mm,
> return young;
> }
>
> -void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
> - pte_t pte)
> +void __mmu_notifier_change_pte(struct mm_struct *mm,
> + unsigned long address,
> + pte_t pte,
> + enum mmu_event event)
> {
> struct mmu_notifier *mn;
> int id;
> @@ -151,13 +153,14 @@ void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
> id = srcu_read_lock(&srcu);
> hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
> if (mn->ops->change_pte)
> - mn->ops->change_pte(mn, mm, address, pte);
> + mn->ops->change_pte(mn, mm, address, pte, event);
> }
> srcu_read_unlock(&srcu, id);
> }
>
> void __mmu_notifier_invalidate_page(struct mm_struct *mm,
> - unsigned long address)
> + unsigned long address,
> + enum mmu_event event)
> {
> struct mmu_notifier *mn;
> int id;
> @@ -165,13 +168,16 @@ void __mmu_notifier_invalidate_page(struct mm_struct *mm,
> id = srcu_read_lock(&srcu);
> hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
> if (mn->ops->invalidate_page)
> - mn->ops->invalidate_page(mn, mm, address);
> + mn->ops->invalidate_page(mn, mm, address, event);
> }
> srcu_read_unlock(&srcu, id);
> }
>
> void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event)
> +
> {
> struct mmu_notifier *mn;
> int id;
> @@ -179,14 +185,17 @@ void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
> id = srcu_read_lock(&srcu);
> hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
> if (mn->ops->invalidate_range_start)
> - mn->ops->invalidate_range_start(mn, mm, start, end);
> + mn->ops->invalidate_range_start(mn, mm, start,
> + end, event);
> }
> srcu_read_unlock(&srcu, id);
> }
> EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_start);
>
> void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> + unsigned long start,
> + unsigned long end,
> + enum mmu_event event)
> {
> struct mmu_notifier *mn;
> int id;
> @@ -204,7 +213,8 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
> if (mn->ops->invalidate_range)
> mn->ops->invalidate_range(mn, mm, start, end);
> if (mn->ops->invalidate_range_end)
> - mn->ops->invalidate_range_end(mn, mm, start, end);
> + mn->ops->invalidate_range_end(mn, mm, start,
> + end, event);
> }
> srcu_read_unlock(&srcu, id);
> }
> diff --git a/mm/mprotect.c b/mm/mprotect.c
> index e7d6f11..a57e8af 100644
> --- a/mm/mprotect.c
> +++ b/mm/mprotect.c
> @@ -155,7 +155,8 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
> /* invoke the mmu notifier if the pmd is populated */
> if (!mni_start) {
> mni_start = addr;
> - mmu_notifier_invalidate_range_start(mm, mni_start, end);
> + mmu_notifier_invalidate_range_start(mm, mni_start,
> + end, MMU_MPROT);
> }
>
> if (pmd_trans_huge(*pmd)) {
> @@ -183,7 +184,8 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
> } while (pmd++, addr = next, addr != end);
>
> if (mni_start)
> - mmu_notifier_invalidate_range_end(mm, mni_start, end);
> + mmu_notifier_invalidate_range_end(mm, mni_start, end,
> + MMU_MPROT);
>
> if (nr_huge_updates)
> count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
> diff --git a/mm/mremap.c b/mm/mremap.c
> index a7c93ec..72051cf 100644
> --- a/mm/mremap.c
> +++ b/mm/mremap.c
> @@ -176,7 +176,8 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
>
> mmun_start = old_addr;
> mmun_end = old_end;
> - mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
>
> for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
> cond_resched();
> @@ -228,7 +229,8 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
> if (likely(need_flush))
> flush_tlb_range(vma, old_end-len, old_addr);
>
> - mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start,
> + mmun_end, MMU_MIGRATE);
>
> return len + old_addr - old_end; /* how much done */
> }
> diff --git a/mm/rmap.c b/mm/rmap.c
> index 9c04594..74c51e0 100644
> --- a/mm/rmap.c
> +++ b/mm/rmap.c
> @@ -915,7 +915,7 @@ static int page_mkclean_one(struct page *page, struct vm_area_struct *vma,
> pte_unmap_unlock(pte, ptl);
>
> if (ret) {
> - mmu_notifier_invalidate_page(mm, address);
> + mmu_notifier_invalidate_page(mm, address, MMU_WRITE_BACK);
> (*cleaned)++;
> }
> out:
> @@ -1338,7 +1338,7 @@ discard:
> out_unmap:
> pte_unmap_unlock(pte, ptl);
> if (ret != SWAP_FAIL && !(flags & TTU_MUNLOCK))
> - mmu_notifier_invalidate_page(mm, address);
> + mmu_notifier_invalidate_page(mm, address, MMU_MIGRATE);
> out:
> return ret;
>
> diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
> index f202c40..d0b1060 100644
> --- a/virt/kvm/kvm_main.c
> +++ b/virt/kvm/kvm_main.c
> @@ -260,7 +260,8 @@ static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
>
> static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
> struct mm_struct *mm,
> - unsigned long address)
> + unsigned long address,
> + enum mmu_event event)
> {
> struct kvm *kvm = mmu_notifier_to_kvm(mn);
> int need_tlb_flush, idx;
> @@ -302,7 +303,8 @@ static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
> static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
> struct mm_struct *mm,
> unsigned long address,
> - pte_t pte)
> + pte_t pte,
> + enum mmu_event event)
> {
> struct kvm *kvm = mmu_notifier_to_kvm(mn);
> int idx;
> @@ -318,7 +320,8 @@ static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
> static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
> struct mm_struct *mm,
> unsigned long start,
> - unsigned long end)
> + unsigned long end,
> + enum mmu_event event)
> {
> struct kvm *kvm = mmu_notifier_to_kvm(mn);
> int need_tlb_flush = 0, idx;
> @@ -344,7 +347,8 @@ static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
> static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
> struct mm_struct *mm,
> unsigned long start,
> - unsigned long end)
> + unsigned long end,
> + enum mmu_event event)
> {
> struct kvm *kvm = mmu_notifier_to_kvm(mn);
>
> --
> 1.9.3
>
>
thanks,
John H.