Re: [kvm-devel] [patch 2/6] mmu_notifier: Callbacks to invalidateaddress ranges

From: Christoph Lameter
Date: Wed Jan 30 2008 - 21:08:32 EST


Patch to


1. Remove sync on notifier_release. Must be called when only a
single process remain.

2. Add invalidate_range_start/end. This should allow safe removal
of ranges of external ptes without having to resort to a callback
for every individual page.

This must be able to nest so the driver needs to keep a refcount of range
invalidates and wait if the refcount != 0.


---
include/linux/mmu_notifier.h | 11 +++++++++--
mm/fremap.c | 3 ++-
mm/hugetlb.c | 3 ++-
mm/memory.c | 16 ++++++++++------
mm/mmu_notifier.c | 9 ++++-----
5 files changed, 27 insertions(+), 15 deletions(-)

Index: linux-2.6/mm/mmu_notifier.c
===================================================================
--- linux-2.6.orig/mm/mmu_notifier.c 2008-01-30 17:58:48.000000000 -0800
+++ linux-2.6/mm/mmu_notifier.c 2008-01-30 18:00:26.000000000 -0800
@@ -13,23 +13,22 @@
#include <linux/mm.h>
#include <linux/mmu_notifier.h>

+/*
+ * No synchronization. This function can only be called when only a single
+ * process remains that performs teardown.
+ */
void mmu_notifier_release(struct mm_struct *mm)
{
struct mmu_notifier *mn;
struct hlist_node *n, *t;

if (unlikely(!hlist_empty(&mm->mmu_notifier.head))) {
- down_write(&mm->mmap_sem);
- rcu_read_lock();
hlist_for_each_entry_safe_rcu(mn, n, t,
&mm->mmu_notifier.head, hlist) {
hlist_del_rcu(&mn->hlist);
if (mn->ops->release)
mn->ops->release(mn, mm);
}
- rcu_read_unlock();
- up_write(&mm->mmap_sem);
- synchronize_rcu();
}
}

Index: linux-2.6/include/linux/mmu_notifier.h
===================================================================
--- linux-2.6.orig/include/linux/mmu_notifier.h 2008-01-30 17:58:48.000000000 -0800
+++ linux-2.6/include/linux/mmu_notifier.h 2008-01-30 18:00:26.000000000 -0800
@@ -67,15 +67,22 @@ struct mmu_notifier_ops {
int dummy);

/*
+ * invalidate_range_begin() and invalidate_range_end() are paired.
+ *
+ * invalidate_range_begin must clear all references in the range
+ * and stop the establishment of new references.
+ *
+ * invalidate_range_end() reenables the establishment of references.
+ *
* lock indicates that the function is called under spinlock.
*/
void (*invalidate_range_begin)(struct mmu_notifier *mn,
struct mm_struct *mm,
+ unsigned long start, unsigned long end,
int lock);

void (*invalidate_range_end)(struct mmu_notifier *mn,
- struct mm_struct *mm,
- unsigned long start, unsigned long end);
+ struct mm_struct *mm);
};

struct mmu_rmap_notifier_ops;
Index: linux-2.6/mm/fremap.c
===================================================================
--- linux-2.6.orig/mm/fremap.c 2008-01-30 17:58:48.000000000 -0800
+++ linux-2.6/mm/fremap.c 2008-01-30 18:00:26.000000000 -0800
@@ -212,8 +212,9 @@ asmlinkage long sys_remap_file_pages(uns
spin_unlock(&mapping->i_mmap_lock);
}

+ mmu_notifier(invalidate_range_start, mm, start, start + size, 0);
err = populate_range(mm, vma, start, size, pgoff);
- mmu_notifier(invalidate_range, mm, start, start + size, 0);
+ mmu_notifier(invalidate_range_end, mm);
if (!err && !(flags & MAP_NONBLOCK)) {
if (unlikely(has_write_lock)) {
downgrade_write(&mm->mmap_sem);
Index: linux-2.6/mm/hugetlb.c
===================================================================
--- linux-2.6.orig/mm/hugetlb.c 2008-01-30 17:58:48.000000000 -0800
+++ linux-2.6/mm/hugetlb.c 2008-01-30 18:00:26.000000000 -0800
@@ -744,6 +744,7 @@ void __unmap_hugepage_range(struct vm_ar
BUG_ON(start & ~HPAGE_MASK);
BUG_ON(end & ~HPAGE_MASK);

+ mmu_notifier(invalidate_range_start, mm, start, end, 1);
spin_lock(&mm->page_table_lock);
for (address = start; address < end; address += HPAGE_SIZE) {
ptep = huge_pte_offset(mm, address);
@@ -764,7 +765,7 @@ void __unmap_hugepage_range(struct vm_ar
}
spin_unlock(&mm->page_table_lock);
flush_tlb_range(vma, start, end);
- mmu_notifier(invalidate_range, mm, start, end, 1);
+ mmu_notifier(invalidate_range_end, mm);
list_for_each_entry_safe(page, tmp, &page_list, lru) {
list_del(&page->lru);
put_page(page);
Index: linux-2.6/mm/memory.c
===================================================================
--- linux-2.6.orig/mm/memory.c 2008-01-30 17:58:48.000000000 -0800
+++ linux-2.6/mm/memory.c 2008-01-30 18:00:51.000000000 -0800
@@ -888,11 +888,12 @@ unsigned long zap_page_range(struct vm_a
lru_add_drain();
tlb = tlb_gather_mmu(mm, 0);
update_hiwater_rss(mm);
+ mmu_notifier(invalidate_range_start, mm, address, end,
+ (details ? (details->i_mmap_lock != NULL) : 0));
end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details);
if (tlb)
tlb_finish_mmu(tlb, address, end);
- mmu_notifier(invalidate_range, mm, address, end,
- (details ? (details->i_mmap_lock != NULL) : 0));
+ mmu_notifier(invalidate_range_end, mm);
return end;
}

@@ -1355,6 +1356,7 @@ int remap_pfn_range(struct vm_area_struc
pfn -= addr >> PAGE_SHIFT;
pgd = pgd_offset(mm, addr);
flush_cache_range(vma, addr, end);
+ mmu_notifier(invalidate_range_start, mm, start, end, 0);
do {
next = pgd_addr_end(addr, end);
err = remap_pud_range(mm, pgd, addr, next,
@@ -1362,7 +1364,7 @@ int remap_pfn_range(struct vm_area_struc
if (err)
break;
} while (pgd++, addr = next, addr != end);
- mmu_notifier(invalidate_range, mm, start, end, 0);
+ mmu_notifier(invalidate_range_end, mm);
return err;
}
EXPORT_SYMBOL(remap_pfn_range);
@@ -1450,6 +1452,7 @@ int apply_to_page_range(struct mm_struct
int err;

BUG_ON(addr >= end);
+ mmu_notifier(invalidate_range_start, mm, start, end, 0);
pgd = pgd_offset(mm, addr);
do {
next = pgd_addr_end(addr, end);
@@ -1457,7 +1460,7 @@ int apply_to_page_range(struct mm_struct
if (err)
break;
} while (pgd++, addr = next, addr != end);
- mmu_notifier(invalidate_range, mm, start, end, 0);
+ mmu_notifier(invalidate_range_end, mm);
return err;
}
EXPORT_SYMBOL_GPL(apply_to_page_range);
@@ -1635,6 +1638,8 @@ gotten:
goto oom;
cow_user_page(new_page, old_page, address, vma);

+ mmu_notifier(invalidate_range_start, mm, address,
+ address + PAGE_SIZE - 1, 0);
/*
* Re-check the pte - we dropped the lock
*/
@@ -1673,8 +1678,7 @@ gotten:
page_cache_release(old_page);
unlock:
pte_unmap_unlock(page_table, ptl);
- mmu_notifier(invalidate_range, mm, address,
- address + PAGE_SIZE - 1, 0);
+ mmu_notifier(invalidate_range_end, mm);
if (dirty_page) {
if (vma->vm_file)
file_update_time(vma->vm_file);

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/