Linus,
This patch splits up the existing tlb_remove_page into
tlb_remove_tlb_entry (for pages that are/were mapped into userspace)
and tlb_remove_page, as you suggested. It also adds the necessary
stuff for PPC, which has its own include/asm-ppc/tlb.h now. This
works on at least one PPC machine. :)
Thanks,
Paul.
diff -urN linux-2.5/include/asm-generic/tlb.h pmac-2.5/include/asm-generic/tlb.h
--- linux-2.5/include/asm-generic/tlb.h Sun May 19 21:04:28 2002
+++ pmac-2.5/include/asm-generic/tlb.h Mon May 20 11:46:18 2002
@@ -83,11 +83,10 @@
tlb_flush_mmu(tlb, start, end);
}
-
-/* void tlb_remove_page(mmu_gather_t *tlb, pte_t *ptep, unsigned long addr)
- * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while
- * handling the additional races in SMP caused by other CPUs caching valid
- * mappings in their TLBs.
+/* void tlb_remove_page(mmu_gather_t *tlb, struct page *page)
+ * This should free the page given after flushing any reference
+ * to it from the TLB. This should be done no later than the
+ * next call to tlb_finish_mmu for this tlb.
*/
static inline void tlb_remove_page(mmu_gather_t *tlb, struct page *page)
{
@@ -99,6 +98,18 @@
tlb->pages[tlb->nr++] = page;
if (tlb->nr >= FREE_PTE_NR)
tlb_flush_mmu(tlb, 0, 0);
+}
+
+/* void tlb_remove_tlb_entry(mmu_gather_t *tlb, struct page *page, unsigned long address)
+ * This is similar to tlb_remove_page, except that we are given the
+ * virtual address at which the page was mapped. The address parameter
+ * is unused here but is used on some architectures.
+ */
+static inline void tlb_remove_tlb_entry(mmu_gather_t *tlb, struct page *page,
+ unsigned long address)
+{
+ tlb->freed++;
+ tlb_remove_page(tlb, page);
}
#endif /* _ASM_GENERIC__TLB_H */
diff -urN linux-2.5/mm/memory.c pmac-2.5/mm/memory.c
--- linux-2.5/mm/memory.c Thu May 16 20:31:42 2002
+++ pmac-2.5/mm/memory.c Mon May 20 14:06:58 2002
@@ -353,7 +353,7 @@
if (!PageReserved(page)) {
if (pte_dirty(pte))
set_page_dirty(page);
- tlb_remove_page(tlb, page);
+ tlb_remove_tlb_entry(tlb, page, address+offset);
}
}
} else {
diff -urN linux-2.5/include/asm-ppc/tlb.h pmac-2.5/include/asm-ppc/tlb.h
--- linux-2.5/include/asm-ppc/tlb.h Tue Feb 5 18:40:23 2002
+++ pmac-2.5/include/asm-ppc/tlb.h Mon May 20 20:14:20 2002
@@ -1,4 +1,141 @@
/*
- * BK Id: SCCS/s.tlb.h 1.5 05/17/01 18:14:26 cort
+ * include/asm-ppc/tlb.h
+ *
+ * TLB shootdown code for PPC
+ *
+ * Based on include/asm-generic/tlb.h, which is
+ * Copyright 2001 Red Hat, Inc.
+ * Based on code from mm/memory.c Copyright Linus Torvalds and others.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
*/
-#include <asm-generic/tlb.h>
+#ifndef _ASM_PPC__TLB_H
+#define _ASM_PPC__TLB_H
+
+#include <linux/config.h>
+#include <asm/tlbflush.h>
+#include <asm/page.h>
+
+/*
+ * This makes sizeof(mmu_gather_t) a power of 2.
+ * We assume we get some advantage from batching up the invalidations.
+ * It would be nice to measure how much ...
+ */
+#define FREE_PTE_NR 507
+
+/* mmu_gather_t is an opaque type used by the mm code for passing around any
+ * data needed by arch specific code for tlb_remove_page. This structure can
+ * be per-CPU or per-MM as the page table lock is held for the duration of TLB
+ * shootdown.
+ */
+typedef struct free_pte_ctx {
+ struct mm_struct *mm;
+ unsigned long nr; /* set to ~0UL means fast mode */
+ unsigned long freed;
+ unsigned long start;
+ unsigned long end;
+ struct page * pages[FREE_PTE_NR];
+} mmu_gather_t;
+
+/* Declared in arch/ppc/mm/init.c. */
+extern mmu_gather_t mmu_gathers[NR_CPUS];
+
+/*
+ * Actually do the flushes that we have gathered up, and
+ * then free the corresponding pages.
+ */
+static inline void tlb_flush_mmu(mmu_gather_t *tlb)
+{
+ unsigned long nr;
+
+ nr = tlb->nr;
+ if (nr != 0) {
+ unsigned long i;
+ flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end + PAGE_SIZE);
+ tlb->nr = 0;
+ for (i = 0; i < nr; i++)
+ free_page_and_swap_cache(tlb->pages[i]);
+ }
+}
+
+/* tlb_gather_mmu
+ * Return a pointer to an initialized mmu_gather_t.
+ */
+static inline mmu_gather_t *tlb_gather_mmu(struct mm_struct *mm)
+{
+ mmu_gather_t *tlb = &mmu_gathers[smp_processor_id()];
+
+ tlb->mm = mm;
+ tlb->freed = 0;
+ tlb->nr = 0;
+ return tlb;
+}
+
+/* tlb_finish_mmu
+ * Called at the end of the shootdown operation to free up any resources
+ * that were required. The page table lock is still held at this point.
+ */
+static inline void tlb_finish_mmu(mmu_gather_t *tlb, unsigned long start, unsigned long end)
+{
+ int freed = tlb->freed;
+ struct mm_struct *mm = tlb->mm;
+ int rss = mm->rss;
+
+ if (rss < freed)
+ freed = rss;
+ mm->rss = rss - freed;
+}
+
+/* Nothing needed here in fact... */
+#define tlb_start_vma(tlb, vma) do { } while (0)
+
+/*
+ * flush_tlb_mm_range looks at the pte pages for the range of addresses
+ * in order to check the _PAGE_HASHPTE bit. Thus we can't defer
+ * the tlb_flush_mmu call to tlb_finish_mmu time, since by then the
+ * pointers to the pte pages in the pgdir have been zeroed.
+ * Instead we do the tlb_flush_mmu here. In future we could possibly
+ * do something cleverer, like keeping our own pointer(s) to the pte
+ * page(s) that we are interested in.
+ */
+static inline void tlb_end_vma(mmu_gather_t *tlb, struct vm_area_struct *vma)
+{
+ tlb_flush_mmu(tlb);
+}
+
+/* void tlb_remove_page(mmu_gather_t *tlb, struct page *page)
+ *
+ * On PPC this should never be called.
+ */
+static inline void tlb_remove_page(mmu_gather_t *tlb, struct page *page)
+{
+ BUG();
+}
+
+/* void tlb_remove_tlb_entry(mmu_gather_t *tlb, struct page *page, unsigned long address)
+ * This should free the page given after flushing any reference
+ * to it from the MMU hash table and TLB. This should be done no
+ * later than the next call to tlb_finish_mmu for this tlb.
+ * We get given the virtual address at which the page was mapped.
+ */
+static inline void tlb_remove_tlb_entry(mmu_gather_t *tlb, struct page *page,
+ unsigned long address)
+{
+ tlb->freed++;
+
+ if (tlb->nr == 0)
+ tlb->start = address;
+ else if (address - tlb->end > 32 * PAGE_SIZE) {
+ tlb_flush_mmu(tlb);
+ tlb->start = address;
+ }
+ tlb->end = address;
+ tlb->pages[tlb->nr++] = page;
+ if (tlb->nr >= FREE_PTE_NR)
+ tlb_flush_mmu(tlb);
+}
+
+#endif /* _ASM_PPC__TLB_H */
diff -urN linux-2.5/include/asm-ppc/tlbflush.h pmac-2.5/include/asm-ppc/tlbflush.h
--- linux-2.5/include/asm-ppc/tlbflush.h Fri May 10 10:14:59 2002
+++ pmac-2.5/include/asm-ppc/tlbflush.h Mon May 20 12:00:24 2002
@@ -27,6 +27,9 @@
{ __tlbia(); }
static inline void flush_tlb_mm(struct mm_struct *mm)
{ __tlbia(); }
+static inline void flush_tlb_mm_range(struct mm_struct *mm,
+ unsigned long start, unsigned long end);
+ { __tlbia(); }
static inline void flush_tlb_page(struct vm_area_struct *vma,
unsigned long vmaddr)
{ _tlbie(vmaddr); }
@@ -45,6 +48,9 @@
{ __tlbia(); }
static inline void flush_tlb_mm(struct mm_struct *mm)
{ __tlbia(); }
+static inline void flush_tlb_mm_range(struct mm_struct *mm,
+ unsigned long start, unsigned long end)
+ { __tlbia(); }
static inline void flush_tlb_page(struct vm_area_struct *vma,
unsigned long vmaddr)
{ _tlbie(vmaddr); }
@@ -61,6 +67,8 @@
struct vm_area_struct;
extern void flush_tlb_all(void);
extern void flush_tlb_mm(struct mm_struct *mm);
+extern void flush_tlb_mm_range(struct mm_struct *mm,
+ unsigned long start, unsigned long end);
extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end);
diff -urN linux-2.5/include/asm-ppc/pgalloc.h pmac-2.5/include/asm-ppc/pgalloc.h
--- linux-2.5/include/asm-ppc/pgalloc.h Wed Apr 10 17:56:21 2002
+++ pmac-2.5/include/asm-ppc/pgalloc.h Mon May 20 11:18:03 2002
@@ -20,6 +20,7 @@
*/
#define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(x) do { } while (0)
+#define pmd_free_tlb(tlb,x) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
#define pmd_populate_kernel(mm, pmd, pte) \
@@ -31,6 +32,8 @@
extern struct page *pte_alloc_one(struct mm_struct *mm, unsigned long addr);
extern void pte_free_kernel(pte_t *pte);
extern void pte_free(struct page *pte);
+
+#define pte_free_tlb(tlb, pte) pte_free((pte))
#define check_pgt_cache() do { } while (0)
diff -urN linux-2.5/arch/ppc/mm/tlb.c pmac-2.5/arch/ppc/mm/tlb.c
--- linux-2.5/arch/ppc/mm/tlb.c Mon Apr 15 09:48:49 2002
+++ pmac-2.5/arch/ppc/mm/tlb.c Mon May 20 22:21:40 2002
@@ -59,7 +59,7 @@
#define FINISH_FLUSH do { } while (0)
#endif
-static void flush_range(struct mm_struct *mm, unsigned long start,
+void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
unsigned long end)
{
pmd_t *pmd;
@@ -110,7 +110,7 @@
*/
printk(KERN_ERR "flush_tlb_all called from %p\n",
__builtin_return_address(0));
- flush_range(&init_mm, TASK_SIZE, ~0UL);
+ flush_tlb_mm_range(&init_mm, TASK_SIZE, ~0UL);
FINISH_FLUSH;
}
@@ -119,7 +119,7 @@
*/
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
- flush_range(&init_mm, start, end);
+ flush_tlb_mm_range(&init_mm, start, end);
FINISH_FLUSH;
}
@@ -130,18 +130,15 @@
*/
void flush_tlb_mm(struct mm_struct *mm)
{
+ struct vm_area_struct *mp;
+
if (Hash == 0) {
_tlbia();
return;
}
- if (mm->map_count) {
- struct vm_area_struct *mp;
- for (mp = mm->mmap; mp != NULL; mp = mp->vm_next)
- flush_range(mp->vm_mm, mp->vm_start, mp->vm_end);
- } else {
- flush_range(mm, 0, TASK_SIZE);
- }
+ for (mp = mm->mmap; mp != NULL; mp = mp->vm_next)
+ flush_tlb_mm_range(mp->vm_mm, mp->vm_start, mp->vm_end);
FINISH_FLUSH;
}
@@ -170,6 +167,6 @@
void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
- flush_range(vma->vm_mm, start, end);
+ flush_tlb_mm_range(vma->vm_mm, start, end);
FINISH_FLUSH;
}
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
This archive was generated by hypermail 2b29 : Thu May 23 2002 - 22:00:19 EST