[PATCH] draft of per-cpu flush flag

From: Tycho Andersen
Date: Thu Aug 31 2017 - 17:03:06 EST


Signed-off-by: Tycho Andersen <tycho@xxxxxxxxxx>
---
mm/xpfo.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------
1 file changed, 70 insertions(+), 8 deletions(-)

diff --git a/mm/xpfo.c b/mm/xpfo.c
index 0b178ad5a37e..5f9aeaaa40d2 100644
--- a/mm/xpfo.c
+++ b/mm/xpfo.c
@@ -31,6 +31,8 @@ struct xpfo {
bool inited; /* Map counter and lock initialized */
atomic_t mapcount; /* Counter for balancing map/unmap requests */
spinlock_t maplock; /* Lock to serialize map/unmap requests */
+ void *mapped; /* per-cpu variable to indicate whether this
+ CPU has mapped this page or not */
};

DEFINE_STATIC_KEY_FALSE(xpfo_inited);
@@ -78,6 +80,43 @@ static inline struct xpfo *lookup_xpfo(struct page *page)
return (void *)page_ext + page_xpfo_ops.offset;
}

+/*
+ * Return the map status of this page. Note that the cpu needs to be pinned,
+ * either via get_cpu() or a spin lock.
+ */
+static bool xpfo_test_unmapped(struct xpfo *xpfo)
+{
+ if (!xpfo->mapped) {
+ return test_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags);
+ } else {
+ return *(bool *)per_cpu_ptr(xpfo->mapped,
+ smp_processor_id());
+ }
+}
+
+/*
+ * Set the unmapped status of this page. Returns the previous state. Note that
+ * the cpu needs to be pinnned, either via get_cpu() or a spin lock.
+ */
+static bool xpfo_test_set_unmapped(struct xpfo *xpfo, bool unmapped)
+{
+ if (!xpfo->mapped) {
+ if (unmapped)
+ return test_and_set_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags);
+ else
+ return test_and_clear_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags);
+ } else {
+ bool *p, prev;
+
+ p = per_cpu_ptr(xpfo->mapped, smp_processor_id());
+ prev = *p;
+ *p = unmapped;
+
+ return prev;
+ }
+}
+
+
void xpfo_alloc_pages(struct page *page, int order, gfp_t gfp)
{
int i, flush_tlb = 0;
@@ -91,7 +130,7 @@ void xpfo_alloc_pages(struct page *page, int order, gfp_t gfp)
if (!xpfo)
continue;

- WARN(test_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags),
+ WARN(xpfo_test_unmapped(xpfo),
"xpfo: unmapped page being allocated\n");

/* Initialize the map lock and map counter */
@@ -99,7 +138,9 @@ void xpfo_alloc_pages(struct page *page, int order, gfp_t gfp)
spin_lock_init(&xpfo->maplock);
atomic_set(&xpfo->mapcount, 0);
xpfo->inited = true;
+ xpfo->mapped = NULL;
}
+
WARN(atomic_read(&xpfo->mapcount),
"xpfo: already mapped page being allocated\n");

@@ -168,13 +209,19 @@ void xpfo_kmap(void *kaddr, struct page *page)
return;

spin_lock(&xpfo->maplock);
+ if (!xpfo->mapped) {
+ xpfo->mapped = alloc_percpu(bool);
+ if (!xpfo->mapped)
+ WARN_ON("xpfo: percpu flag allocation failed\n");
+ }

/*
* The page was previously allocated to user space, so map it back
* into the kernel. No TLB flush required.
*/
- if ((atomic_inc_return(&xpfo->mapcount) == 1) &&
- test_and_clear_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags))
+ xpfo_test_set_unmapped(xpfo, false);
+
+ if (atomic_inc_return(&xpfo->mapcount) == 1)
set_kpte(kaddr, page, PAGE_KERNEL);

spin_unlock(&xpfo->maplock);
@@ -205,10 +252,25 @@ void xpfo_kunmap(void *kaddr, struct page *page)
* The page is to be allocated back to user space, so unmap it from the
* kernel, flush the TLB and tag it as a user page.
*/
- if (atomic_dec_return(&xpfo->mapcount) == 0) {
- WARN(test_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags),
- "xpfo: unmapping already unmapped page\n");
- set_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags);
+ if (xpfo->mapped) {
+ /*
+ * We have a per-cpu map, and we know it is mapped on this
+ * process, so let's flush our local TLB.
+ */
+ xpfo_test_set_unmapped(xpfo, true);
+
+ /*
+ * I think this is incorrect -- the page should still be mapped
+ * by the other cpus, it's just the TLB entry here is a bit stale.
+ */
+ set_kpte(kaddr, page, __pgprot(0));
+ __flush_tlb_one((unsigned long) kaddr);
+ } else if (atomic_dec_return(&xpfo->mapcount) == 0) {
+ /*
+ * No per-cpu map, so let's just do a best effort and
+ * unmap/flush all the TLBs when the count reaches 0.
+ */
+ xpfo_test_set_unmapped(xpfo, true);
set_kpte(kaddr, page, __pgprot(0));
flush_tlb_kernel_range((unsigned long) kaddr,
(unsigned long) kaddr + PAGE_SIZE);
@@ -229,7 +291,7 @@ bool xpfo_page_is_unmapped(struct page *page)
if (unlikely(!xpfo) && !xpfo->inited)
return false;

- return test_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags);
+ return xpfo_test_unmapped(xpfo);
}
EXPORT_SYMBOL(xpfo_page_is_unmapped);

--
2.11.0