[PATCH] x86: enable RCU based table free when PARAVIRT

[Vitaly Kuznetsov]

Signed-off-by: Vitaly Kuznetsov <vkuznets@xxxxxxxxxx>
---
 arch/x86/Kconfig      |  1 +
 arch/x86/mm/pgtable.c | 16 ++++++++++++++++
 mm/memory.c           |  5 +++++
 3 files changed, 22 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 781521b7cf9e..9c1666ea04c9 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -167,6 +167,7 @@ config X86
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_REGS_AND_STACK_ACCESS_API
+	select HAVE_RCU_TABLE_FREE              if SMP && PARAVIRT
 	select HAVE_RELIABLE_STACKTRACE		if X86_64 && FRAME_POINTER && STACK_VALIDATION
 	select HAVE_STACK_VALIDATION		if X86_64
 	select HAVE_SYSCALL_TRACEPOINTS
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 508a708eb9a6..b6aaab9fb3b8 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -56,7 +56,11 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
 {
 	pgtable_page_dtor(pte);
 	paravirt_release_pte(page_to_pfn(pte));
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+	tlb_remove_table(tlb, pte);
+#else
 	tlb_remove_page(tlb, pte);
+#endif
 }
 
 #if CONFIG_PGTABLE_LEVELS > 2
@@ -72,21 +76,33 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
 	tlb->need_flush_all = 1;
 #endif
 	pgtable_pmd_page_dtor(page);
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+	tlb_remove_table(tlb, page);
+#else
 	tlb_remove_page(tlb, page);
+#endif
 }
 
 #if CONFIG_PGTABLE_LEVELS > 3
 void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
 {
 	paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+	tlb_remove_table(tlb, virt_to_page(pud));
+#else
 	tlb_remove_page(tlb, virt_to_page(pud));
+#endif
 }
 
 #if CONFIG_PGTABLE_LEVELS > 4
 void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d)
 {
 	paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT);
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+	tlb_remove_table(tlb, virt_to_page(p4d));
+#else
 	tlb_remove_page(tlb, virt_to_page(p4d));
+#endif
 }
 #endif	/* CONFIG_PGTABLE_LEVELS > 4 */
 #endif	/* CONFIG_PGTABLE_LEVELS > 3 */
diff --git a/mm/memory.c b/mm/memory.c
index e158f7ac6730..18d6671b6ae2 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -329,6 +329,11 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_
  * See the comment near struct mmu_table_batch.
  */
 
+static void __tlb_remove_table(void *table)
+{
+	free_page_and_swap_cache(table);
+}
+
 static void tlb_remove_table_smp_sync(void *arg)
 {
 	/* Simply deliver the interrupt */
-- 
2.13.4


--=-=-=--