[RFC PATCH 11/18] mm: enable per-NUMA node kernel text and rodata replication

[Nikita Panov]

Acked-by: Artem Kuzin <artem.kuzin@xxxxxxxxxx>
Acked-by: Alexander Grubnikov <alexander.grubnikov@xxxxxxxxxx>
Acked-by: Ilya Hanov <ilya.hanov@xxxxxxxxxxxxxxxxxxx>
Acked-by: Denis Darvish <darvish.denis@xxxxxxxxxx>
Signed-off-by: Nikita Panov <panov.nikita@xxxxxxxxxx>
---
 arch/arm64/mm/context.c |  1 -
 arch/arm64/mm/mmu.c     |  8 ++++++--
 include/linux/mm.h      |  3 +++
 init/main.c             | 17 +++++++++++++++++
 mm/mm_init.c            |  3 +++
 5 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index 3afdae62784e..0f4a28b87469 100644
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -11,7 +11,6 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/mm.h>
-#include <linux/numa_kernel_replication.h>
 
 #include <asm/cpufeature.h>
 #include <asm/mmu_context.h>
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index fa12461c0d8e..4bc8b748e303 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1040,10 +1040,10 @@ void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
 				 pgd_pgtable_alloc_special_mm, flags);
 }
 
+#ifdef CONFIG_KERNEL_REPLICATION
 static void populate_mappings_prot(phys_addr_t phys, unsigned long virt,
 				   phys_addr_t size, pgprot_t prot)
 {
-#ifdef CONFIG_KERNEL_REPLICATION
 	int nid;
 
 	for_each_memory_node(nid) {
@@ -1051,10 +1051,14 @@ static void populate_mappings_prot(phys_addr_t phys, unsigned long virt,
 			page_to_phys(walk_to_page_node(nid, (void *)virt)),
 			virt, size, prot, NULL, 0);
 	}
+}
 #else
+static void populate_mappings_prot(phys_addr_t phys, unsigned long virt,
+				   phys_addr_t size, pgprot_t prot)
+{
 	early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0);
-#endif /* CONFIG_KERNEL_REPLICATION */
 }
+#endif /* CONFIG_KERNEL_REPLICATION */
 
 static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
 				phys_addr_t size, pgprot_t prot)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 8a7083c02203..0f3592038823 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -4240,6 +4240,9 @@ extern int __meminit early_pfn_to_nid(unsigned long pfn);
 #endif
 
 extern void mem_init(void);
+#ifdef CONFIG_KERNEL_REPLICATION
+extern void preallocate_vmalloc_pages(void);
+#endif
 extern void __init mmap_init(void);
 
 extern void __show_mem(unsigned int flags, const nodemask_t *nodemask, int max_zone_idx);
diff --git a/init/main.c b/init/main.c
index d9d936707c19..249307e46818 100644
--- a/init/main.c
+++ b/init/main.c
@@ -109,6 +109,7 @@
 #include <linux/unaligned.h>
 #include <linux/vdso_datastore.h>
 #include <linux/hazptr.h>
+#include <linux/numa_kernel_replication.h>
 #include <net/net_namespace.h>
 
 #include <asm/io.h>
@@ -1038,12 +1039,20 @@ void start_kernel(void)
 	 * These use large bootmem allocations and must precede
 	 * initalization of page allocator
 	 */
+	numa_replication_init();
 	setup_log_buf(0);
 	vfs_caches_init_early();
 	sort_main_extable();
 	trap_init();
 	mm_core_init();
 	maple_tree_init();
+	/*
+	 * Kernel text replication should be done before
+	 * alloc/init first mm struct, due to it is necessary
+	 * to setup per-NUMA node translation tables and kernel
+	 * instances properly.
+	 */
+	numa_replicate_kernel_text();
 	poking_init();
 	ftrace_init();
 
@@ -1571,6 +1580,14 @@ static int __ref kernel_init(void *unused)
 	free_initmem();
 	mark_readonly();
 
+	/*
+	 * RODATA replication is done here due to
+	 * it is necessary to finalize the kernel
+	 * and modules initialization before
+	 */
+	numa_replicate_kernel_rodata();
+	numa_replication_fini();
+
 	/*
 	 * Kernel mappings are now finalized - update the userspace page-table
 	 * to finalize PTI.
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 1533aebafb68..d2b13d8887c8 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -2679,6 +2679,9 @@ void __init mm_core_init(void)
 	ptlock_cache_init();
 	pgtable_cache_init();
 	debug_objects_mem_init();
+#ifdef CONFIG_KERNEL_REPLICATION
+	preallocate_vmalloc_pages();
+#endif
 	vmalloc_init();
 	/* If no deferred init page_ext now, as vmap is fully initialized */
 	if (!deferred_struct_pages)
-- 
2.34.1