[RFC PATCH 17/18] mm: init kernel modules with replication support

[Nikita Panov]

Acked-by: Artem Kuzin <artem.kuzin@xxxxxxxxxx>
Acked-by: Alexander Grubnikov <alexander.grubnikov@xxxxxxxxxx>
Acked-by: Ilya Hanov <ilya.hanov@xxxxxxxxxxxxxxxxxxx>
Acked-by: Denis Darvish <darvish.denis@xxxxxxxxxx>
Signed-off-by: Nikita Panov <panov.nikita@xxxxxxxxxx>
---
 include/linux/moduleloader.h |  4 ++++
 kernel/module/main.c         | 17 ++++++++++++++++
 kernel/module/strict_rwx.c   | 12 +++++------
 mm/execmem.c                 | 39 +++++++++++++++++++++++++++++++-----
 4 files changed, 61 insertions(+), 11 deletions(-)

diff --git a/include/linux/moduleloader.h b/include/linux/moduleloader.h
index e395461d59e5..1d100dfc8251 100644
--- a/include/linux/moduleloader.h
+++ b/include/linux/moduleloader.h
@@ -25,6 +25,10 @@ int module_frob_arch_sections(Elf_Ehdr *hdr,
 /* Additional bytes needed by arch in front of individual sections */
 unsigned int arch_mod_section_prepend(struct module *mod, unsigned int section);
 
+#ifdef CONFIG_KERNEL_REPLICATION
+void module_replicate(void *ptr);
+#endif /* CONFIG_KERNEL_REPLICATION */
+
 /* Determines if the section name is an init section (that is only used during
  * module loading).
  */
diff --git a/kernel/module/main.c b/kernel/module/main.c
index d0e1e0bd2ad0..d5c8f041be48 100644
--- a/kernel/module/main.c
+++ b/kernel/module/main.c
@@ -60,6 +60,7 @@
 #include <linux/codetag.h>
 #include <linux/debugfs.h>
 #include <linux/execmem.h>
+#include <linux/numa_kernel_replication.h>
 #include <uapi/linux/module.h>
 #include "internal.h"
 
@@ -1338,6 +1339,18 @@ void __weak module_arch_freeing_init(struct module *mod)
 {
 }
 
+#ifdef CONFIG_KERNEL_REPLICATION
+static int sections_to_replicate[] = {MOD_TEXT, MOD_RODATA};
+
+static void module_replicate_sections(struct module *mod)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(sections_to_replicate); i++)
+		module_replicate(mod->mem[sections_to_replicate[i]].base);
+}
+#endif /* CONFIG_KERNEL_REPLICATION */
+
 static int module_memory_alloc(struct module *mod, enum mod_mem_type type)
 {
 	unsigned int size = PAGE_ALIGN(mod->mem[type].size);
@@ -3330,6 +3343,10 @@ static int complete_formation(struct module *mod, struct load_info *info)
 	module_bug_finalize(info->hdr, info->sechdrs, mod);
 	module_cfi_finalize(info->hdr, info->sechdrs, mod);
 
+#ifdef CONFIG_KERNEL_REPLICATION
+	module_replicate_sections(mod);
+#endif
+
 	err = module_enable_rodata_ro(mod);
 	if (err)
 		goto out_strict_rwx;
diff --git a/kernel/module/strict_rwx.c b/kernel/module/strict_rwx.c
index 8fd438529fbc..07ad1af69eaa 100644
--- a/kernel/module/strict_rwx.c
+++ b/kernel/module/strict_rwx.c
@@ -39,9 +39,9 @@ int module_enable_text_rox(const struct module *mod)
 		if (mem->is_rox)
 			ret = execmem_restore_rox(mem->base, mem->size);
 		else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
-			ret = module_set_memory(mod, type, set_memory_rox);
+			ret = module_set_memory(mod, type, numa_set_memory_rox);
 		else
-			ret = module_set_memory(mod, type, set_memory_x);
+			ret = module_set_memory(mod, type, numa_set_memory_x);
 		if (ret)
 			return ret;
 	}
@@ -55,10 +55,10 @@ int module_enable_rodata_ro(const struct module *mod)
 	if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX) || !rodata_enabled)
 		return 0;
 
-	ret = module_set_memory(mod, MOD_RODATA, set_memory_ro);
+	ret = module_set_memory(mod, MOD_RODATA, numa_set_memory_ro);
 	if (ret)
 		return ret;
-	ret = module_set_memory(mod, MOD_INIT_RODATA, set_memory_ro);
+	ret = module_set_memory(mod, MOD_INIT_RODATA, numa_set_memory_ro);
 	if (ret)
 		return ret;
 
@@ -70,7 +70,7 @@ int module_enable_rodata_ro_after_init(const struct module *mod)
 	if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX) || !rodata_enabled)
 		return 0;
 
-	return module_set_memory(mod, MOD_RO_AFTER_INIT, set_memory_ro);
+	return module_set_memory(mod, MOD_RO_AFTER_INIT, numa_set_memory_ro);
 }
 
 int module_enable_data_nx(const struct module *mod)
@@ -79,7 +79,7 @@ int module_enable_data_nx(const struct module *mod)
 		return 0;
 
 	for_class_mod_mem_type(type, data) {
-		int ret = module_set_memory(mod, type, set_memory_nx);
+		int ret = module_set_memory(mod, type, numa_set_memory_nx);
 
 		if (ret)
 			return ret;
diff --git a/mm/execmem.c b/mm/execmem.c
index 74a178a87e75..ea4b15c8a788 100644
--- a/mm/execmem.c
+++ b/mm/execmem.c
@@ -16,6 +16,7 @@
 #include <linux/set_memory.h>
 #include <linux/moduleloader.h>
 #include <linux/text-patching.h>
+#include <linux/numa_kernel_replication.h>
 
 #include <asm/tlbflush.h>
 
@@ -26,8 +27,9 @@ static struct execmem_info *execmem_info __ro_after_init;
 static struct execmem_info default_execmem_info __ro_after_init;
 
 #ifdef CONFIG_MMU
-static void *execmem_vmalloc(struct execmem_range *range, size_t size,
-			     pgprot_t pgprot, unsigned long vm_flags)
+
+static void *execmem_vmalloc_node(struct execmem_range *range, size_t size,
+			     pgprot_t pgprot, unsigned long vm_flags, int node)
 {
 	bool kasan = range->flags & EXECMEM_KASAN_SHADOW;
 	gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN;
@@ -40,13 +42,13 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size,
 		vm_flags |= VM_DEFER_KMEMLEAK;
 
 	p = __vmalloc_node_range(size, align, start, end, gfp_flags,
-				 pgprot, vm_flags, NUMA_NO_NODE,
+				 pgprot, vm_flags, node,
 				 __builtin_return_address(0));
 	if (!p && range->fallback_start) {
 		start = range->fallback_start;
 		end = range->fallback_end;
 		p = __vmalloc_node_range(size, align, start, end, gfp_flags,
-					 pgprot, vm_flags, NUMA_NO_NODE,
+					 pgprot, vm_flags, node,
 					 __builtin_return_address(0));
 	}
 
@@ -63,6 +65,26 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size,
 	return p;
 }
 
+#ifdef CONFIG_KERNEL_REPLICATION
+static void *execmem_vmalloc_type(struct execmem_range *range, size_t size,
+			     pgprot_t pgprot, unsigned long vm_flags, enum execmem_type type)
+{
+	if (is_text_replicated() && (type == EXECMEM_MODULE_TEXT || type == EXECMEM_MODULE_DATA))
+		/* Need to specify some numa node id for correct allocation and further replication */
+		return execmem_vmalloc_node(range, size, pgprot,
+					vm_flags | VM_NUMA_SHARED, numa_node_id());
+	else
+		return execmem_vmalloc_node(range, size, pgprot,
+					vm_flags, NUMA_NO_NODE);
+}
+#else
+static void *execmem_vmalloc_type(struct execmem_range *range, size_t size,
+			     pgprot_t pgprot, unsigned long vm_flags, enum execmem_type type)
+{
+	return execmem_vmalloc_node(range, size, pgprot, vm_flags, NUMA_NO_NODE);
+}
+#endif
+
 struct vm_struct *execmem_vmap(size_t size)
 {
 	struct execmem_range *range = &execmem_info->ranges[EXECMEM_MODULE_DATA];
@@ -87,6 +109,13 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size,
 #endif /* CONFIG_MMU */
 
 #ifdef CONFIG_ARCH_HAS_EXECMEM_ROX
+
+static void *execmem_vmalloc(struct execmem_range *range, size_t size,
+			     pgprot_t pgprot, unsigned long vm_flags)
+{
+	return execmem_vmalloc_node(range, size, pgprot, vm_flags, NUMA_NO_NODE);
+}
+
 struct execmem_cache {
 	struct mutex mutex;
 	struct maple_tree busy_areas;
@@ -475,7 +504,7 @@ void *execmem_alloc(enum execmem_type type, size_t size)
 	if (use_cache)
 		p = execmem_cache_alloc(range, size);
 	else
-		p = execmem_vmalloc(range, size, pgprot, vm_flags);
+		p = execmem_vmalloc_type(range, size, pgprot, vm_flags, type);
 
 	return kasan_reset_tag(p);
 }
-- 
2.34.1