[patch 24/38] module: Add layout for callthunks tracking

[Thomas Gleixner]

From: Peter Zijlstra <peterz@xxxxxxxxxxxxx>

Various things will need to be able to tell if a specific address is a
callthunk or not (ORC, BPF, static_call). In order to answer this
question in the face of modules it is necessary to (quickly) find the
module associated with a specific (callthunk) address.

Extend the __module_address() infrastructure with knowledge of the
(per module) callthunk range.

Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
Signed-off-by: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
---
 include/linux/module.h      |   21 +++++++++++++++++++--
 kernel/module/internal.h    |    8 ++++++++
 kernel/module/main.c        |    6 ++++++
 kernel/module/tree_lookup.c |   17 ++++++++++++++++-
 4 files changed, 49 insertions(+), 3 deletions(-)

--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -424,6 +424,9 @@ struct module {
 	/* Core layout: rbtree is accessed frequently, so keep together. */
 	struct module_layout core_layout __module_layout_align;
 	struct module_layout init_layout;
+#ifdef CONFIG_CALL_THUNKS
+	struct module_layout thunk_layout;
+#endif
 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
 	struct module_layout data_layout;
 #endif
@@ -590,9 +593,23 @@ static inline bool within_module_init(un
 	       addr < (unsigned long)mod->init_layout.base + mod->init_layout.size;
 }
 
-static inline bool within_module(unsigned long addr, const struct module *mod)
+static inline bool within_module_thunk(unsigned long addr,
+				       const struct module *mod)
+{
+#ifdef CONFIG_CALL_THUNKS
+	return (unsigned long)mod->thunk_layout.base <= addr &&
+	       addr < (unsigned long)mod->thunk_layout.base + mod->thunk_layout.size;
+#else
+	return false;
+#endif
+}
+
+static inline bool within_module(unsigned long addr,
+				 const struct module *mod)
 {
-	return within_module_init(addr, mod) || within_module_core(addr, mod);
+	return within_module_core(addr, mod)  ||
+	       within_module_thunk(addr, mod) ||
+	       within_module_init(addr, mod);
 }
 
 /* Search for module by name: must be in a RCU-sched critical section. */
--- a/kernel/module/internal.h
+++ b/kernel/module/internal.h
@@ -219,6 +219,14 @@ static inline struct module *mod_find(un
 }
 #endif /* CONFIG_MODULES_TREE_LOOKUP */
 
+#if defined(CONFIG_MODULES_TREE_LOOKUP) && defined(CONFIG_CALL_THUNKS)
+void mod_tree_insert_thunk(struct module *mod);
+void mod_tree_remove_thunk(struct module *mod);
+#else
+static inline void mod_tree_insert_thunk(struct module *mod) { }
+static inline void mod_tree_remove_thunk(struct module *mod) { }
+#endif
+
 void module_enable_ro(const struct module *mod, bool after_init);
 void module_enable_nx(const struct module *mod);
 void module_enable_x(const struct module *mod);
--- a/kernel/module/main.c
+++ b/kernel/module/main.c
@@ -1154,6 +1154,7 @@ static void free_module(struct module *m
 	 */
 	mutex_lock(&module_mutex);
 	mod->state = MODULE_STATE_UNFORMED;
+	mod_tree_remove_thunk(mod);
 	mutex_unlock(&module_mutex);
 
 	/* Remove dynamic debug info */
@@ -2770,6 +2771,10 @@ static int load_module(struct load_info
 	if (err < 0)
 		goto free_modinfo;
 
+	mutex_lock(&module_mutex);
+	mod_tree_insert_thunk(mod);
+	mutex_unlock(&module_mutex);
+
 	flush_module_icache(mod);
 
 	/* Setup CFI for the module. */
@@ -2859,6 +2864,7 @@ static int load_module(struct load_info
 	mutex_lock(&module_mutex);
 	/* Unlink carefully: kallsyms could be walking list. */
 	list_del_rcu(&mod->list);
+	mod_tree_remove_thunk(mod);
 	mod_tree_remove(mod);
 	wake_up_all(&module_wq);
 	/* Wait for RCU-sched synchronizing before releasing mod->list. */
--- a/kernel/module/tree_lookup.c
+++ b/kernel/module/tree_lookup.c
@@ -66,11 +66,26 @@ static noinline void __mod_tree_insert(s
 	latch_tree_insert(&node->node, &tree->root, &mod_tree_ops);
 }
 
-static void __mod_tree_remove(struct mod_tree_node *node, struct mod_tree_root *tree)
+static noinline void __mod_tree_remove(struct mod_tree_node *node, struct mod_tree_root *tree)
 {
 	latch_tree_erase(&node->node, &tree->root, &mod_tree_ops);
 }
 
+#ifdef CONFIG_CALL_THUNKS
+void mod_tree_insert_thunk(struct module *mod)
+{
+	mod->thunk_layout.mtn.mod = mod;
+	if (mod->thunk_layout.size)
+		__mod_tree_insert(&mod->thunk_layout.mtn, &mod_tree);
+}
+
+void mod_tree_remove_thunk(struct module *mod)
+{
+	if (mod->thunk_layout.size)
+		__mod_tree_remove(&mod->thunk_layout.mtn, &mod_tree);
+}
+#endif
+
 /*
  * These modifications: insert, remove_init and remove; are serialized by the
  * module_mutex.