[RFC PATCH 4/7] static_call: fix broken static_call_query() for non-exported keys
From: Ard Biesheuvel
Date: Tue Nov 09 2021 - 11:46:26 EST
static_call_query() accesses the func member of the static call key
directly, which means that it is broken for cases where it is used from
a module and the key resides elsewhere and is not exported.
Let's add a helper that returns this value, and export it from the same
module that the key resides in if the key itself is not exported. This
way, we can always get the value regardless of whether the key is
exported or not.
Note that the non-NULL check of &STATIC_CALL_KEY(...) does not typically
result in a load: for ISAs that support relocatable immediates, the
address is patched into the instruction stream.
For example, on ARM/Thumb2, we get
14a: f240 0300 movw r3, #0
14a: R_ARM_THM_MOVW_ABS_NC __SCK__pv_steal_clock
14e: f2c0 0300 movt r3, #0
14e: R_ARM_THM_MOVT_ABS __SCK__pv_steal_clock
152: b10b cbz r3, 158 <foo+0x14>
154: 6818 ldr r0, [r3, #0]
156: 4770 bx lr
158: f7ff bffe b.w 0 <__SCQ__pv_steal_clock>
158: R_ARM_THM_JUMP24 __SCQ__pv_steal_clock
I.e., it performs a conditional branch if zero (CBZ) on a quantity that
was loaded using an MOVW/MOVT move-immediate pair, and either loads the
func member directly, or tail calls __SCQ__pv_steal_clock if they key is
not exported.
Signed-off-by: Ard Biesheuvel <ardb@xxxxxxxxxx>
---
include/linux/static_call.h | 9 ++++++++-
include/linux/static_call_types.h | 18 +++++++++++++++++-
tools/include/linux/static_call_types.h | 18 +++++++++++++++++-
3 files changed, 42 insertions(+), 3 deletions(-)
diff --git a/include/linux/static_call.h b/include/linux/static_call.h
index 3bba0bcba844..391f737496eb 100644
--- a/include/linux/static_call.h
+++ b/include/linux/static_call.h
@@ -149,7 +149,10 @@ extern void arch_static_call_transform(void *site, void *tramp, void *func, bool
STATIC_CALL_TRAMP_ADDR(name), __F); \
})
-#define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func))
+#define EXPORT_STATIC_CALL_QUERY(name, sfx) \
+ typeof(STATIC_CALL_QUERY(name)()) STATIC_CALL_QUERY(name)(void) \
+ { return STATIC_CALL_KEY(name).func; } \
+ EXPORT_SYMBOL ## sfx (STATIC_CALL_QUERY(name))
#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
@@ -200,9 +203,11 @@ extern long __static_call_return0(void);
/* Leave the key unexported, so modules can't change static call targets: */
#define EXPORT_STATIC_CALL_TRAMP(name) \
+ EXPORT_STATIC_CALL_QUERY(name,); \
EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \
EXPORT_STATIC_CALL_GETKEY_HELPER(name)
#define EXPORT_STATIC_CALL_TRAMP_GPL(name) \
+ EXPORT_STATIC_CALL_QUERY(name, _GPL); \
EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \
EXPORT_STATIC_CALL_GETKEY_HELPER(name)
@@ -253,8 +258,10 @@ static inline long __static_call_return0(void)
/* Leave the key unexported, so modules can't change static call targets: */
#define EXPORT_STATIC_CALL_TRAMP(name) \
+ EXPORT_STATIC_CALL_QUERY(name,); \
EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
#define EXPORT_STATIC_CALL_TRAMP_GPL(name) \
+ EXPORT_STATIC_CALL_QUERY(name, _GPL); \
EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
#else /* Generic implementation */
diff --git a/include/linux/static_call_types.h b/include/linux/static_call_types.h
index a31782909e43..2fce9aa8a995 100644
--- a/include/linux/static_call_types.h
+++ b/include/linux/static_call_types.h
@@ -23,6 +23,9 @@
#define STATIC_CALL_GETKEY_PREFIX_LEN (sizeof(STATIC_CALL_GETKEY_PREFIX_STR) - 1)
#define STATIC_CALL_GETKEY(name) __PASTE(STATIC_CALL_GETKEY_PREFIX, name)
+#define STATIC_CALL_QUERY_PREFIX __SCQ__
+#define STATIC_CALL_QUERY(name) __PASTE(STATIC_CALL_QUERY_PREFIX, name)
+
/*
* Flags in the low bits of static_call_site::key.
*/
@@ -43,7 +46,20 @@ struct static_call_site {
#define DECLARE_STATIC_CALL(name, func) \
extern __weak struct static_call_key STATIC_CALL_KEY(name); \
extern __weak struct static_call_key *STATIC_CALL_GETKEY(name)(void);\
- extern typeof(func) STATIC_CALL_TRAMP(name);
+ extern __weak typeof(func) *STATIC_CALL_QUERY(name)(void); \
+ extern typeof(func) STATIC_CALL_TRAMP(name)
+
+#define __static_call_query(name) \
+ ((typeof(STATIC_CALL_QUERY(name)()))READ_ONCE(STATIC_CALL_KEY(name).func))
+
+#ifdef MODULE
+/* the key might not be exported */
+#define static_call_query(name) \
+ (&STATIC_CALL_KEY(name) ? __static_call_query(name) \
+ : STATIC_CALL_QUERY(name)())
+#else
+#define static_call_query(name) __static_call_query(name)
+#endif
#ifdef CONFIG_HAVE_STATIC_CALL
diff --git a/tools/include/linux/static_call_types.h b/tools/include/linux/static_call_types.h
index a31782909e43..2fce9aa8a995 100644
--- a/tools/include/linux/static_call_types.h
+++ b/tools/include/linux/static_call_types.h
@@ -23,6 +23,9 @@
#define STATIC_CALL_GETKEY_PREFIX_LEN (sizeof(STATIC_CALL_GETKEY_PREFIX_STR) - 1)
#define STATIC_CALL_GETKEY(name) __PASTE(STATIC_CALL_GETKEY_PREFIX, name)
+#define STATIC_CALL_QUERY_PREFIX __SCQ__
+#define STATIC_CALL_QUERY(name) __PASTE(STATIC_CALL_QUERY_PREFIX, name)
+
/*
* Flags in the low bits of static_call_site::key.
*/
@@ -43,7 +46,20 @@ struct static_call_site {
#define DECLARE_STATIC_CALL(name, func) \
extern __weak struct static_call_key STATIC_CALL_KEY(name); \
extern __weak struct static_call_key *STATIC_CALL_GETKEY(name)(void);\
- extern typeof(func) STATIC_CALL_TRAMP(name);
+ extern __weak typeof(func) *STATIC_CALL_QUERY(name)(void); \
+ extern typeof(func) STATIC_CALL_TRAMP(name)
+
+#define __static_call_query(name) \
+ ((typeof(STATIC_CALL_QUERY(name)()))READ_ONCE(STATIC_CALL_KEY(name).func))
+
+#ifdef MODULE
+/* the key might not be exported */
+#define static_call_query(name) \
+ (&STATIC_CALL_KEY(name) ? __static_call_query(name) \
+ : STATIC_CALL_QUERY(name)())
+#else
+#define static_call_query(name) __static_call_query(name)
+#endif
#ifdef CONFIG_HAVE_STATIC_CALL
--
2.30.2