From: Josh Poimboeuf <[email protected]> Static calls are a replacement for global function pointers. They use code patching to allow direct calls to be used instead of indirect calls. They give the flexibility of function pointers, but with improved performance. This is especially important for cases where retpolines would otherwise be used, as retpolines can significantly impact performance.
The concept and code are an extension of previous work done by Ard Biesheuvel and Steven Rostedt: https://lkml.kernel.org/r/[email protected] https://lkml.kernel.org/r/[email protected] There are two implementations, depending on arch support: 1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL) 2) basic function pointers For more details, see the comments in include/linux/static_call.h. Cc: [email protected] Cc: Steven Rostedt <[email protected]> Cc: Julia Cartwright <[email protected]> Cc: Ingo Molnar <[email protected]> Cc: Ard Biesheuvel <[email protected]> Cc: Jason Baron <[email protected]> Cc: Rasmus Villemoes <[email protected]> Cc: Daniel Bristot de Oliveira <[email protected]> Cc: Linus Torvalds <[email protected]> Cc: Jiri Kosina <[email protected]> Cc: Edward Cree <[email protected]> Cc: Thomas Gleixner <[email protected]> Cc: Masami Hiramatsu <[email protected]> Cc: Borislav Petkov <[email protected]> Cc: David Laight <[email protected]> Cc: Jessica Yu <[email protected]> Cc: Nadav Amit <[email protected]> Cc: Andy Lutomirski <[email protected]> Cc: "H. Peter Anvin" <[email protected]> Signed-off-by: Josh Poimboeuf <[email protected]> Signed-off-by: Peter Zijlstra (Intel) <[email protected]> Link: https://lkml.kernel.org/r/a01f733889ebf4bc447507ab8041a60378eaa89f.1547073843.git.jpoim...@redhat.com --- arch/Kconfig | 3 include/linux/static_call.h | 135 ++++++++++++++++++++++++++++++++++++++ include/linux/static_call_types.h | 13 +++ 3 files changed, 151 insertions(+) create mode 100644 include/linux/static_call.h create mode 100644 include/linux/static_call_types.h --- a/arch/Kconfig +++ b/arch/Kconfig @@ -927,6 +927,9 @@ config LOCK_EVENT_COUNTS the chance of application behavior change because of timing differences. The counts are reported via debugfs. +config HAVE_STATIC_CALL + bool + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig" --- /dev/null +++ b/include/linux/static_call.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_STATIC_CALL_H +#define _LINUX_STATIC_CALL_H + +/* + * Static call support + * + * Static calls use code patching to hard-code function pointers into direct + * branch instructions. They give the flexibility of function pointers, but + * with improved performance. This is especially important for cases where + * retpolines would otherwise be used, as retpolines can significantly impact + * performance. + * + * + * API overview: + * + * DECLARE_STATIC_CALL(key, func); + * DEFINE_STATIC_CALL(key, func); + * static_call(key, args...); + * static_call_update(key, func); + * + * + * Usage example: + * + * # Start with the following functions (with identical prototypes): + * int func_a(int arg1, int arg2); + * int func_b(int arg1, int arg2); + * + * # Define a 'my_key' reference, associated with func_a() by default + * DEFINE_STATIC_CALL(my_key, func_a); + * + * # Call func_a() + * static_call(my_key, arg1, arg2); + * + * # Update 'my_key' to point to func_b() + * static_call_update(my_key, func_b); + * + * # Call func_b() + * static_call(my_key, arg1, arg2); + * + * + * Implementation details: + * + * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). + * Otherwise basic indirect calls are used (with function pointers). + * + * Each static_call() site calls into a trampoline associated with the key. + * The trampoline has a direct branch to the default function. Updates to a + * key will modify the trampoline's branch destination. + */ + +#include <linux/types.h> +#include <linux/cpu.h> +#include <linux/static_call_types.h> + +#ifdef CONFIG_HAVE_STATIC_CALL +#include <asm/static_call.h> +extern void arch_static_call_transform(void *site, void *tramp, void *func); +#endif + + +#define DECLARE_STATIC_CALL(key, func) \ + extern struct static_call_key key; \ + extern typeof(func) STATIC_CALL_TRAMP(key) + + +#if defined(CONFIG_HAVE_STATIC_CALL) + +struct static_call_key { + void *func, *tramp; +}; + +#define DEFINE_STATIC_CALL(key, _func) \ + DECLARE_STATIC_CALL(key, _func); \ + struct static_call_key key = { \ + .func = _func, \ + .tramp = STATIC_CALL_TRAMP(key), \ + }; \ + ARCH_DEFINE_STATIC_CALL_TRAMP(key, _func) + +#define static_call(key, args...) STATIC_CALL_TRAMP(key)(args) + +static inline void __static_call_update(struct static_call_key *key, void *func) +{ + cpus_read_lock(); + WRITE_ONCE(key->func, func); + arch_static_call_transform(NULL, key->tramp, func); + cpus_read_unlock(); +} + +#define static_call_update(key, func) \ +({ \ + BUILD_BUG_ON(!__same_type(func, STATIC_CALL_TRAMP(key))); \ + __static_call_update(&key, func); \ +}) + +#define EXPORT_STATIC_CALL(key) \ + EXPORT_SYMBOL(STATIC_CALL_TRAMP(key)) + +#define EXPORT_STATIC_CALL_GPL(key) \ + EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(key)) + + +#else /* Generic implementation */ + +struct static_call_key { + void *func; +}; + +#define DEFINE_STATIC_CALL(key, _func) \ + DECLARE_STATIC_CALL(key, _func); \ + struct static_call_key key = { \ + .func = _func, \ + } + +#define static_call(key, args...) \ + ((typeof(STATIC_CALL_TRAMP(key))*)(key.func))(args) + +static inline void __static_call_update(struct static_call_key *key, void *func) +{ + WRITE_ONCE(key->func, func); +} + +#define static_call_update(key, func) \ +({ \ + BUILD_BUG_ON(!__same_type(func, STATIC_CALL_TRAMP(key))); \ + __static_call_update(&key, func); \ +}) + +#define EXPORT_STATIC_CALL(key) EXPORT_SYMBOL(key) +#define EXPORT_STATIC_CALL_GPL(key) EXPORT_SYMBOL_GPL(key) + +#endif /* CONFIG_HAVE_STATIC_CALL */ + +#endif /* _LINUX_STATIC_CALL_H */ --- /dev/null +++ b/include/linux/static_call_types.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _STATIC_CALL_TYPES_H +#define _STATIC_CALL_TYPES_H + +#include <linux/stringify.h> + +#define STATIC_CALL_TRAMP_PREFIX ____static_call_tramp_ +#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) + +#define STATIC_CALL_TRAMP(key) __PASTE(STATIC_CALL_TRAMP_PREFIX, key) +#define STATIC_CALL_TRAMP_STR(key) __stringify(STATIC_CALL_TRAMP(key)) + +#endif /* _STATIC_CALL_TYPES_H */
