From: "Steven Rostedt (Red Hat)" <rost...@goodmis.org>
Some ftrace function tracing callbacks use RCU (perf), thus if
it gets called from tracing a function outside of the RCU tracking,
like in entering or leaving NO_HZ idle/userspace, the RCU read locks
in the callback are useless.
As normal function tracing does not use RCU, and function tracing
happens to help debug RCU, we do not want to limit all callbacks
from tracing these "unsafe" RCU functions. Instead, we create an
infrastructure that will let us mark these unsafe RCU functions
and we will only allow callbacks that explicitly say they do not
use RCU to be called by these functions.
This patch adds the infrastructure to create the hash of functions
that are RCU unsafe. This is done with the FTRACE_UNSAFE_RCU()
macro. It works like EXPORT_SYMBOL() does. Simply place the macro
under the RCU unsafe function:
void func(void)
{
[...]
}
FTRACE_UNSAFE_RCU(func);
Cc: Jiri Olsa <jo...@redhat.com>
Cc: Paul E. McKenney <paul...@linux.vnet.ibm.com>
Signed-off-by: Steven Rostedt <rost...@goodmis.org>
---
include/asm-generic/vmlinux.lds.h | 10 +++
include/linux/ftrace.h | 38 +++++++++++
kernel/trace/ftrace.c | 135 ++++++++++++++++++++++++++++++++++---
3 files changed, 174 insertions(+), 9 deletions(-)
diff --git a/include/asm-generic/vmlinux.lds.h
b/include/asm-generic/vmlinux.lds.h
index 69732d2..fdfddd2 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -93,6 +93,15 @@
#define MCOUNT_REC()
#endif
+#ifdef CONFIG_FUNCTION_TRACER
+#define FTRACE_UNSAFE_RCU() . = ALIGN(8); \
+ VMLINUX_SYMBOL(__start_ftrace_unsafe_rcu) = .; \
+ *(_ftrace_unsafe_rcu) \
+ VMLINUX_SYMBOL(__stop_ftrace_unsafe_rcu) = .;
+#else
+#define FTRACE_UNSAFE_RCU()
+#endif
+
#ifdef CONFIG_TRACE_BRANCH_PROFILING
#define LIKELY_PROFILE()
VMLINUX_SYMBOL(__start_annotated_branch_profile) = .; \
*(_ftrace_annotated_branch)
\
@@ -479,6 +488,7 @@
MEM_DISCARD(init.data) \
KERNEL_CTORS() \
MCOUNT_REC() \
+ FTRACE_UNSAFE_RCU() \
*(.init.rodata) \
FTRACE_EVENTS() \
TRACE_SYSCALLS() \
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 9f15c00..1d17a82 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -92,6 +92,9 @@ typedef void (*ftrace_func_t)(unsigned long ip, unsigned long
parent_ip,
* STUB - The ftrace_ops is just a place holder.
* INITIALIZED - The ftrace_ops has already been initialized (first use time
* register_ftrace_function() is called, it will initialized the
ops)
+ * RCU_SAFE - The callback uses no rcu type locking. This allows the
+ * callback to be called in locations that RCU is not active.
+ * (like going into or coming out of idle NO_HZ)
*/
enum {
FTRACE_OPS_FL_ENABLED = 1 << 0,
@@ -103,6 +106,7 @@ enum {
FTRACE_OPS_FL_RECURSION_SAFE = 1 << 6,
FTRACE_OPS_FL_STUB = 1 << 7,
FTRACE_OPS_FL_INITIALIZED = 1 << 8,
+ FTRACE_OPS_FL_RCU_SAFE = 1 << 9,
};
struct ftrace_ops {
@@ -219,6 +223,40 @@ static inline int ftrace_function_local_disabled(struct
ftrace_ops *ops)
extern void ftrace_stub(unsigned long a0, unsigned long a1,
struct ftrace_ops *op, struct pt_regs *regs);
+/*
+ * In order to speed up boot, save both the name and the
+ * address of the function to find to add to hashes. The
+ * list of function mcount addresses are sorted by the address,
+ * but we still need to use the name to find the actual location
+ * as the mcount call is usually not at the address of the
+ * start of the function.
+ */
+struct ftrace_func_finder {
+ const char *name;
+ unsigned long ip;
+};
+
+/*
+ * For functions that are called when RCU is not tracking the CPU
+ * (like for entering or leaving NO_HZ mode, and RCU then ignores
+ * the CPU), they need to be marked with FTRACE_UNSAFE_RCU().
+ * This will prevent all function tracing callbacks from calling
+ * this function unless the callback explicitly states that it
+ * doesn't use RCU with the FTRACE_OPS_FL_RCU_SAFE flag.
+ *
+ * The usage of FTRACE_UNSAFE_RCU() is the same as EXPORT_SYMBOL().
+ * Just place the macro underneath the function that is unsafe.
+ */
+#define FTRACE_UNSAFE_RCU(func) \
+ static struct ftrace_func_finder __ftrace_unsafe_rcu_##func \
+ __initdata = { \
+ .name = #func, \
+ .ip = (unsigned long)func, \
+ }; \
+ struct ftrace_func_finder *__ftrace_unsafe_rcu_ptr_##func \
+ __attribute__((__section__("_ftrace_unsafe_rcu"))) = \
+ &__ftrace_unsafe_rcu_##func
+
#else /* !CONFIG_FUNCTION_TRACER */
/*
* (un)register_ftrace_function must be a macro since the ops parameter
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index a6d098c..3750360 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -1356,6 +1356,23 @@ ftrace_hash_rec_disable(struct ftrace_ops *ops, int
filter_hash);
static void
ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
+static int ftrace_convert_size_to_bits(int size)
+{
+ int bits;
+
+ /*
+ * Make the hash size about 1/2 the # found
+ */
+ for (size /= 2; size; size >>= 1)
+ bits++;
+
+ /* Don't allocate too much */
+ if (bits > FTRACE_HASH_MAX_BITS)
+ bits = FTRACE_HASH_MAX_BITS;
+
+ return bits;
+}
+
static int
ftrace_hash_move(struct ftrace_ops *ops, int enable,
struct ftrace_hash **dst, struct ftrace_hash *src)
@@ -1388,15 +1405,7 @@ ftrace_hash_move(struct ftrace_ops *ops, int enable,
goto out;
}
- /*
- * Make the hash size about 1/2 the # found
- */
- for (size /= 2; size; size >>= 1)
- bits++;
-
- /* Don't allocate too much */
- if (bits > FTRACE_HASH_MAX_BITS)
- bits = FTRACE_HASH_MAX_BITS;
+ bits = ftrace_convert_size_to_bits(size);
ret = -ENOMEM;
new_hash = alloc_ftrace_hash(bits);
@@ -1486,6 +1495,74 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long
ip, void *regs)
}
+static __init int ftrace_find_ip(const void *a, const void *b)
+{
+ const struct dyn_ftrace *key = a;
+ const struct dyn_ftrace *rec = b;
+
+ if (key->ip == rec->ip)
+ return 0;
+
+ if (key->ip < rec->ip) {
+ /* If previous is less than, then this is our func */
+ rec--;
+ if (rec->ip < key->ip)
+ return 0;
+ return -1;
+ }
+
+ /* All else, we are greater */
+ return 1;
+}
+
+/*
+ * Find the mcount caller location given the ip address of the
+ * function that contains it. As the mcount caller is usually
+ * after the mcount itself.
+ *
+ * Done for just core functions at boot.
+ */
+static __init unsigned long ftrace_mcount_from_func(unsigned long ip)
+{
+ struct ftrace_page *pg, *last_pg = NULL;
+ struct dyn_ftrace *rec;
+ struct dyn_ftrace key;
+
+ key.ip = ip;
+
+ for (pg = ftrace_pages_start; pg; last_pg = pg, pg = pg->next) {
+ if (ip >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
+ continue;
+
+ /*
+ * Do the first record manually, as we need to check
+ * the previous record from the previous page
+ */
+ if (ip < pg->records[0].ip) {
+ /* First page? Then we found our record! */
+ if (!last_pg)
+ return pg->records[0].ip;
+
+ rec = &last_pg->records[last_pg->index - 1];
+ if (rec->ip < ip)
+ return pg->records[0].ip;
+
+ /* Not found */
+ return 0;
+ }
+
+ rec = bsearch(&key, pg->records + 1, pg->index - 1,
+ sizeof(struct dyn_ftrace),
+ ftrace_find_ip);
+ if (rec)
+ return rec->ip;
+
+ break;
+ }
+
+ return 0;
+}
+
static int ftrace_cmp_recs(const void *a, const void *b)
{
const struct dyn_ftrace *key = a;
@@ -4211,6 +4288,44 @@ struct notifier_block ftrace_module_exit_nb = {
.priority = INT_MIN, /* Run after anything that can remove kprobes */
};
+extern struct ftrace_func_finder *__start_ftrace_unsafe_rcu[];
+extern struct ftrace_func_finder *__stop_ftrace_unsafe_rcu[];
+
+static struct ftrace_hash *ftrace_unsafe_rcu;
+
+static void __init create_unsafe_rcu_hash(void)
+{
+ struct ftrace_func_finder *finder;
+ struct ftrace_func_finder **p;
+ unsigned long ip;
+ char str[KSYM_SYMBOL_LEN];
+ int count;
+
+ count = __stop_ftrace_unsafe_rcu - __start_ftrace_unsafe_rcu;
+ if (!count)
+ return;
+
+ count = ftrace_convert_size_to_bits(count);
+ ftrace_unsafe_rcu = alloc_ftrace_hash(count);
+
+ for (p = __start_ftrace_unsafe_rcu;
+ p < __stop_ftrace_unsafe_rcu;
+ p++) {
+ finder = *p;
+
+ ip = ftrace_mcount_from_func(finder->ip);
+ if (WARN_ON_ONCE(!ip))
+ continue;
+
+ /* Make sure this is our ip */
+ kallsyms_lookup(ip, NULL, NULL, NULL, str);
+ if (WARN_ON_ONCE(strcmp(str, finder->name) != 0))
+ continue;
+
+ add_hash_entry(ftrace_unsafe_rcu, ip);
+ }
+}
+
extern unsigned long __start_mcount_loc[];
extern unsigned long __stop_mcount_loc[];
@@ -4250,6 +4365,8 @@ void __init ftrace_init(void)
if (ret)
pr_warning("Failed to register trace ftrace module exit
notifier\n");
+ create_unsafe_rcu_hash();
+
set_ftrace_early_filters();
return;
--
1.7.10.4
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
