This primarily speeds up clock_gettime(CLOCK_THREAD_CPUTIME_ID, ..). We
use the following method to compute the thread cpu time:
t0 = process start
t1 = most recent context switch time
t2 = time at which the vsyscall is invoked
thread_cpu_time = sum(time slices between t0 to t1) + (t2 - t1)
= current->se.sum_exec_runtime + now - sched_clock()
At context switch time We stash away
adj_sched_time = sum_exec_runtime - sched_clock()
in a per-cpu struct in the VVAR page and then compute
thread_cpu_time = adj_sched_time + now
All computations are done in nanosecs on systems where TSC is stable. If
TSC is unstable, we fallback to a regular syscall.
Benchmark data:
for (i = 0; i < 100000000; i++) {
clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts);
sum += ts.tv_sec * NSECS_PER_SEC + ts.tv_nsec;
}
Baseline:
real 1m3.428s
user 0m5.190s
sys 0m58.220s
patched:
real 0m4.912s
user 0m4.910s
sys 0m0.000s
This should speed up profilers that need to query thread cpu time a lot
to do fine-grained timestamps.
No statistically significant regression was detected on x86_64 context
switch code. Most archs that don't support vsyscalls will have this code
disabled via jump labels.
Cc: Andy Lutomirski <[email protected]>
Cc: H. Peter Anvin <[email protected]>
Cc: Ingo Molnar <[email protected]>
Signed-off-by: Kumar Sundararajan <[email protected]>
Signed-off-by: Arun Sharma <[email protected]>
Signed-off-by: Chris Mason <[email protected]>
Signed-off-by: Shaohua Li <[email protected]>
---
arch/x86/include/asm/vdso.h | 9 +++++++-
arch/x86/kernel/tsc.c | 14 ++++++++++++
arch/x86/vdso/vclock_gettime.c | 49 ++++++++++++++++++++++++++++++++++++++++++
arch/x86/vdso/vma.c | 4 ++++
kernel/sched/core.c | 3 +++
5 files changed, 78 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h
index 42d6d2c..cbbab3a 100644
--- a/arch/x86/include/asm/vdso.h
+++ b/arch/x86/include/asm/vdso.h
@@ -54,14 +54,21 @@ extern void __init init_vdso_image(const struct vdso_image
*image);
struct vdso_percpu_data {
/* layout: | cpu ID | context switch count | */
unsigned long cpu_cs_count;
+
+ unsigned long long adj_sched_time;
+ unsigned long long cyc2ns_offset;
+ unsigned long cyc2ns;
} ____cacheline_aligned;
struct vdso_data {
- int dummy;
+ unsigned int thread_cputime_disabled;
struct vdso_percpu_data vpercpu[0];
};
extern struct vdso_data vdso_data;
+struct static_key;
+extern struct static_key vcpu_thread_cputime_enabled;
+
#ifdef CONFIG_SMP
#define VDSO_CS_COUNT_BITS \
(sizeof(unsigned long) * 8 - NR_CPUS_BITS)
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index b7e50bb..83f8091 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -21,6 +21,7 @@
#include <asm/hypervisor.h>
#include <asm/nmi.h>
#include <asm/x86_init.h>
+#include <asm/vdso.h>
unsigned int __read_mostly cpu_khz; /* TSC clocks / usec, not used here */
EXPORT_SYMBOL(cpu_khz);
@@ -263,6 +264,11 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int
cpu)
data->cyc2ns_offset = ns_now -
mul_u64_u32_shr(tsc_now, data->cyc2ns_mul, CYC2NS_SCALE_FACTOR);
+#ifdef CONFIG_VDSO_CS_DETECT
+ vdso_data.vpercpu[cpu].cyc2ns = data->cyc2ns_mul;
+ vdso_data.vpercpu[cpu].cyc2ns_offset = data->cyc2ns_offset;
+#endif
+
cyc2ns_write_end(cpu, data);
done:
@@ -989,6 +995,10 @@ void mark_tsc_unstable(char *reason)
tsc_unstable = 1;
clear_sched_clock_stable();
disable_sched_clock_irqtime();
+#ifdef CONFIG_VDSO_CS_DETECT
+ vdso_data.thread_cputime_disabled = 1;
+ static_key_slow_dec(&vcpu_thread_cputime_enabled);
+#endif
pr_info("Marking TSC unstable due to %s\n", reason);
/* Change only the rating, when not registered */
if (clocksource_tsc.mult)
@@ -1202,6 +1212,10 @@ void __init tsc_init(void)
tsc_disabled = 0;
static_key_slow_inc(&__use_tsc);
+#ifdef CONFIG_VDSO_CS_DETECT
+ vdso_data.thread_cputime_disabled = !cpu_has(&boot_cpu_data,
+ X86_FEATURE_RDTSCP);
+#endif
if (!no_sched_irq_time)
enable_sched_clock_irqtime();
diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c
index 438b3be..46c03af2 100644
--- a/arch/x86/vdso/vclock_gettime.c
+++ b/arch/x86/vdso/vclock_gettime.c
@@ -299,6 +299,48 @@ notrace unsigned long __vdso_get_context_switch_count(void)
smp_rmb();
return VVAR(vdso_data).vpercpu[cpu].cpu_cs_count;
}
+
+#define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
+notrace static inline unsigned long long __cycles_2_ns(unsigned long long cyc,
+ unsigned long long scale,
+ unsigned long long offset)
+{
+ unsigned long long ns = offset;
+ ns += mul_u64_u32_shr(cyc, scale, CYC2NS_SCALE_FACTOR);
+ return ns;
+}
+
+notrace static unsigned long do_thread_cpu_time(void)
+{
+ unsigned int p;
+ u_int64_t tscval;
+ unsigned long long adj_sched_time;
+ unsigned long long scale;
+ unsigned long long offset;
+ const struct vdso_data *vp = &VVAR(vdso_data);
+ int cpu;
+ unsigned long cs;
+
+ do {
+ cs = __vdso_get_context_switch_count();
+
+ rdtscpll(tscval, p);
+ cpu = p & VGETCPU_CPU_MASK;
+ adj_sched_time = vp->vpercpu[cpu].adj_sched_time;
+ scale = vp->vpercpu[cpu].cyc2ns;
+ offset = vp->vpercpu[cpu].cyc2ns_offset;
+
+ } while (unlikely(cs != __vdso_get_context_switch_count()));
+
+ return __cycles_2_ns(tscval, scale, offset) + adj_sched_time;
+}
+
+notrace static inline void ns_to_ts(u64 ns, struct timespec *ts)
+{
+ u32 rem;
+ ts->tv_sec = div_u64_rem(ns, NSEC_PER_SEC, &rem);
+ ts->tv_nsec = rem;
+}
#endif
notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
@@ -318,6 +360,13 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct
timespec *ts)
case CLOCK_MONOTONIC_COARSE:
do_monotonic_coarse(ts);
break;
+#if defined(CONFIG_VDSO_CS_DETECT) && defined(CONFIG_X86_64)
+ case CLOCK_THREAD_CPUTIME_ID:
+ if (VVAR(vdso_data).thread_cputime_disabled)
+ goto fallback;
+ ns_to_ts(do_thread_cpu_time(), ts);
+ break;
+#endif
default:
goto fallback;
}
diff --git a/arch/x86/vdso/vma.c b/arch/x86/vdso/vma.c
index 400e454..aefdd93 100644
--- a/arch/x86/vdso/vma.c
+++ b/arch/x86/vdso/vma.c
@@ -10,6 +10,7 @@
#include <linux/init.h>
#include <linux/random.h>
#include <linux/elf.h>
+#include <linux/jump_label.h>
#include <asm/vsyscall.h>
#include <asm/vgtod.h>
#include <asm/proto.h>
@@ -24,6 +25,7 @@ unsigned int __read_mostly vdso64_enabled = 1;
extern unsigned short vdso_sync_cpuid;
DEFINE_VVAR(struct vdso_data, vdso_data);
+struct static_key vcpu_thread_cputime_enabled;
#endif
void __init init_vdso_image(const struct vdso_image *image)
@@ -54,6 +56,8 @@ static int __init init_vdso(void)
init_vdso_image(&vdso_image_x32);
#endif
+ if (!vdso_data.thread_cputime_disabled)
+ static_key_slow_inc(&vcpu_thread_cputime_enabled);
return 0;
}
subsys_initcall(init_vdso);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5e4100a..e0882fb 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2242,6 +2242,9 @@ static void finish_task_switch(struct rq *rq, struct
task_struct *prev)
int cpu = smp_processor_id();
vdso_inc_cpu_cs_count(cpu);
+ if (static_key_false(&vcpu_thread_cputime_enabled))
+ vdso_data.vpercpu[cpu].adj_sched_time =
+ current->se.sum_exec_runtime - sched_clock();
#endif
rq->prev_mm = NULL;
--
1.8.1
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
