Module Name: src
Committed By: msaitoh
Date: Mon Jun 15 09:09:24 UTC 2020
Modified Files:
src/sys/arch/amd64/amd64: cpufunc.S
src/sys/arch/i386/i386: cpufunc.S
src/sys/arch/x86/include: cpu_counter.h cpufunc.h
src/sys/arch/x86/x86: cpu.c hyperv.c tsc.c tsc.h
src/sys/rump/librump/rumpkern/arch/x86: rump_x86_cpu_counter.c
Log Message:
Serialize rdtsc using with lfence, mfence or cpuid to read TSC more precisely.
x86/x86/tsc.c rev. 1.67 reduced cache problem and got big improvement, but it
still has room. I measured the effect of lfence, mfence, cpuid and rdtscp.
The impact to TSC skew and/or drift is:
AMD: mfence > rdtscp > cpuid > lfence-serialize > lfence = nomodify
Intel: lfence > rdtscp > cpuid > nomodify
So, mfence is the best on AMD and lfence is the best on Intel. If it has no
SSE2, we can use cpuid.
NOTE:
- An AMD's document says DE_CFG_LFENCE_SERIALIZE bit can be used for
serializing, but it's not so good.
- On Intel i386(not amd64), it seems the improvement is very little.
- rdtscp instruct can be used as serializing instruction + rdtsc, but
it's not good as [lm]fence. Both Intel and AMD's document say that
the latency of rdtscp is bigger than rdtsc, so I suspect the difference
of the result comes from it.
To generate a diff of this commit:
cvs rdiff -u -r1.60 -r1.61 src/sys/arch/amd64/amd64/cpufunc.S
cvs rdiff -u -r1.46 -r1.47 src/sys/arch/i386/i386/cpufunc.S
cvs rdiff -u -r1.6 -r1.7 src/sys/arch/x86/include/cpu_counter.h
cvs rdiff -u -r1.40 -r1.41 src/sys/arch/x86/include/cpufunc.h
cvs rdiff -u -r1.193 -r1.194 src/sys/arch/x86/x86/cpu.c
cvs rdiff -u -r1.9 -r1.10 src/sys/arch/x86/x86/hyperv.c
cvs rdiff -u -r1.50 -r1.51 src/sys/arch/x86/x86/tsc.c
cvs rdiff -u -r1.6 -r1.7 src/sys/arch/x86/x86/tsc.h
cvs rdiff -u -r1.1 -r1.2 \
src/sys/rump/librump/rumpkern/arch/x86/rump_x86_cpu_counter.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/cpufunc.S
diff -u src/sys/arch/amd64/amd64/cpufunc.S:1.60 src/sys/arch/amd64/amd64/cpufunc.S:1.61
--- src/sys/arch/amd64/amd64/cpufunc.S:1.60 Sat Jun 13 23:58:51 2020
+++ src/sys/arch/amd64/amd64/cpufunc.S Mon Jun 15 09:09:23 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.S,v 1.60 2020/06/13 23:58:51 ad Exp $ */
+/* $NetBSD: cpufunc.S,v 1.61 2020/06/15 09:09:23 msaitoh Exp $ */
/*
* Copyright (c) 1998, 2007, 2008, 2020 The NetBSD Foundation, Inc.
@@ -207,38 +207,62 @@ END(x86_hotpatch)
#endif /* !XENPV */
/*
- * Could be exact same as cpu_counter, but KMSAN needs to have the correct
- * size of the return value.
+ * cpu_counter and cpu_counter32 could be exact same, but KMSAN needs to have
+ * the correct size of the return value.
*/
-ENTRY(cpu_counter32)
- movq CPUVAR(CURLWP), %rcx
-1:
- movq L_NCSW(%rcx), %rdi
- rdtsc
- addl CPUVAR(CC_SKEW), %eax
- cmpq %rdi, L_NCSW(%rcx)
- jne 2f
- KMSAN_INIT_RET(4)
- ret
-2:
- jmp 1b
-END(cpu_counter32)
+#define SERIALIZE_lfence lfence
+#define SERIALIZE_mfence mfence
-ENTRY(cpu_counter)
- movq CPUVAR(CURLWP), %rcx
-1:
- movq L_NCSW(%rcx), %rdi
- rdtsc
- shlq $32, %rdx
- orq %rdx, %rax
- addq CPUVAR(CC_SKEW), %rax
- cmpq %rdi, L_NCSW(%rcx)
- jne 2f
- KMSAN_INIT_RET(8)
- ret
-2:
- jmp 1b
-END(cpu_counter)
+#define ADD_counter32 addl CPUVAR(CC_SKEW), %eax
+#define ADD_counter shlq $32, %rdx ;\
+ orq %rdx, %rax ;\
+ addq CPUVAR(CC_SKEW), %rax
+
+#define RSIZE_counter32 4
+#define RSIZE_counter 8
+
+#define CPU_COUNTER_FENCE(counter, fence) \
+ENTRY(cpu_ ## counter ## _ ## fence) ;\
+ movq CPUVAR(CURLWP), %rcx ;\
+1: ;\
+ movq L_NCSW(%rcx), %rdi ;\
+ SERIALIZE_ ## fence ;\
+ rdtsc ;\
+ ADD_ ## counter ;\
+ cmpq %rdi, L_NCSW(%rcx) ;\
+ jne 2f ;\
+ KMSAN_INIT_RET(RSIZE_ ## counter) ;\
+ ret ;\
+2: ;\
+ jmp 1b ;\
+END(cpu_ ## counter ## _ ## fence)
+
+CPU_COUNTER_FENCE(counter, lfence)
+CPU_COUNTER_FENCE(counter, mfence)
+CPU_COUNTER_FENCE(counter32, lfence)
+CPU_COUNTER_FENCE(counter32, mfence)
+
+#define CPU_COUNTER_CPUID(counter) \
+ENTRY(cpu_ ## counter ## _cpuid) ;\
+ movq %rbx, %r9 ;\
+ movq CPUVAR(CURLWP), %r8 ;\
+1: ;\
+ movq L_NCSW(%r8), %rdi ;\
+ xor %eax, %eax ;\
+ cpuid ;\
+ rdtsc ;\
+ ADD_ ## counter ;\
+ cmpq %rdi, L_NCSW(%r8) ;\
+ jne 2f ;\
+ movq %r9, %rbx ;\
+ KMSAN_INIT_RET(RSIZE_ ## counter) ;\
+ ret ;\
+2: ;\
+ jmp 1b ;\
+END(cpu_ ## counter ## _cpuid)
+
+CPU_COUNTER_CPUID(counter)
+CPU_COUNTER_CPUID(counter32)
ENTRY(rdmsr_safe)
movq CPUVAR(CURLWP), %r8
Index: src/sys/arch/i386/i386/cpufunc.S
diff -u src/sys/arch/i386/i386/cpufunc.S:1.46 src/sys/arch/i386/i386/cpufunc.S:1.47
--- src/sys/arch/i386/i386/cpufunc.S:1.46 Sat Jun 13 23:58:52 2020
+++ src/sys/arch/i386/i386/cpufunc.S Mon Jun 15 09:09:23 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.S,v 1.46 2020/06/13 23:58:52 ad Exp $ */
+/* $NetBSD: cpufunc.S,v 1.47 2020/06/15 09:09:23 msaitoh Exp $ */
/*-
* Copyright (c) 1998, 2007, 2020 The NetBSD Foundation, Inc.
@@ -38,7 +38,7 @@
#include <sys/errno.h>
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: cpufunc.S,v 1.46 2020/06/13 23:58:52 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpufunc.S,v 1.47 2020/06/15 09:09:23 msaitoh Exp $");
#include "opt_xen.h"
@@ -163,23 +163,59 @@ ENTRY(msr_onfault)
ret
END(msr_onfault)
-ENTRY(cpu_counter)
- pushl %ebx
- movl CPUVAR(CURLWP), %ecx
-1:
- movl L_NCSW(%ecx), %ebx
- rdtsc
- addl CPUVAR(CC_SKEW), %eax
- adcl CPUVAR(CC_SKEW+4), %edx
- cmpl %ebx, L_NCSW(%ecx)
- jne 2f
- popl %ebx
- ret
-2:
- jmp 1b
-END(cpu_counter)
+#define ADD_counter32 addl CPUVAR(CC_SKEW), %eax
+#define ADD_counter ADD_counter32 ;\
+ adcl CPUVAR(CC_SKEW+4), %edx
+
+#define SERIALIZE_lfence lfence
+#define SERIALIZE_mfence mfence
+
+#define CPU_COUNTER_FENCE(counter, fence) \
+ENTRY(cpu_ ## counter ## _ ## fence) ;\
+ pushl %ebx ;\
+ movl CPUVAR(CURLWP), %ecx ;\
+1: ;\
+ movl L_NCSW(%ecx), %ebx ;\
+ SERIALIZE_ ## fence ;\
+ rdtsc ;\
+ ADD_ ## counter ;\
+ cmpl %ebx, L_NCSW(%ecx) ;\
+ jne 2f ;\
+ popl %ebx ;\
+ ret ;\
+2: ;\
+ jmp 1b ;\
+END(cpu_ ## counter ## _ ## fence)
+
+CPU_COUNTER_FENCE(counter, lfence)
+CPU_COUNTER_FENCE(counter, mfence)
+CPU_COUNTER_FENCE(counter32, lfence)
+CPU_COUNTER_FENCE(counter32, mfence)
+
+#define CPU_COUNTER_CPUID(counter) \
+ENTRY(cpu_ ## counter ## _cpuid) ;\
+ pushl %ebx ;\
+ pushl %esi ;\
+ movl CPUVAR(CURLWP), %ecx ;\
+1: ;\
+ movl L_NCSW(%ecx), %esi ;\
+ pushl %ecx ;\
+ xor %eax, %eax ;\
+ cpuid ;\
+ rdtsc ;\
+ ADD_ ## counter ;\
+ popl %ecx ;\
+ cmpl %esi, L_NCSW(%ecx) ;\
+ jne 2f ;\
+ popl %esi ;\
+ popl %ebx ;\
+ ret ;\
+2: ;\
+ jmp 1b ;\
+END(cpu_ ## counter ##_cpuid)
-STRONG_ALIAS(cpu_counter32, cpu_counter)
+CPU_COUNTER_CPUID(counter)
+CPU_COUNTER_CPUID(counter32)
ENTRY(breakpoint)
pushl %ebp
Index: src/sys/arch/x86/include/cpu_counter.h
diff -u src/sys/arch/x86/include/cpu_counter.h:1.6 src/sys/arch/x86/include/cpu_counter.h:1.7
--- src/sys/arch/x86/include/cpu_counter.h:1.6 Fri May 8 22:01:54 2020
+++ src/sys/arch/x86/include/cpu_counter.h Mon Jun 15 09:09:23 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_counter.h,v 1.6 2020/05/08 22:01:54 ad Exp $ */
+/* $NetBSD: cpu_counter.h,v 1.7 2020/06/15 09:09:23 msaitoh Exp $ */
/*-
* Copyright (c) 2000, 2008 The NetBSD Foundation, Inc.
@@ -34,10 +34,19 @@
#ifdef _KERNEL
-uint64_t cpu_counter(void);
-uint32_t cpu_counter32(void);
-uint64_t cpu_frequency(struct cpu_info *);
-int cpu_hascounter(void);
+#include <sys/lwp.h>
+
+extern uint64_t cpu_frequency(struct cpu_info *);
+extern int cpu_hascounter(void);
+extern uint64_t (*cpu_counter)(void);
+extern uint32_t (*cpu_counter32)(void);
+
+extern uint64_t cpu_counter_cpuid(void);
+extern uint64_t cpu_counter_lfence(void);
+extern uint64_t cpu_counter_mfence(void);
+extern uint32_t cpu_counter32_cpuid(void);
+extern uint32_t cpu_counter32_lfence(void);
+extern uint32_t cpu_counter32_mfence(void);
#endif /* _KERNEL */
Index: src/sys/arch/x86/include/cpufunc.h
diff -u src/sys/arch/x86/include/cpufunc.h:1.40 src/sys/arch/x86/include/cpufunc.h:1.41
--- src/sys/arch/x86/include/cpufunc.h:1.40 Sun Jun 14 16:12:05 2020
+++ src/sys/arch/x86/include/cpufunc.h Mon Jun 15 09:09:23 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.h,v 1.40 2020/06/14 16:12:05 riastradh Exp $ */
+/* $NetBSD: cpufunc.h,v 1.41 2020/06/15 09:09:23 msaitoh Exp $ */
/*
* Copyright (c) 1998, 2007, 2019 The NetBSD Foundation, Inc.
@@ -89,19 +89,37 @@ invpcid(register_t op, uint64_t pcid, va
);
}
-static inline uint64_t
-rdtsc(void)
-{
- uint32_t low, high;
+extern uint64_t (*rdtsc)(void);
- __asm volatile (
- "rdtsc"
- : "=a" (low), "=d" (high)
- :
- );
+#define _SERIALIZE_lfence __asm volatile ("lfence")
+#define _SERIALIZE_mfence __asm volatile ("mfence")
+#define _SERIALIZE_cpuid __asm volatile ("xor %%eax, %%eax;cpuid" ::: \
+ "eax", "ebx", "ecx", "edx");
+
+#define RDTSCFUNC(fence) \
+static inline uint64_t \
+rdtsc_##fence(void) \
+{ \
+ uint32_t low, high; \
+ \
+ _SERIALIZE_##fence; \
+ __asm volatile ( \
+ "rdtsc" \
+ : "=a" (low), "=d" (high) \
+ : \
+ ); \
+ \
+ return (low | ((uint64_t)high << 32)); \
+}
+
+RDTSCFUNC(lfence)
+RDTSCFUNC(mfence)
+RDTSCFUNC(cpuid)
+
+#undef _SERIALIZE_LFENCE
+#undef _SERIALIZE_MFENCE
+#undef _SERIALIZE_CPUID
- return (low | ((uint64_t)high << 32));
-}
#ifndef XENPV
struct x86_hotpatch_source {
Index: src/sys/arch/x86/x86/cpu.c
diff -u src/sys/arch/x86/x86/cpu.c:1.193 src/sys/arch/x86/x86/cpu.c:1.194
--- src/sys/arch/x86/x86/cpu.c:1.193 Sat Jun 13 20:01:27 2020
+++ src/sys/arch/x86/x86/cpu.c Mon Jun 15 09:09:24 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.193 2020/06/13 20:01:27 ad Exp $ */
+/* $NetBSD: cpu.c,v 1.194 2020/06/15 09:09:24 msaitoh Exp $ */
/*
* Copyright (c) 2000-2020 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.193 2020/06/13 20:01:27 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.194 2020/06/15 09:09:24 msaitoh Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -444,6 +444,7 @@ cpu_attach(device_t parent, device_t sel
*/
atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
cpu_intr_init(ci);
+ tsc_setfunc(ci);
cpu_get_tsc_freq(ci);
cpu_init(ci);
#ifdef i386
Index: src/sys/arch/x86/x86/hyperv.c
diff -u src/sys/arch/x86/x86/hyperv.c:1.9 src/sys/arch/x86/x86/hyperv.c:1.10
--- src/sys/arch/x86/x86/hyperv.c:1.9 Sun May 17 11:54:39 2020
+++ src/sys/arch/x86/x86/hyperv.c Mon Jun 15 09:09:24 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: hyperv.c,v 1.9 2020/05/17 11:54:39 nonaka Exp $ */
+/* $NetBSD: hyperv.c,v 1.10 2020/06/15 09:09:24 msaitoh Exp $ */
/*-
* Copyright (c) 2009-2012,2016-2017 Microsoft Corp.
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
#ifdef __KERNEL_RCSID
-__KERNEL_RCSID(0, "$NetBSD: hyperv.c,v 1.9 2020/05/17 11:54:39 nonaka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: hyperv.c,v 1.10 2020/06/15 09:09:24 msaitoh Exp $");
#endif
#ifdef __FBSDID
__FBSDID("$FreeBSD: head/sys/dev/hyperv/vmbus/hyperv.c 331757 2018-03-30 02:25:12Z emaste $");
@@ -178,8 +178,10 @@ struct hyperv_ref_tsc {
static struct hyperv_ref_tsc hyperv_ref_tsc;
+static u_int hyperv_tsc_timecount(struct timecounter *);
+
static struct timecounter hyperv_tsc_timecounter = {
- .tc_get_timecount = NULL, /* based on CPU vendor. */
+ .tc_get_timecount = hyperv_tsc_timecount,
.tc_counter_mask = 0xffffffff,
.tc_frequency = HYPERV_TIMER_FREQ,
.tc_name = "Hyper-V-TSC",
@@ -194,51 +196,45 @@ atomic_load_acq_int(volatile u_int *p)
return r;
}
-#define HYPERV_TSC_TIMECOUNT(fence) \
-static uint64_t \
-hyperv_tc64_tsc_##fence(void) \
-{ \
- struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref; \
- uint32_t seq; \
- \
- while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) { \
- uint64_t disc, ret, tsc; \
- uint64_t scale = tsc_ref->tsc_scale; \
- int64_t ofs = tsc_ref->tsc_ofs; \
- \
- x86_##fence(); \
- tsc = cpu_counter(); \
- \
- /* ret = ((tsc * scale) >> 64) + ofs */ \
- __asm__ __volatile__ ("mulq %3" : \
- "=d" (ret), "=a" (disc) : \
- "a" (tsc), "r" (scale)); \
- ret += ofs; \
- \
- __insn_barrier(); \
- if (tsc_ref->tsc_seq == seq) \
- return ret; \
- \
- /* Sequence changed; re-sync. */ \
- } \
- /* Fallback to the generic timecounter, i.e. rdmsr. */ \
- return rdmsr(MSR_HV_TIME_REF_COUNT); \
-} \
- \
-static u_int \
-hyperv_tsc_timecount_##fence(struct timecounter *tc __unused) \
-{ \
- \
- return hyperv_tc64_tsc_##fence(); \
+static uint64_t
+hyperv_tc64_tsc(void)
+{
+ struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref;
+ uint32_t seq;
+
+ while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {
+ uint64_t disc, ret, tsc;
+ uint64_t scale = tsc_ref->tsc_scale;
+ int64_t ofs = tsc_ref->tsc_ofs;
+
+ tsc = cpu_counter();
+
+ /* ret = ((tsc * scale) >> 64) + ofs */
+ __asm__ __volatile__ ("mulq %3" :
+ "=d" (ret), "=a" (disc) :
+ "a" (tsc), "r" (scale));
+ ret += ofs;
+
+ __insn_barrier();
+ if (tsc_ref->tsc_seq == seq)
+ return ret;
+
+ /* Sequence changed; re-sync. */
+ }
+ /* Fallback to the generic timecounter, i.e. rdmsr. */
+ return rdmsr(MSR_HV_TIME_REF_COUNT);
}
-HYPERV_TSC_TIMECOUNT(lfence);
-HYPERV_TSC_TIMECOUNT(mfence);
+static u_int
+hyperv_tsc_timecount(struct timecounter *tc __unused)
+{
+
+ return hyperv_tc64_tsc();
+}
static bool
hyperv_tsc_tcinit(void)
{
- hyperv_tc64_t tc64 = NULL;
uint64_t orig_msr, msr;
if ((hyperv_features &
@@ -247,24 +243,6 @@ hyperv_tsc_tcinit(void)
(cpu_feature[0] & CPUID_SSE2) == 0) /* SSE2 for mfence/lfence */
return false;
- switch (cpu_vendor) {
- case CPUVENDOR_AMD:
- hyperv_tsc_timecounter.tc_get_timecount =
- hyperv_tsc_timecount_mfence;
- tc64 = hyperv_tc64_tsc_mfence;
- break;
-
- case CPUVENDOR_INTEL:
- hyperv_tsc_timecounter.tc_get_timecount =
- hyperv_tsc_timecount_lfence;
- tc64 = hyperv_tc64_tsc_lfence;
- break;
-
- default:
- /* Unsupport CPU vendors. */
- return false;
- }
-
hyperv_ref_tsc.tsc_ref = (void *)uvm_km_alloc(kernel_map,
PAGE_SIZE, PAGE_SIZE, UVM_KMF_WIRED | UVM_KMF_ZERO);
if (hyperv_ref_tsc.tsc_ref == NULL) {
@@ -287,14 +265,14 @@ hyperv_tsc_tcinit(void)
wrmsr(MSR_HV_REFERENCE_TSC, msr);
/* Install 64 bits timecounter method for other modules to use. */
- hyperv_tc64 = tc64;
+ hyperv_tc64 = hyperv_tc64_tsc;
/* Register "enlightened" timecounter. */
tc_init(&hyperv_tsc_timecounter);
return true;
}
-#endif
+#endif /* __amd64__ */
static void
delay_tc(unsigned int n)
Index: src/sys/arch/x86/x86/tsc.c
diff -u src/sys/arch/x86/x86/tsc.c:1.50 src/sys/arch/x86/x86/tsc.c:1.51
--- src/sys/arch/x86/x86/tsc.c:1.50 Sun Jun 14 23:24:20 2020
+++ src/sys/arch/x86/x86/tsc.c Mon Jun 15 09:09:24 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: tsc.c,v 1.50 2020/06/14 23:24:20 ad Exp $ */
+/* $NetBSD: tsc.c,v 1.51 2020/06/15 09:09:24 msaitoh Exp $ */
/*-
* Copyright (c) 2008, 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: tsc.c,v 1.50 2020/06/14 23:24:20 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: tsc.c,v 1.51 2020/06/15 09:09:24 msaitoh Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -59,6 +59,9 @@ static uint64_t tsc_dummy_cacheline __ca
uint64_t tsc_freq __read_mostly; /* exported for sysctl */
static int64_t tsc_drift_max = 1000; /* max cycles */
static int64_t tsc_drift_observed;
+uint64_t (*rdtsc)(void) = rdtsc_cpuid;
+uint64_t (*cpu_counter)(void) = cpu_counter_cpuid;
+uint32_t (*cpu_counter32)(void) = cpu_counter32_cpuid;
int tsc_user_enabled = 1;
@@ -148,6 +151,51 @@ tsc_is_invariant(void)
return invariant;
}
+/* Setup function porniters for rdtsc() and timecounter(9). */
+void
+tsc_setfunc(struct cpu_info *ci)
+{
+ bool use_lfence, use_mfence;
+
+ use_lfence = use_mfence = false;
+
+ /*
+ * XXX On AMD, we might be able to use lfence for some cases:
+ * a) if MSR_DE_CFG exist and the bit 1 is set.
+ * b) family == 0x0f or 0x11. Those have no MSR_DE_CFG and
+ * lfence is always serializing.
+ *
+ * We don't use it because the test result showed mfence was better
+ * than lfence with MSR_DE_CFG.
+ */
+ if (cpu_vendor == CPUVENDOR_AMD)
+ use_mfence = true;
+ else if (cpu_vendor == CPUVENDOR_INTEL)
+ use_lfence = true;
+
+ /* LFENCE and MFENCE are applicable if SSE2 is set. */
+ if ((ci->ci_feat_val[0] & CPUID_SSE2) == 0)
+ use_lfence = use_mfence = false;
+
+#define TSC_SETFUNC(fence) \
+ do { \
+ rdtsc = rdtsc_##fence; \
+ cpu_counter = cpu_counter_##fence; \
+ cpu_counter32 = cpu_counter32_##fence; \
+ } while (/* CONSTCOND */ 0)
+
+
+ if (use_lfence)
+ TSC_SETFUNC(lfence);
+ else if (use_mfence)
+ TSC_SETFUNC(mfence);
+ else
+ TSC_SETFUNC(cpuid);
+
+ aprint_verbose_dev(ci->ci_dev, "Use %s to serialize rdtsc\n",
+ use_lfence ? "lfence" : (use_mfence ? "mfence" : "cpuid"));
+}
+
/*
* Initialize timecounter(9) and DELAY() function of TSC.
*
Index: src/sys/arch/x86/x86/tsc.h
diff -u src/sys/arch/x86/x86/tsc.h:1.6 src/sys/arch/x86/x86/tsc.h:1.7
--- src/sys/arch/x86/x86/tsc.h:1.6 Mon Oct 2 19:23:16 2017
+++ src/sys/arch/x86/x86/tsc.h Mon Jun 15 09:09:24 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: tsc.h,v 1.6 2017/10/02 19:23:16 maxv Exp $ */
+/* $NetBSD: tsc.h,v 1.7 2020/06/15 09:09:24 msaitoh Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -27,6 +27,7 @@
*/
bool tsc_is_invariant(void);
+void tsc_setfunc(struct cpu_info *);
void tsc_tc_init(void);
void tsc_sync_ap(struct cpu_info *);
void tsc_sync_bp(struct cpu_info *);
Index: src/sys/rump/librump/rumpkern/arch/x86/rump_x86_cpu_counter.c
diff -u src/sys/rump/librump/rumpkern/arch/x86/rump_x86_cpu_counter.c:1.1 src/sys/rump/librump/rumpkern/arch/x86/rump_x86_cpu_counter.c:1.2
--- src/sys/rump/librump/rumpkern/arch/x86/rump_x86_cpu_counter.c:1.1 Wed Feb 12 22:28:43 2014
+++ src/sys/rump/librump/rumpkern/arch/x86/rump_x86_cpu_counter.c Mon Jun 15 09:09:24 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: rump_x86_cpu_counter.c,v 1.1 2014/02/12 22:28:43 pooka Exp $ */
+/* $NetBSD: rump_x86_cpu_counter.c,v 1.2 2020/06/15 09:09:24 msaitoh Exp $ */
/*
* Copyright (c) 2010 Antti Kantee. All Rights Reserved.
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: rump_x86_cpu_counter.c,v 1.1 2014/02/12 22:28:43 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: rump_x86_cpu_counter.c,v 1.2 2020/06/15 09:09:24 msaitoh Exp $");
#include <sys/param.h>
@@ -45,22 +45,23 @@ cpu_hascounter(void)
}
uint64_t
-cpu_counter(void)
+cpu_frequency(struct cpu_info *ci)
{
return 0;
}
-uint32_t
-cpu_counter32(void)
+static uint64_t
+nullfunc64(void)
{
-
return 0;
}
-uint64_t
-cpu_frequency(struct cpu_info *ci)
+static uint32_t
+nullfunc32(void)
{
-
return 0;
}
+
+uint64_t (*cpu_counter)(void) = nullfunc64;
+uint32_t (*cpu_counter32)(void) = nullfunc32;
