Module Name: src
Committed By: ad
Date: Fri May 8 22:01:55 UTC 2020
Modified Files:
src/sys/arch/x86/include: cpu_counter.h
src/sys/arch/x86/x86: cpu.c tsc.c
src/sys/dev/ic: hpet.c hpetvar.h
Log Message:
Fix the TSC timecounter (on the systems I have access to):
- Make the early i8254-based calculation of frequency a bit more accurate.
- Keep track of how far the HPET & TSC advance between HPET attach and
secondary CPU boot, and use to compute an accurate value before attaching
the timecounter. Initial idea from joerg@.
- When determining skew and drift between CPUs, make each measurement 1000
times and pick the lowest observed value. Increase the error threshold to
1000 clock cycles.
- Use the frequency computed on the boot CPU for secondary CPUs too.
- Remove cpu_counter_serializing().
To generate a diff of this commit:
cvs rdiff -u -r1.5 -r1.6 src/sys/arch/x86/include/cpu_counter.h
cvs rdiff -u -r1.189 -r1.190 src/sys/arch/x86/x86/cpu.c
cvs rdiff -u -r1.43 -r1.44 src/sys/arch/x86/x86/tsc.c
cvs rdiff -u -r1.15 -r1.16 src/sys/dev/ic/hpet.c
cvs rdiff -u -r1.6 -r1.7 src/sys/dev/ic/hpetvar.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/include/cpu_counter.h
diff -u src/sys/arch/x86/include/cpu_counter.h:1.5 src/sys/arch/x86/include/cpu_counter.h:1.6
--- src/sys/arch/x86/include/cpu_counter.h:1.5 Wed Feb 2 12:26:42 2011
+++ src/sys/arch/x86/include/cpu_counter.h Fri May 8 22:01:54 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_counter.h,v 1.5 2011/02/02 12:26:42 bouyer Exp $ */
+/* $NetBSD: cpu_counter.h,v 1.6 2020/05/08 22:01:54 ad Exp $ */
/*-
* Copyright (c) 2000, 2008 The NetBSD Foundation, Inc.
@@ -35,7 +35,6 @@
#ifdef _KERNEL
uint64_t cpu_counter(void);
-uint64_t cpu_counter_serializing(void);
uint32_t cpu_counter32(void);
uint64_t cpu_frequency(struct cpu_info *);
int cpu_hascounter(void);
Index: src/sys/arch/x86/x86/cpu.c
diff -u src/sys/arch/x86/x86/cpu.c:1.189 src/sys/arch/x86/x86/cpu.c:1.190
--- src/sys/arch/x86/x86/cpu.c:1.189 Sat May 2 16:44:36 2020
+++ src/sys/arch/x86/x86/cpu.c Fri May 8 22:01:55 2020
@@ -1,7 +1,7 @@
-/* $NetBSD: cpu.c,v 1.189 2020/05/02 16:44:36 bouyer Exp $ */
+/* $NetBSD: cpu.c,v 1.190 2020/05/08 22:01:55 ad Exp $ */
/*
- * Copyright (c) 2000-2012 NetBSD Foundation, Inc.
+ * Copyright (c) 2000-2020 NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.189 2020/05/02 16:44:36 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.190 2020/05/08 22:01:55 ad Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -73,6 +73,7 @@ __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.18
#include "lapic.h"
#include "ioapic.h"
#include "acpica.h"
+#include "hpet.h"
#include <sys/param.h>
#include <sys/proc.h>
@@ -119,6 +120,7 @@ __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.18
#endif
#include <dev/ic/mc146818reg.h>
+#include <dev/ic/hpetvar.h>
#include <i386/isa/nvram.h>
#include <dev/isa/isareg.h>
@@ -433,8 +435,14 @@ cpu_attach(device_t parent, device_t sel
* must be done to allow booting other processors.
*/
if (!again) {
+ /* Make sure DELAY() (likely i8254_delay()) is initialized. */
+ DELAY(1);
+
+ /*
+ * Basic init. Compute an approximate frequency for the TSC
+ * using the i8254. If there's a HPET we'll redo it later.
+ */
atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
- /* Basic init. */
cpu_intr_init(ci);
cpu_get_tsc_freq(ci);
cpu_init(ci);
@@ -451,8 +459,6 @@ cpu_attach(device_t parent, device_t sel
lapic_calibrate_timer(ci);
}
#endif
- /* Make sure DELAY() is initialized. */
- DELAY(1);
kcsan_cpu_init(ci);
again = true;
}
@@ -718,7 +724,6 @@ cpu_init(struct cpu_info *ci)
if (ci != &cpu_info_primary) {
/* Synchronize TSC */
- wbinvd();
atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
tsc_sync_ap(ci);
} else {
@@ -734,6 +739,14 @@ cpu_boot_secondary_processors(void)
kcpuset_t *cpus;
u_long i;
+#if NHPET > 0
+ /* Use HPET delay, and re-calibrate TSC on boot CPU using HPET. */
+ if (hpet_delay_p() && x86_delay == i8254_delay) {
+ delay_func = x86_delay = hpet_delay;
+ cpu_get_tsc_freq(curcpu());
+ }
+#endif
+
/* Now that we know the number of CPUs, patch the text segment. */
x86_patch(false);
@@ -842,7 +855,6 @@ cpu_start_secondary(struct cpu_info *ci)
*/
psl = x86_read_psl();
x86_disable_intr();
- wbinvd();
tsc_sync_bp(ci);
x86_write_psl(psl);
}
@@ -873,7 +885,6 @@ cpu_boot_secondary(struct cpu_info *ci)
drift = ci->ci_data.cpu_cc_skew;
psl = x86_read_psl();
x86_disable_intr();
- wbinvd();
tsc_sync_bp(ci);
x86_write_psl(psl);
drift -= ci->ci_data.cpu_cc_skew;
@@ -919,7 +930,6 @@ cpu_hatch(void *v)
* Synchronize the TSC for the first time. Note that interrupts are
* off at this point.
*/
- wbinvd();
atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
tsc_sync_ap(ci);
@@ -1310,21 +1320,43 @@ cpu_shutdown(device_t dv, int how)
void
cpu_get_tsc_freq(struct cpu_info *ci)
{
- uint64_t freq = 0, last_tsc;
+ uint64_t freq = 0, t0, t1;
+ int64_t overhead;
- if (cpu_hascounter())
+ if ((ci->ci_flags & CPUF_PRIMARY) != 0 && cpu_hascounter()) {
freq = cpu_tsc_freq_cpuid(ci);
+#if NHPET > 0
+ if (freq == 0)
+ freq = hpet_tsc_freq();
+#endif
+ if (freq == 0) {
+ /*
+ * Work out the approximate overhead involved below.
+ * Discard the result of the first go around the
+ * loop.
+ */
+ overhead = 0;
+ for (int i = 0; i <= 8; i++) {
+ t0 = cpu_counter();
+ x86_delay(0);
+ t1 = cpu_counter();
+ if (i > 0) {
+ overhead += (t1 - t0);
+ }
+ }
+ overhead >>= 3;
- if (freq != 0) {
- /* Use TSC frequency taken from CPUID. */
- ci->ci_data.cpu_cc_freq = freq;
+ /* Now do the calibration. */
+ t0 = cpu_counter();
+ x86_delay(100000);
+ t1 = cpu_counter();
+ freq = (t1 - t0 - overhead) * 10;
+ }
} else {
- /* Calibrate TSC frequency. */
- last_tsc = cpu_counter_serializing();
- delay_func(100000);
- ci->ci_data.cpu_cc_freq =
- (cpu_counter_serializing() - last_tsc) * 10;
+ freq = cpu_info_primary.ci_data.cpu_cc_freq;
}
+
+ ci->ci_data.cpu_cc_freq = freq;
}
void
Index: src/sys/arch/x86/x86/tsc.c
diff -u src/sys/arch/x86/x86/tsc.c:1.43 src/sys/arch/x86/x86/tsc.c:1.44
--- src/sys/arch/x86/x86/tsc.c:1.43 Sat Apr 25 15:26:18 2020
+++ src/sys/arch/x86/x86/tsc.c Fri May 8 22:01:55 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: tsc.c,v 1.43 2020/04/25 15:26:18 bouyer Exp $ */
+/* $NetBSD: tsc.c,v 1.44 2020/05/08 22:01:55 ad Exp $ */
/*-
* Copyright (c) 2008, 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: tsc.c,v 1.43 2020/04/25 15:26:18 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: tsc.c,v 1.44 2020/05/08 22:01:55 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -47,12 +47,14 @@ __KERNEL_RCSID(0, "$NetBSD: tsc.c,v 1.43
#include "tsc.h"
+#define TSC_SYNC_ROUNDS 1000
+#define ABS(a) ((a) >= 0 ? (a) : -(a))
+
u_int tsc_get_timecount(struct timecounter *);
uint64_t tsc_freq; /* exported for sysctl */
-static int64_t tsc_drift_max = 250; /* max cycles */
+static int64_t tsc_drift_max = 1000; /* max cycles */
static int64_t tsc_drift_observed;
-static bool tsc_good;
int tsc_user_enabled = 1;
@@ -158,9 +160,6 @@ tsc_tc_init(void)
ci = curcpu();
tsc_freq = ci->ci_data.cpu_cc_freq;
- tsc_good = (cpu_feature[0] & CPUID_MSR) != 0 &&
- (rdmsr(MSR_TSC) != 0 || rdmsr(MSR_TSC) != 0);
-
invariant = tsc_is_invariant();
if (!invariant) {
aprint_debug("TSC not known invariant on this CPU\n");
@@ -206,13 +205,12 @@ tsc_read_bp(struct cpu_info *ci, uint64_
/* Flag it and read our TSC. */
atomic_or_uint(&ci->ci_flags, CPUF_SYNCTSC);
- bptsc = (rdtsc() >> 1);
/* Wait for remote to complete, and read ours again. */
while ((ci->ci_flags & CPUF_SYNCTSC) != 0) {
__insn_barrier();
}
- bptsc += (rdtsc() >> 1);
+ bptsc = rdtsc();
/* Wait for the results to come in. */
while (tsc_sync_cpu == ci) {
@@ -229,17 +227,21 @@ tsc_read_bp(struct cpu_info *ci, uint64_
void
tsc_sync_bp(struct cpu_info *ci)
{
- int64_t bptsc, aptsc, bsum = 0, asum = 0;
+ int64_t bptsc, aptsc, val, diff;
- tsc_read_bp(ci, &bptsc, &aptsc); /* discarded - cache effects */
- for (int i = 0; i < 8; i++) {
+ if (!cpu_hascounter())
+ return;
+
+ val = INT64_MAX;
+ for (int i = 0; i < TSC_SYNC_ROUNDS; i++) {
tsc_read_bp(ci, &bptsc, &aptsc);
- bsum += bptsc;
- asum += aptsc;
+ diff = bptsc - aptsc;
+ if (ABS(diff) < ABS(val)) {
+ val = diff;
+ }
}
- /* Compute final value to adjust for skew. */
- ci->ci_data.cpu_cc_skew = (bsum - asum) >> 3;
+ ci->ci_data.cpu_cc_skew = val;
}
/*
@@ -255,11 +257,10 @@ tsc_post_ap(struct cpu_info *ci)
while ((ci->ci_flags & CPUF_SYNCTSC) == 0) {
__insn_barrier();
}
- tsc = (rdtsc() >> 1);
/* Instruct primary to read its counter. */
atomic_and_uint(&ci->ci_flags, ~CPUF_SYNCTSC);
- tsc += (rdtsc() >> 1);
+ tsc = rdtsc();
/* Post result. Ensure the whole value goes out atomically. */
(void)atomic_swap_64(&tsc_sync_val, tsc);
@@ -273,8 +274,10 @@ void
tsc_sync_ap(struct cpu_info *ci)
{
- tsc_post_ap(ci);
- for (int i = 0; i < 8; i++) {
+ if (!cpu_hascounter())
+ return;
+
+ for (int i = 0; i < TSC_SYNC_ROUNDS; i++) {
tsc_post_ap(ci);
}
}
@@ -321,12 +324,3 @@ cpu_hascounter(void)
return cpu_feature[0] & CPUID_TSC;
}
-
-uint64_t
-cpu_counter_serializing(void)
-{
- if (tsc_good)
- return rdmsr(MSR_TSC);
- else
- return cpu_counter();
-}
Index: src/sys/dev/ic/hpet.c
diff -u src/sys/dev/ic/hpet.c:1.15 src/sys/dev/ic/hpet.c:1.16
--- src/sys/dev/ic/hpet.c:1.15 Fri Apr 24 22:25:07 2020
+++ src/sys/dev/ic/hpet.c Fri May 8 22:01:54 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: hpet.c,v 1.15 2020/04/24 22:25:07 ad Exp $ */
+/* $NetBSD: hpet.c,v 1.16 2020/05/08 22:01:54 ad Exp $ */
/*
* Copyright (c) 2006 Nicolas Joly
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: hpet.c,v 1.15 2020/04/24 22:25:07 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: hpet.c,v 1.16 2020/05/08 22:01:54 ad Exp $");
#include <sys/systm.h>
#include <sys/device.h>
@@ -45,6 +45,8 @@ __KERNEL_RCSID(0, "$NetBSD: hpet.c,v 1.1
#include <sys/bus.h>
#include <sys/lock.h>
+#include <machine/cpu_counter.h>
+
#include <dev/ic/hpetreg.h>
#include <dev/ic/hpetvar.h>
@@ -52,6 +54,8 @@ static u_int hpet_get_timecount(struct t
static bool hpet_resume(device_t, const pmf_qual_t *);
static struct hpet_softc *hpet0 __read_mostly;
+static uint32_t hpet_attach_val;
+static uint64_t hpet_attach_tsc;
int
hpet_detach(device_t dv, int flags)
@@ -143,6 +147,14 @@ hpet_attach_subr(device_t dv)
eval = bus_space_read_4(sc->sc_memt, sc->sc_memh, HPET_MCOUNT_LO);
val = eval - sval;
sc->sc_adj = (int64_t)val * sc->sc_period / 1000;
+
+ /* Store attach-time values for computing TSC frequency later. */
+ if (cpu_hascounter()) {
+ (void)cpu_counter();
+ val = bus_space_read_4(sc->sc_memt, sc->sc_memh, HPET_MCOUNT_LO);
+ hpet_attach_tsc = cpu_counter();
+ hpet_attach_val = val;
+ }
}
static u_int
@@ -198,6 +210,40 @@ hpet_delay(unsigned int us)
}
}
+uint64_t
+hpet_tsc_freq(void)
+{
+ struct hpet_softc *sc;
+ uint64_t td, val, freq;
+ uint32_t hd;
+ int s;
+
+ if (hpet0 == NULL || !cpu_hascounter())
+ return 0;
+
+ /* Slow down if we got here from attach in under 0.1s. */
+ sc = hpet0;
+ hd = bus_space_read_4(sc->sc_memt, sc->sc_memh, HPET_MCOUNT_LO);
+ hd -= hpet_attach_val;
+ if (hd < (uint64_t)100000 * 1000000000 / sc->sc_period)
+ hpet_delay(100000);
+
+ /*
+ * Determine TSC freq by comparing how far the TSC and HPET have
+ * advanced since attach time. Take the cost of reading HPET
+ * register into account and round result to the nearest 1000.
+ */
+ s = splhigh();
+ (void)cpu_counter();
+ hd = bus_space_read_4(sc->sc_memt, sc->sc_memh, HPET_MCOUNT_LO);
+ td = cpu_counter();
+ splx(s);
+ hd -= hpet_attach_val;
+ val = ((uint64_t)hd * sc->sc_period - sc->sc_adj) / 100000000;
+ freq = (td - hpet_attach_tsc) * 10000000 / val;
+ return rounddown(freq + 500, 1000);
+}
+
MODULE(MODULE_CLASS_DRIVER, hpet, NULL);
#ifdef _MODULE
Index: src/sys/dev/ic/hpetvar.h
diff -u src/sys/dev/ic/hpetvar.h:1.6 src/sys/dev/ic/hpetvar.h:1.7
--- src/sys/dev/ic/hpetvar.h:1.6 Fri Apr 24 23:29:17 2020
+++ src/sys/dev/ic/hpetvar.h Fri May 8 22:01:54 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: hpetvar.h,v 1.6 2020/04/24 23:29:17 ad Exp $ */
+/* $NetBSD: hpetvar.h,v 1.7 2020/05/08 22:01:54 ad Exp $ */
/*
* Copyright (c) 2006 Nicolas Joly
@@ -49,5 +49,6 @@ void hpet_attach_subr(device_t);
int hpet_detach(device_t, int flags);
void hpet_delay(unsigned int);
bool hpet_delay_p(void);
+uint64_t hpet_tsc_freq(void);
#endif /* _DEV_IC_HPETVAR_H_ */
