Module Name: src
Committed By: thorpej
Date: Sat Oct 10 03:05:04 UTC 2020
Modified Files:
src/sys/arch/alpha/alpha: clock.c cpu.c interrupt.c ipifuncs.c
src/sys/arch/alpha/include: alpha.h cpu_counter.h intr.h
src/sys/kern: kern_cctr.c
src/sys/sys: cctr.h
Log Message:
Fix cycle counter-based time keeping on Alpha in MP environments by using
a simpler calibration algorithm for the CC timecounter. Proposed in 2018
by Naruaki Etomi:
https://mail-index.netbsd.org/tech-kern/2018/01/14/msg022940.html
This patch is largely based on the proposed change, but avoids changing
any other timecounter logic, and re-factors things a bit to keep them
as MI as possible.
To generate a diff of this commit:
cvs rdiff -u -r1.45 -r1.46 src/sys/arch/alpha/alpha/clock.c
cvs rdiff -u -r1.101 -r1.102 src/sys/arch/alpha/alpha/cpu.c
cvs rdiff -u -r1.91 -r1.92 src/sys/arch/alpha/alpha/interrupt.c
cvs rdiff -u -r1.53 -r1.54 src/sys/arch/alpha/alpha/ipifuncs.c
cvs rdiff -u -r1.42 -r1.43 src/sys/arch/alpha/include/alpha.h
cvs rdiff -u -r1.6 -r1.7 src/sys/arch/alpha/include/cpu_counter.h
cvs rdiff -u -r1.82 -r1.83 src/sys/arch/alpha/include/intr.h
cvs rdiff -u -r1.10 -r1.11 src/sys/kern/kern_cctr.c
cvs rdiff -u -r1.3 -r1.4 src/sys/sys/cctr.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/alpha/alpha/clock.c
diff -u src/sys/arch/alpha/alpha/clock.c:1.45 src/sys/arch/alpha/alpha/clock.c:1.46
--- src/sys/arch/alpha/alpha/clock.c:1.45 Tue Sep 29 01:33:00 2020
+++ src/sys/arch/alpha/alpha/clock.c Sat Oct 10 03:05:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: clock.c,v 1.45 2020/09/29 01:33:00 thorpej Exp $ */
+/* $NetBSD: clock.c,v 1.46 2020/10/10 03:05:04 thorpej Exp $ */
/*
* Copyright (c) 1988 University of Utah.
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: clock.c,v 1.45 2020/09/29 01:33:00 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: clock.c,v 1.46 2020/10/10 03:05:04 thorpej Exp $");
#include <sys/param.h>
#include <sys/kernel.h>
@@ -59,6 +59,8 @@ __KERNEL_RCSID(0, "$NetBSD: clock.c,v 1.
void (*clock_init)(void *);
void *clockdev;
+int alpha_use_cctr; /* != 0 if we're using the PCC timecounter */
+
void
clockattach(void (*fns)(void *), void *dev)
{
@@ -111,6 +113,7 @@ cpu_initclocks(void)
if (! alpha_is_qemu) {
const uint64_t pcc_freq = cpu_frequency(curcpu());
cc_init(NULL, pcc_freq, "PCC", PCC_QUAL);
+ alpha_use_cctr = 1;
}
/*
Index: src/sys/arch/alpha/alpha/cpu.c
diff -u src/sys/arch/alpha/alpha/cpu.c:1.101 src/sys/arch/alpha/alpha/cpu.c:1.102
--- src/sys/arch/alpha/alpha/cpu.c:1.101 Tue Sep 29 01:33:00 2020
+++ src/sys/arch/alpha/alpha/cpu.c Sat Oct 10 03:05:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.101 2020/09/29 01:33:00 thorpej Exp $ */
+/* $NetBSD: cpu.c,v 1.102 2020/10/10 03:05:04 thorpej Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
@@ -59,7 +59,7 @@
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.101 2020/09/29 01:33:00 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.102 2020/10/10 03:05:04 thorpej Exp $");
#include "opt_ddb.h"
#include "opt_multiprocessor.h"
@@ -602,7 +602,9 @@ cpu_hatch(struct cpu_info *ci)
ALPHA_TBIA();
alpha_pal_imb();
- cc_calibrate_cpu(ci);
+ if (alpha_use_cctr) {
+ cc_init_secondary(ci);
+ }
cpu_initclocks_secondary();
}
Index: src/sys/arch/alpha/alpha/interrupt.c
diff -u src/sys/arch/alpha/alpha/interrupt.c:1.91 src/sys/arch/alpha/alpha/interrupt.c:1.92
--- src/sys/arch/alpha/alpha/interrupt.c:1.91 Sat Sep 26 21:07:48 2020
+++ src/sys/arch/alpha/alpha/interrupt.c Sat Oct 10 03:05:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: interrupt.c,v 1.91 2020/09/26 21:07:48 thorpej Exp $ */
+/* $NetBSD: interrupt.c,v 1.92 2020/10/10 03:05:04 thorpej Exp $ */
/*-
* Copyright (c) 2000, 2001 The NetBSD Foundation, Inc.
@@ -65,7 +65,7 @@
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: interrupt.c,v 1.91 2020/09/26 21:07:48 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: interrupt.c,v 1.92 2020/10/10 03:05:04 thorpej Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -228,6 +228,12 @@ interrupt(unsigned long a0, unsigned lon
*/
(*platform.clockintr)((struct clockframe *)framep);
+#if defined(MULTIPROCESSOR)
+ if (alpha_use_cctr) {
+ cc_hardclock(ci);
+ }
+#endif /* MULTIPROCESSOR */
+
/*
* If it's time to call the scheduler clock,
* do so.
Index: src/sys/arch/alpha/alpha/ipifuncs.c
diff -u src/sys/arch/alpha/alpha/ipifuncs.c:1.53 src/sys/arch/alpha/alpha/ipifuncs.c:1.54
--- src/sys/arch/alpha/alpha/ipifuncs.c:1.53 Thu Sep 3 02:03:14 2020
+++ src/sys/arch/alpha/alpha/ipifuncs.c Sat Oct 10 03:05:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: ipifuncs.c,v 1.53 2020/09/03 02:03:14 thorpej Exp $ */
+/* $NetBSD: ipifuncs.c,v 1.54 2020/10/10 03:05:04 thorpej Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.53 2020/09/03 02:03:14 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.54 2020/10/10 03:05:04 thorpej Exp $");
/*
* Interprocessor interrupt handlers.
@@ -61,7 +61,7 @@ __KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v
typedef void (*ipifunc_t)(struct cpu_info *, struct trapframe *);
static void alpha_ipi_halt(struct cpu_info *, struct trapframe *);
-static void alpha_ipi_microset(struct cpu_info *, struct trapframe *);
+static void alpha_ipi_primary_cc(struct cpu_info *, struct trapframe *);
static void alpha_ipi_ast(struct cpu_info *, struct trapframe *);
static void alpha_ipi_pause(struct cpu_info *, struct trapframe *);
static void alpha_ipi_xcall(struct cpu_info *, struct trapframe *);
@@ -69,7 +69,7 @@ static void alpha_ipi_generic(struct cpu
const ipifunc_t ipifuncs[ALPHA_NIPIS] = {
[ilog2(ALPHA_IPI_HALT)] = alpha_ipi_halt,
- [ilog2(ALPHA_IPI_MICROSET)] = alpha_ipi_microset,
+ [ilog2(ALPHA_IPI_PRIMARY_CC)] = alpha_ipi_primary_cc,
[ilog2(ALPHA_IPI_SHOOTDOWN)] = pmap_tlb_shootdown_ipi,
[ilog2(ALPHA_IPI_AST)] = alpha_ipi_ast,
[ilog2(ALPHA_IPI_PAUSE)] = alpha_ipi_pause,
@@ -79,7 +79,7 @@ const ipifunc_t ipifuncs[ALPHA_NIPIS] =
const char * const ipinames[ALPHA_NIPIS] = {
[ilog2(ALPHA_IPI_HALT)] = "halt ipi",
- [ilog2(ALPHA_IPI_MICROSET)] = "microset ipi",
+ [ilog2(ALPHA_IPI_PRIMARY_CC)] = "primary cc ipi",
[ilog2(ALPHA_IPI_SHOOTDOWN)] = "shootdown ipi",
[ilog2(ALPHA_IPI_AST)] = "ast ipi",
[ilog2(ALPHA_IPI_PAUSE)] = "pause ipi",
@@ -250,11 +250,12 @@ alpha_ipi_halt(struct cpu_info * const c
}
static void
-alpha_ipi_microset(struct cpu_info * const ci,
+alpha_ipi_primary_cc(struct cpu_info * const ci __unused,
struct trapframe * const framep __unused)
{
-
- cc_calibrate_cpu(ci);
+ int const s = splhigh();
+ cc_primary_cc();
+ splx(s);
}
static void
Index: src/sys/arch/alpha/include/alpha.h
diff -u src/sys/arch/alpha/include/alpha.h:1.42 src/sys/arch/alpha/include/alpha.h:1.43
--- src/sys/arch/alpha/include/alpha.h:1.42 Sat Oct 3 17:31:46 2020
+++ src/sys/arch/alpha/include/alpha.h Sat Oct 10 03:05:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: alpha.h,v 1.42 2020/10/03 17:31:46 thorpej Exp $ */
+/* $NetBSD: alpha.h,v 1.43 2020/10/10 03:05:04 thorpej Exp $ */
/*
* Copyright (c) 1988 University of Utah.
@@ -68,6 +68,7 @@ struct rpb;
struct trapframe;
extern bool alpha_is_qemu;
+extern int alpha_use_cctr;
extern u_long cpu_implver; /* from IMPLVER instruction */
extern u_long cpu_amask; /* from AMASK instruction */
extern int bootdev_debug;
Index: src/sys/arch/alpha/include/cpu_counter.h
diff -u src/sys/arch/alpha/include/cpu_counter.h:1.6 src/sys/arch/alpha/include/cpu_counter.h:1.7
--- src/sys/arch/alpha/include/cpu_counter.h:1.6 Mon Apr 28 20:23:11 2008
+++ src/sys/arch/alpha/include/cpu_counter.h Sat Oct 10 03:05:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_counter.h,v 1.6 2008/04/28 20:23:11 martin Exp $ */
+/* $NetBSD: cpu_counter.h,v 1.7 2020/10/10 03:05:04 thorpej Exp $ */
/*-
* Copyright (c) 2000 The NetBSD Foundation, Inc.
@@ -41,8 +41,8 @@
#include <machine/cpu.h>
#include <machine/rpb.h>
-#define cc_calibrate_mp(ci) \
- alpha_multicast_ipi(cpus_running, ALPHA_IPI_MICROSET)
+#define cc_get_primary_cc() \
+ alpha_send_ipi(hwrpb->rpb_primary_cpu_id, ALPHA_IPI_PRIMARY_CC)
/* Process Cycle Counter is always available. */
#define cpu_hascounter() (1)
Index: src/sys/arch/alpha/include/intr.h
diff -u src/sys/arch/alpha/include/intr.h:1.82 src/sys/arch/alpha/include/intr.h:1.83
--- src/sys/arch/alpha/include/intr.h:1.82 Sat Sep 26 21:07:48 2020
+++ src/sys/arch/alpha/include/intr.h Sat Oct 10 03:05:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: intr.h,v 1.82 2020/09/26 21:07:48 thorpej Exp $ */
+/* $NetBSD: intr.h,v 1.83 2020/10/10 03:05:04 thorpej Exp $ */
/*-
* Copyright (c) 2000, 2001, 2002 The NetBSD Foundation, Inc.
@@ -173,7 +173,7 @@ void alpha_softint_switchto(struct lwp *
* Interprocessor interrupts. In order how we want them processed.
*/
#define ALPHA_IPI_HALT (1UL << 0)
-#define ALPHA_IPI_MICROSET (1UL << 1)
+#define ALPHA_IPI_PRIMARY_CC (1UL << 1)
#define ALPHA_IPI_SHOOTDOWN (1UL << 2)
#define ALPHA_IPI_AST (1UL << 3)
#define ALPHA_IPI_PAUSE (1UL << 4)
Index: src/sys/kern/kern_cctr.c
diff -u src/sys/kern/kern_cctr.c:1.10 src/sys/kern/kern_cctr.c:1.11
--- src/sys/kern/kern_cctr.c:1.10 Mon Jun 24 06:24:33 2019
+++ src/sys/kern/kern_cctr.c Sat Oct 10 03:05:04 2020
@@ -1,12 +1,10 @@
-/* $NetBSD: kern_cctr.c,v 1.10 2019/06/24 06:24:33 skrll Exp $ */
+/* $NetBSD: kern_cctr.c,v 1.11 2020/10/10 03:05:04 thorpej Exp $ */
/*-
- * Copyright (c) 2006, 2008 The NetBSD Foundation, Inc.
+ * Copyright (c) 2020 Jason R. Thorpe
+ * Copyright (c) 2018 Naruaki Etomi
* All rights reserved.
*
- * re-implementation of TSC for MP systems merging cc_microtime and
- * TSC for timecounters by Frank Kardel
- *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -16,70 +14,60 @@
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
- * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-/* basic calibration ideas are (kern_microtime.c): */
-/******************************************************************************
- * *
- * Copyright (c) David L. Mills 1993, 1994 *
- * *
- * Permission to use, copy, modify, and distribute this software and its *
- * documentation for any purpose and without fee is hereby granted, provided *
- * that the above copyright notice appears in all copies and that both the *
- * copyright notice and this permission notice appear in supporting *
- * documentation, and that the name University of Delaware not be used in *
- * advertising or publicity pertaining to distribution of the software *
- * without specific, written prior permission. The University of Delaware *
- * makes no representations about the suitability this software for any *
- * purpose. It is provided "as is" without express or implied warranty. *
- * *
- ******************************************************************************/
-
-/* reminiscents from older version of this file are: */
-/*-
- * Copyright (c) 1998-2003 Poul-Henning Kamp
- * All rights reserved.
+/*
+ * Most of the following was adapted from the Linux/ia64 cycle counter
+ * synchronization algorithm:
*
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
+ * IA-64 Linux Kernel: Design and Implementation p356-p361
+ * (Hewlett-Packard Professional Books)
+ *
+ * Here's a rough description of how it works.
+ *
+ * The primary CPU is the reference monotonic counter. Each secondary
+ * CPU is responsible for knowing the offset of its own cycle counter
+ * relative to the primary's. When the time counter is read, the CC
+ * value is adjusted by this delta.
*
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * Calibration happens periodically, and works like this:
+ *
+ * Secondary CPU Primary CPU
+ * T0 = local CC
+ * Send IPI to publish reference CC
+ * --------->
+ * (assume this happens at Tavg) Publish reference CC
+ * <-----------------------
+ * Notice publication
+ * T1 = local CC
+ *
+ * Tavg = (T0 + T1) / 2
+ *
+ * Delta = Tavg - Published primary CC value
+ *
+ * We trigger calibration roughly once a second; the period is actually
+ * skewed based on the CPU index in order to avoid lock contention. The
+ * calibration interval does not need to be precise, and so this is fine.
*/
#include <sys/cdefs.h>
-/* __FBSDID("$FreeBSD: src/sys/i386/i386/tsc.c,v 1.204 2003/10/21 18:28:34 silby Exp $"); */
-__KERNEL_RCSID(0, "$NetBSD: kern_cctr.c,v 1.10 2019/06/24 06:24:33 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_cctr.c,v 1.11 2020/10/10 03:05:04 thorpej Exp $");
#include <sys/param.h>
+#include <sys/atomic.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
+#include <sys/timepps.h>
#include <sys/time.h>
#include <sys/timetc.h>
#include <sys/kernel.h>
@@ -89,15 +77,20 @@ __KERNEL_RCSID(0, "$NetBSD: kern_cctr.c,
/* XXX make cc_timecounter.tc_frequency settable by sysctl() */
-static timecounter_pps_t cc_calibrate;
-
-void cc_calibrate_cpu(struct cpu_info *);
-
-static int64_t cc_cal_val; /* last calibrate time stamp */
+#if defined(MULTIPROCESSOR)
+static uint32_t cc_primary __cacheline_aligned;
+static uint32_t cc_calibration_state __cacheline_aligned;
+static kmutex_t cc_calibration_lock __cacheline_aligned;
+
+#define CC_CAL_START 0 /* initial state */
+#define CC_CAL_PRIMARY_READY 1 /* primary CPU ready to respond */
+#define CC_CAL_SECONDARY_READY 2 /* secondary CPU ready to receive */
+#define CC_CAL_FINISHED 3 /* calibration attempt complete */
+#endif /* MULTIPROCESSOR */
static struct timecounter cc_timecounter = {
.tc_get_timecount = cc_get_timecount,
- .tc_poll_pps = cc_calibrate,
+ .tc_poll_pps = NULL,
.tc_counter_mask = ~0u,
.tc_frequency = 0,
.tc_name = "unknown cycle counter",
@@ -112,11 +105,28 @@ static struct timecounter cc_timecounter
};
/*
- * initialize cycle counter based timecounter
+ * Initialize cycle counter based timecounter. This must be done on the
+ * primary CPU.
*/
struct timecounter *
cc_init(timecounter_get_t getcc, uint64_t freq, const char *name, int quality)
{
+ static bool cc_init_done __diagused;
+ struct cpu_info * const ci = curcpu();
+
+ KASSERT(!cc_init_done);
+ KASSERT(cold);
+ KASSERT(CPU_IS_PRIMARY(ci));
+
+#if defined(MULTIPROCESSOR)
+ mutex_init(&cc_calibration_lock, MUTEX_DEFAULT, IPL_HIGH);
+#endif
+
+ cc_init_done = true;
+
+ ci->ci_cc.cc_delta = 0;
+ ci->ci_cc.cc_ticks = 0;
+ ci->ci_cc.cc_cal_ticks = 0;
if (getcc != NULL)
cc_timecounter.tc_get_timecount = getcc;
@@ -130,176 +140,141 @@ cc_init(timecounter_get_t getcc, uint64_
}
/*
+ * Initialize cycle counter timecounter calibration data on a secondary
+ * CPU. Must be called on that secondary CPU.
+ */
+void
+cc_init_secondary(struct cpu_info * const ci)
+{
+ KASSERT(!CPU_IS_PRIMARY(curcpu()));
+ KASSERT(ci == curcpu());
+
+ ci->ci_cc.cc_ticks = 0;
+
+ /*
+ * It's not critical that calibration be performed in
+ * precise intervals, so skew when calibration is done
+ * on each secondary CPU based on it's CPU index to
+ * avoid contending on the calibration lock.
+ */
+ ci->ci_cc.cc_cal_ticks = hz - cpu_index(ci);
+ KASSERT(ci->ci_cc.cc_cal_ticks);
+
+ cc_calibrate_cpu(ci);
+}
+
+/*
* pick up tick count scaled to reference tick count
*/
u_int
cc_get_timecount(struct timecounter *tc)
{
- struct cpu_info *ci;
- int64_t rcc, cc, ncsw;
- u_int gen;
+#if defined(MULTIPROCESSOR)
+ int64_t rcc, ncsw;
retry:
ncsw = curlwp->l_ncsw;
- __insn_barrier();
- ci = curcpu();
- if (ci->ci_cc.cc_denom == 0) {
- /*
- * This is our first time here on this CPU. Just
- * start with reasonable initial values.
- */
- ci->ci_cc.cc_cc = cpu_counter32();
- ci->ci_cc.cc_val = 0;
- if (ci->ci_cc.cc_gen == 0)
- ci->ci_cc.cc_gen++;
-
- ci->ci_cc.cc_denom = cpu_frequency(ci);
- if (ci->ci_cc.cc_denom == 0)
- ci->ci_cc.cc_denom = cc_timecounter.tc_frequency;
- ci->ci_cc.cc_delta = ci->ci_cc.cc_denom;
- }
- /*
- * read counter and re-read when the re-calibration
- * strikes inbetween
- */
- do {
- /* pick up current generation number */
- gen = ci->ci_cc.cc_gen;
-
- /* determine local delta ticks */
- cc = cpu_counter32() - ci->ci_cc.cc_cc;
- if (cc < 0)
- cc += 0x100000000LL;
-
- /* scale to primary */
- rcc = (cc * ci->ci_cc.cc_delta) / ci->ci_cc.cc_denom
- + ci->ci_cc.cc_val;
- } while (gen == 0 || gen != ci->ci_cc.cc_gen);
__insn_barrier();
+ /* N.B. the delta is always 0 on the primary. */
+ rcc = cpu_counter32() - curcpu()->ci_cc.cc_delta;
+ __insn_barrier();
+
if (ncsw != curlwp->l_ncsw) {
/* Was preempted */
goto retry;
}
return rcc;
+#else
+ return cpu_counter32();
+#endif /* MULTIPROCESSOR */
}
-/*
- * called once per clock tick via the pps callback
- * for the calibration of the TSC counters.
- * it is called only for the PRIMARY cpu. all
- * other cpus are called via a broadcast IPI
- * calibration interval is 1 second - we call
- * the calibration code only every hz calls
- */
-static void
-cc_calibrate(struct timecounter *tc)
+#if defined(MULTIPROCESSOR)
+static inline bool
+cc_get_delta(struct cpu_info * const ci)
{
- static int calls;
- struct cpu_info *ci;
+ int64_t t0, t1, tcenter = 0;
- KASSERT(kpreempt_disabled());
+ t0 = cpu_counter32();
- /*
- * XXX: for high interrupt frequency
- * support: ++calls < hz / tc_tick
- */
- if (++calls < hz)
- return;
-
- calls = 0;
- ci = curcpu();
- /* pick up reference ticks */
- cc_cal_val = cpu_counter32();
+ atomic_store_release(&cc_calibration_state, CC_CAL_SECONDARY_READY);
-#if defined(MULTIPROCESSOR)
- cc_calibrate_mp(ci);
-#endif
- cc_calibrate_cpu(ci);
+ for (;;) {
+ if (atomic_load_acquire(&cc_calibration_state) ==
+ CC_CAL_FINISHED) {
+ break;
+ }
+ }
+
+ t1 = cpu_counter32();
+
+ if (t1 < t0) {
+ /* Overflow! */
+ return false;
+ }
+
+ /* average t0 and t1 without overflow: */
+ tcenter = (t0 >> 1) + (t1 >> 1);
+ if ((t0 & 1) + (t1 & 1) == 2)
+ tcenter++;
+
+ ci->ci_cc.cc_delta = tcenter - cc_primary;
+
+ return true;
}
+#endif /* MULTIPROCESSOR */
/*
- * This routine is called about once per second directly by the master
- * processor and via an interprocessor interrupt for other processors.
- * It determines the CC frequency of each processor relative to the
- * master clock and the time this determination is made. These values
- * are used by cc_get_timecount() to interpolate the ticks between
- * timer interrupts. Note that we assume the kernel variables have
- * been zeroed early in life.
+ * Called on secondary CPUs to calibrate their cycle counter offset
+ * relative to the primary CPU.
*/
void
-cc_calibrate_cpu(struct cpu_info *ci)
+cc_calibrate_cpu(struct cpu_info * const ci)
{
- u_int gen;
- int64_t val;
- int64_t delta, denom;
- int s;
-#ifdef TIMECOUNTER_DEBUG
- int64_t factor, old_factor;
-#endif
- val = cc_cal_val;
+#if defined(MULTIPROCESSOR)
+ KASSERT(!CPU_IS_PRIMARY(ci));
+
+ mutex_spin_enter(&cc_calibration_lock);
- s = splhigh();
- /* create next generation number */
- gen = ci->ci_cc.cc_gen;
- gen++;
- if (gen == 0)
- gen++;
-
- /* update in progress */
- ci->ci_cc.cc_gen = 0;
-
- denom = ci->ci_cc.cc_cc;
- ci->ci_cc.cc_cc = cpu_counter32();
-
- if (ci->ci_cc.cc_denom == 0) {
- /*
- * This is our first time here on this CPU. Just
- * start with reasonable initial values.
- */
- ci->ci_cc.cc_val = val;
- ci->ci_cc.cc_denom = cpu_frequency(ci);
- if (ci->ci_cc.cc_denom == 0)
- ci->ci_cc.cc_denom = cc_timecounter.tc_frequency;
- ci->ci_cc.cc_delta = ci->ci_cc.cc_denom;
- ci->ci_cc.cc_gen = gen;
- splx(s);
- return;
+ retry:
+ atomic_store_release(&cc_calibration_state, CC_CAL_START);
+
+ /* Trigger primary CPU. */
+ cc_get_primary_cc();
+
+ for (;;) {
+ if (atomic_load_acquire(&cc_calibration_state) ==
+ CC_CAL_PRIMARY_READY) {
+ break;
+ }
}
-#ifdef TIMECOUNTER_DEBUG
- old_factor = (ci->ci_cc.cc_delta * 1000 ) / ci->ci_cc.cc_denom;
-#endif
+ if (! cc_get_delta(ci)) {
+ goto retry;
+ }
+
+ mutex_exit(&cc_calibration_lock);
+#endif /* MULTIPROCESSOR */
+}
+
+void
+cc_primary_cc(void)
+{
+#if defined(MULTIPROCESSOR)
+ /* N.B. We expect all interrupts to be blocked. */
+
+ atomic_store_release(&cc_calibration_state, CC_CAL_PRIMARY_READY);
+
+ for (;;) {
+ if (atomic_load_acquire(&cc_calibration_state) ==
+ CC_CAL_SECONDARY_READY) {
+ break;
+ }
+ }
- /* local ticks per period */
- denom = ci->ci_cc.cc_cc - denom;
- if (denom < 0)
- denom += 0x100000000LL;
-
- ci->ci_cc.cc_denom = denom;
-
- /* reference ticks per period */
- delta = val - ci->ci_cc.cc_val;
- if (delta < 0)
- delta += 0x100000000LL;
-
- ci->ci_cc.cc_val = val;
- ci->ci_cc.cc_delta = delta;
-
- /* publish new generation number */
- ci->ci_cc.cc_gen = gen;
- splx(s);
-
-#ifdef TIMECOUNTER_DEBUG
- factor = (delta * 1000) / denom - old_factor;
- if (factor < 0)
- factor = -factor;
-
- if (factor > old_factor / 10)
- printf("cc_calibrate_cpu[%u]: 10%% exceeded - delta %"
- PRId64 ", denom %" PRId64 ", factor %" PRId64
- ", old factor %" PRId64"\n", ci->ci_index,
- delta, denom, (delta * 1000) / denom, old_factor);
-#endif /* TIMECOUNTER_DEBUG */
+ cc_primary = cpu_counter32();
+ atomic_store_release(&cc_calibration_state, CC_CAL_FINISHED);
+#endif /* MULTIPROCESSOR */
}
Index: src/sys/sys/cctr.h
diff -u src/sys/sys/cctr.h:1.3 src/sys/sys/cctr.h:1.4
--- src/sys/sys/cctr.h:1.3 Mon Apr 28 20:24:10 2008
+++ src/sys/sys/cctr.h Sat Oct 10 03:05:04 2020
@@ -1,7 +1,7 @@
-/* $NetBSD: cctr.h,v 1.3 2008/04/28 20:24:10 martin Exp $ */
+/* $NetBSD: cctr.h,v 1.4 2020/10/10 03:05:04 thorpej Exp $ */
/*-
- * Copyright (c) 2004 The NetBSD Foundation, Inc.
+ * Copyright (c) 2004, 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -35,19 +35,28 @@
* Variables used by cycle counter in kern_cctr.c.
*/
struct cctr_state {
- volatile u_int cc_gen; /* generation number for this data set */
- volatile int64_t cc_val; /* reference CC value at calibration time */
- volatile int64_t cc_cc; /* local CC value at calibration time */
- volatile int64_t cc_delta; /* reference CC difference for
- last calibration period */
- volatile int64_t cc_denom; /* local CC difference for
- last calibration period */
+ volatile int64_t cc_delta; /* delta from primary CPU CC */
+ u_int cc_cal_ticks; /* # ticks between calibrations */
+ u_int cc_ticks; /* # ticks since last calibration */
};
+#ifdef _KERNEL
struct cpu_info;
-void cc_calibrate_cpu(struct cpu_info *);
struct timecounter *cc_init(timecounter_get_t, uint64_t, const char *, int);
-u_int cc_get_timecount(struct timecounter *);
+void cc_init_secondary(struct cpu_info *);
+u_int cc_get_timecount(struct timecounter *);
+void cc_calibrate_cpu(struct cpu_info *);
+void cc_primary_cc(void);
+
+#define cc_hardclock(ci) \
+do { \
+ if ((ci)->ci_cc.cc_cal_ticks && \
+ ++(ci)->ci_cc.cc_ticks == (ci)->ci_cc.cc_cal_ticks) { \
+ (ci)->ci_cc.cc_ticks = 0; \
+ cc_calibrate_cpu((ci)); \
+ } \
+} while (/*CONSTCOND*/0)
+#endif /* _KERNEL */
#endif /* _SYS_CCTR_H_ */
