Author: marcel
Date: Sat Jun 25 02:15:14 2011
New Revision: 223526
URL: http://svn.freebsd.org/changeset/base/223526
Log:
Switch to the event timers infrastructure. This includes:
o Setting td_intr_frame to the XIVs trap frame because it's referenced
by the ET event handler.
o Signal EOI to the CPU before calling the registered XIV handlers.
This prevents lost ITC interrupts, which cause starvation in one-shot
mode.
o Adding support for IPI_HARDCLOCK with corresponding per-CPU counters.
o Have the APs call cpu_initclocks() so as to limited the scattering of
clock related initialization. cpu_initclocks() calls the <self>_bsp()
or <self>_ap() version accordingly.
o Uncomment the ET clock handling in cpu_idle().
o Update the DDB 'show pcpu' output for the new MD fields.
o Entirely rewritten ia64_ih_clock(). Note that we don't create as many
clock XIVs as we have CPUs, as is done on PowerPC. It doesn't scale.
We can only have 240 XIVs and we can have more CPUs than that. There's
a single intrcnt index for the cumulative clock ticks and we keep per
CPU counts in the PCPU stats structure.
o Register the ITC by hooking SI_SUB_CONFIGURE (2nd order).
Open issues:
o Clock interrupts can still be lost. Some tweaking is still necessary.
Thanks to: mav@ for his support, feedback and explanations.
ET stats while committing:
eris% sysctl machdep.cpu | grep nclks
machdep.cpu.0.nclks: 24007
machdep.cpu.1.nclks: 22895
machdep.cpu.2.nclks: 13523
machdep.cpu.3.nclks: 9342
machdep.cpu.4.nclks: 9103
machdep.cpu.5.nclks: 9298
machdep.cpu.6.nclks: 10039
machdep.cpu.7.nclks: 9479
eris% vmstat -i | grep clock
clock 108599 50
Modified:
head/sys/conf/files.ia64
head/sys/ia64/ia64/clock.c
head/sys/ia64/ia64/db_machdep.c
head/sys/ia64/ia64/interrupt.c
head/sys/ia64/ia64/machdep.c
head/sys/ia64/ia64/mp_machdep.c
head/sys/ia64/include/pcpu.h
head/sys/ia64/include/smp.h
Modified: head/sys/conf/files.ia64
==============================================================================
--- head/sys/conf/files.ia64 Sat Jun 25 00:34:40 2011 (r223525)
+++ head/sys/conf/files.ia64 Sat Jun 25 02:15:14 2011 (r223526)
@@ -120,6 +120,7 @@ ia64/pci/pci_cfgreg.c optional pci
isa/syscons_isa.c optional sc
isa/vga_isa.c optional vga
kern/imgact_elf32.c optional compat_freebsd32
+kern/kern_clocksource.c standard
libkern/bcmp.c standard
libkern/ffsl.c standard
libkern/fls.c standard
Modified: head/sys/ia64/ia64/clock.c
==============================================================================
--- head/sys/ia64/ia64/clock.c Sat Jun 25 00:34:40 2011 (r223525)
+++ head/sys/ia64/ia64/clock.c Sat Jun 25 02:15:14 2011 (r223526)
@@ -32,9 +32,11 @@ __FBSDID("$FreeBSD$");
#include <sys/bus.h>
#include <sys/interrupt.h>
#include <sys/priority.h>
+#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
+#include <sys/timeet.h>
#include <sys/timetc.h>
#include <sys/pcpu.h>
@@ -45,26 +47,12 @@ __FBSDID("$FreeBSD$");
#include <machine/md_var.h>
#include <machine/smp.h>
-SYSCTL_NODE(_debug, OID_AUTO, clock, CTLFLAG_RW, 0, "clock statistics");
-
-static int adjust_edges = 0;
-SYSCTL_INT(_debug_clock, OID_AUTO, adjust_edges, CTLFLAG_RD,
- &adjust_edges, 0, "Number of times ITC got more than 12.5% behind");
-
-static int adjust_excess = 0;
-SYSCTL_INT(_debug_clock, OID_AUTO, adjust_excess, CTLFLAG_RD,
- &adjust_excess, 0, "Total number of ignored ITC interrupts");
-
-static int adjust_lost = 0;
-SYSCTL_INT(_debug_clock, OID_AUTO, adjust_lost, CTLFLAG_RD,
- &adjust_lost, 0, "Total number of lost ITC interrupts");
-
-static int adjust_ticks = 0;
-SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD,
- &adjust_ticks, 0, "Total number of ITC interrupts with adjustment");
+#define CLOCK_ET_OFF 0
+#define CLOCK_ET_PERIODIC 1
+#define CLOCK_ET_ONESHOT 2
+static struct eventtimer ia64_clock_et;
static u_int ia64_clock_xiv;
-static uint64_t ia64_clock_reload;
#ifndef SMP
static timecounter_get_t ia64_get_timecount;
@@ -87,75 +75,100 @@ ia64_get_timecount(struct timecounter* t
static u_int
ia64_ih_clock(struct thread *td, u_int xiv, struct trapframe *tf)
{
- uint64_t adj, clk, itc;
- int64_t delta;
- int count;
+ struct eventtimer *et;
+ uint64_t itc, load;
+ uint32_t mode;
PCPU_INC(md.stats.pcs_nclks);
+ intrcnt[INTRCNT_CLOCK]++;
- if (PCPU_GET(cpuid) == 0) {
- /*
- * Clock processing on the BSP.
- */
- intrcnt[INTRCNT_CLOCK]++;
-
- itc = ia64_get_itc();
-
- adj = PCPU_GET(md.clockadj);
- clk = PCPU_GET(md.clock);
-
- delta = itc - clk;
- count = 0;
- while (delta >= ia64_clock_reload) {
-#ifdef SMP
- ipi_all_but_self(ia64_clock_xiv);
-#endif
- hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
- if (profprocs != 0)
- profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
- statclock(TRAPF_USERMODE(tf));
- delta -= ia64_clock_reload;
- clk += ia64_clock_reload;
- if (adj != 0)
- adjust_ticks++;
- count++;
- }
- ia64_set_itm(ia64_get_itc() + ia64_clock_reload - adj);
- ia64_srlz_d();
- if (count > 0) {
- adjust_lost += count - 1;
- if (delta > (ia64_clock_reload >> 3)) {
- if (adj == 0)
- adjust_edges++;
- adj = ia64_clock_reload >> 4;
- } else
- adj = 0;
- } else {
- adj = 0;
- adjust_excess++;
- }
- PCPU_SET(md.clock, clk);
- PCPU_SET(md.clockadj, adj);
+ itc = ia64_get_itc();
+ PCPU_SET(md.clock, itc);
+
+ mode = PCPU_GET(md.clock_mode);
+ if (mode == CLOCK_ET_PERIODIC) {
+ load = PCPU_GET(md.clock_load);
+ ia64_set_itm(itc + load);
+ } else
+ ia64_set_itv((1 << 16) | xiv);
+ ia64_srlz_d();
+
+ et = &ia64_clock_et;
+ if (et->et_active)
+ et->et_event_cb(et, et->et_arg);
+ return (0);
+}
+
+/*
+ * Event timer start method.
+ */
+static int
+ia64_clock_start(struct eventtimer *et, struct bintime *first,
+ struct bintime *period)
+{
+ u_long itc, load;
+ register_t is;
+
+ if (period != NULL) {
+ PCPU_SET(md.clock_mode, CLOCK_ET_PERIODIC);
+ load = (et->et_frequency * (period->frac >> 32)) >> 32;
+ if (period->sec > 0)
+ load += et->et_frequency * period->sec;
} else {
- /*
- * Clock processing on the BSP.
- */
- hardclock_cpu(TRAPF_USERMODE(tf));
- if (profprocs != 0)
- profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
- statclock(TRAPF_USERMODE(tf));
+ PCPU_SET(md.clock_mode, CLOCK_ET_ONESHOT);
+ load = 0;
}
+ PCPU_SET(md.clock_load, load);
+
+ if (first != NULL) {
+ load = (et->et_frequency * (first->frac >> 32)) >> 32;
+ if (first->sec > 0)
+ load += et->et_frequency * first->sec;
+ }
+
+ is = intr_disable();
+ itc = ia64_get_itc();
+ ia64_set_itm(itc + load);
+ ia64_set_itv(ia64_clock_xiv);
+ ia64_srlz_d();
+ intr_restore(is);
+ return (0);
+}
+
+/*
+ * Event timer stop method.
+ */
+static int
+ia64_clock_stop(struct eventtimer *et)
+{
+
+ ia64_set_itv((1 << 16) | ia64_clock_xiv);
+ ia64_srlz_d();
+ PCPU_SET(md.clock_mode, CLOCK_ET_OFF);
+ PCPU_SET(md.clock_load, 0);
return (0);
}
/*
- * Start the real-time and statistics clocks. We use ar.itc and cr.itm
- * to implement a 1000hz clock.
+ * We call cpu_initclocks() on the APs as well. It allows us to
+ * group common initialization in the same function.
*/
void
cpu_initclocks()
{
+
+ ia64_clock_stop(NULL);
+ if (PCPU_GET(cpuid) == 0)
+ cpu_initclocks_bsp();
+ else
+ cpu_initclocks_ap();
+}
+
+static void
+clock_configure(void *dummy)
+{
+ struct eventtimer *et;
u_long itc_freq;
ia64_clock_xiv = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IPI,
@@ -165,31 +178,23 @@ cpu_initclocks()
itc_freq = (u_long)ia64_itc_freq() * 1000000ul;
- stathz = hz;
- ia64_clock_reload = (itc_freq + hz/2) / hz;
+ et = &ia64_clock_et;
+ et->et_name = "ITC";
+ et->et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU;
+ et->et_quality = 1000;
+ et->et_frequency = itc_freq;
+ et->et_min_period.sec = 0;
+ et->et_min_period.frac = ((1ul << 32) / itc_freq) << 32;
+ et->et_max_period.sec = 0xfffffff0 / itc_freq;
+ et->et_max_period.frac = ((0xfffffffeul << 32) / itc_freq) << 32;
+ et->et_start = ia64_clock_start;
+ et->et_stop = ia64_clock_stop;
+ et->et_priv = NULL;
+ et_register(et);
#ifndef SMP
ia64_timecounter.tc_frequency = itc_freq;
tc_init(&ia64_timecounter);
#endif
-
- PCPU_SET(md.clockadj, 0);
- PCPU_SET(md.clock, ia64_get_itc());
- ia64_set_itm(PCPU_GET(md.clock) + ia64_clock_reload);
- ia64_set_itv(ia64_clock_xiv);
- ia64_srlz_d();
-}
-
-void
-cpu_startprofclock(void)
-{
-
- /* nothing to do */
-}
-
-void
-cpu_stopprofclock(void)
-{
-
- /* nothing to do */
}
+SYSINIT(clkcfg, SI_SUB_CONFIGURE, SI_ORDER_SECOND, clock_configure, NULL);
Modified: head/sys/ia64/ia64/db_machdep.c
==============================================================================
--- head/sys/ia64/ia64/db_machdep.c Sat Jun 25 00:34:40 2011
(r223525)
+++ head/sys/ia64/ia64/db_machdep.c Sat Jun 25 02:15:14 2011
(r223526)
@@ -578,11 +578,13 @@ db_show_mdpcpu(struct pcpu *pc)
{
struct pcpu_md *md = &pc->pc_md;
- db_printf("MD: vhpt = %#lx\n", md->vhpt);
- db_printf("MD: lid = %#lx\n", md->lid);
- db_printf("MD: clock = %#lx/%#lx\n", md->clock, md->clockadj);
- db_printf("MD: stats = %p\n", &md->stats);
- db_printf("MD: pmap = %p\n", md->current_pmap);
+ db_printf("MD: vhpt = %#lx\n", md->vhpt);
+ db_printf("MD: lid = %#lx\n", md->lid);
+ db_printf("MD: clock = %#lx\n", md->clock);
+ db_printf("MD: clock_mode = %u\n", md->clock_mode);
+ db_printf("MD: clock_load = %#lx\n", md->clock_load);
+ db_printf("MD: stats = %p\n", &md->stats);
+ db_printf("MD: pmap = %p\n", md->current_pmap);
}
void
Modified: head/sys/ia64/ia64/interrupt.c
==============================================================================
--- head/sys/ia64/ia64/interrupt.c Sat Jun 25 00:34:40 2011
(r223525)
+++ head/sys/ia64/ia64/interrupt.c Sat Jun 25 02:15:14 2011
(r223526)
@@ -309,6 +309,7 @@ void
ia64_handle_intr(struct trapframe *tf)
{
struct thread *td;
+ struct trapframe *stf;
u_int xiv;
td = curthread;
@@ -323,17 +324,20 @@ ia64_handle_intr(struct trapframe *tf)
}
critical_enter();
+ stf = td->td_intr_frame;
+ td->td_intr_frame = tf;
do {
+ ia64_set_eoi(0);
+ ia64_srlz_d();
CTR2(KTR_INTR, "INTR: ITC=%u, XIV=%u",
(u_int)tf->tf_special.ifa, xiv);
(ia64_handler[xiv])(td, xiv, tf);
- ia64_set_eoi(0);
- ia64_srlz_d();
xiv = ia64_get_ivr();
ia64_srlz_d();
} while (xiv != 15);
+ td->td_intr_frame = stf;
critical_exit();
out:
Modified: head/sys/ia64/ia64/machdep.c
==============================================================================
--- head/sys/ia64/ia64/machdep.c Sat Jun 25 00:34:40 2011
(r223525)
+++ head/sys/ia64/ia64/machdep.c Sat Jun 25 02:15:14 2011
(r223526)
@@ -347,6 +347,11 @@ cpu_startup(void *dummy)
SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
+ "nhardclocks", CTLFLAG_RD, &pcs->pcs_nhardclocks,
+ "Number of IPI_HARDCLOCK interrupts");
+
+ SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
+ SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
"nhighfps", CTLFLAG_RD, &pcs->pcs_nhighfps,
"Number of IPI_HIGH_FP interrupts");
@@ -416,12 +421,10 @@ cpu_idle(int busy)
{
register_t ie;
-#if 0
if (!busy) {
critical_enter();
cpu_idleclock();
}
-#endif
ie = intr_disable();
KASSERT(ie != 0, ("%s called with interrupts disabled\n", __func__));
@@ -436,12 +439,10 @@ cpu_idle(int busy)
ia64_enable_intr();
}
-#if 0
if (!busy) {
cpu_activeclock();
critical_exit();
}
-#endif
}
int
Modified: head/sys/ia64/ia64/mp_machdep.c
==============================================================================
--- head/sys/ia64/ia64/mp_machdep.c Sat Jun 25 00:34:40 2011
(r223525)
+++ head/sys/ia64/ia64/mp_machdep.c Sat Jun 25 02:15:14 2011
(r223526)
@@ -77,6 +77,7 @@ void ia64_ap_startup(void);
struct ia64_ap_state ia64_ap_state;
int ia64_ipi_ast;
+int ia64_ipi_hardclock;
int ia64_ipi_highfp;
int ia64_ipi_nmi;
int ia64_ipi_preempt;
@@ -108,6 +109,16 @@ ia64_ih_ast(struct thread *td, u_int xiv
}
static u_int
+ia64_ih_hardclock(struct thread *td, u_int xiv, struct trapframe *tf)
+{
+
+ PCPU_INC(md.stats.pcs_nhardclocks);
+ CTR1(KTR_SMP, "IPI_HARDCLOCK, cpuid=%d", PCPU_GET(cpuid));
+ hardclockintr();
+ return (0);
+}
+
+static u_int
ia64_ih_highfp(struct thread *td, u_int xiv, struct trapframe *tf)
{
@@ -233,10 +244,11 @@ ia64_ap_startup(void)
CTR1(KTR_SMP, "SMP: cpu%d launched", PCPU_GET(cpuid));
- /* Mask interval timer interrupts on APs. */
- ia64_set_itv(0x10000);
+ cpu_initclocks();
+
ia64_set_tpr(0);
ia64_srlz_d();
+
ia64_enable_intr();
sched_throw(NULL);
@@ -413,6 +425,8 @@ cpu_mp_unleash(void *dummy)
/* Allocate XIVs for IPIs */
ia64_ipi_ast = ia64_xiv_alloc(PI_DULL, IA64_XIV_IPI, ia64_ih_ast);
+ ia64_ipi_hardclock = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IPI,
+ ia64_ih_hardclock);
ia64_ipi_highfp = ia64_xiv_alloc(PI_AV, IA64_XIV_IPI, ia64_ih_highfp);
ia64_ipi_preempt = ia64_xiv_alloc(PI_SOFT, IA64_XIV_IPI,
ia64_ih_preempt);
Modified: head/sys/ia64/include/pcpu.h
==============================================================================
--- head/sys/ia64/include/pcpu.h Sat Jun 25 00:34:40 2011
(r223525)
+++ head/sys/ia64/include/pcpu.h Sat Jun 25 02:15:14 2011
(r223526)
@@ -37,6 +37,7 @@ struct pcpu_stats {
u_long pcs_nasts; /* IPI_AST counter. */
u_long pcs_nclks; /* Clock interrupt counter. */
u_long pcs_nextints; /* ExtINT counter. */
+ u_long pcs_nhardclocks; /* IPI_HARDCLOCK counter. */
u_long pcs_nhighfps; /* IPI_HIGH_FP counter. */
u_long pcs_nhwints; /* Hardware int. counter. */
u_long pcs_npreempts; /* IPI_PREEMPT counter. */
@@ -51,7 +52,8 @@ struct pcpu_md {
vm_offset_t vhpt; /* Address of VHPT */
uint64_t lid; /* local CPU ID */
uint64_t clock; /* Clock counter. */
- uint64_t clockadj; /* Clock adjust. */
+ uint64_t clock_load; /* Clock reload value. */
+ uint32_t clock_mode; /* Clock ET mode */
uint32_t awake:1; /* CPU is awake? */
struct pcpu_stats stats; /* Interrupt stats. */
#ifdef _KERNEL
Modified: head/sys/ia64/include/smp.h
==============================================================================
--- head/sys/ia64/include/smp.h Sat Jun 25 00:34:40 2011 (r223525)
+++ head/sys/ia64/include/smp.h Sat Jun 25 02:15:14 2011 (r223526)
@@ -7,6 +7,7 @@
#ifdef _KERNEL
#define IPI_AST ia64_ipi_ast
+#define IPI_HARDCLOCK ia64_ipi_hardclock
#define IPI_PREEMPT ia64_ipi_preempt
#define IPI_RENDEZVOUS ia64_ipi_rndzvs
#define IPI_STOP ia64_ipi_stop
@@ -37,6 +38,7 @@ struct ia64_ap_state {
};
extern int ia64_ipi_ast;
+extern int ia64_ipi_hardclock;
extern int ia64_ipi_highfp;
extern int ia64_ipi_nmi;
extern int ia64_ipi_preempt;
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
