Module Name: src
Committed By: cherry
Date: Thu Aug 4 09:07:47 UTC 2011
Modified Files:
src/sys/arch/xen/include [cherry-xenmp]: evtchn.h hypervisor.h intr.h
src/sys/arch/xen/x86 [cherry-xenmp]: hypervisor_machdep.c
src/sys/arch/xen/xen [cherry-xenmp]: evtchn.c xenevt.c
Log Message:
first cut at per-cpu event handling
To generate a diff of this commit:
cvs rdiff -u -r1.18.10.1 -r1.18.10.2 src/sys/arch/xen/include/evtchn.h
cvs rdiff -u -r1.31 -r1.31.10.1 src/sys/arch/xen/include/hypervisor.h
cvs rdiff -u -r1.31.10.2 -r1.31.10.3 src/sys/arch/xen/include/intr.h
cvs rdiff -u -r1.14.2.1 -r1.14.2.2 src/sys/arch/xen/x86/hypervisor_machdep.c
cvs rdiff -u -r1.47.6.1 -r1.47.6.2 src/sys/arch/xen/xen/evtchn.c
cvs rdiff -u -r1.37 -r1.37.2.1 src/sys/arch/xen/xen/xenevt.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/xen/include/evtchn.h
diff -u src/sys/arch/xen/include/evtchn.h:1.18.10.1 src/sys/arch/xen/include/evtchn.h:1.18.10.2
--- src/sys/arch/xen/include/evtchn.h:1.18.10.1 Fri Jun 3 13:27:40 2011
+++ src/sys/arch/xen/include/evtchn.h Thu Aug 4 09:07:46 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: evtchn.h,v 1.18.10.1 2011/06/03 13:27:40 cherry Exp $ */
+/* $NetBSD: evtchn.h,v 1.18.10.2 2011/08/04 09:07:46 cherry Exp $ */
/*
*
@@ -42,7 +42,9 @@
int event_remove_handler(int, int (*func)(void *), void *);
struct intrhand;
-void event_set_iplhandler(struct intrhand *, int);
+void event_set_iplhandler(struct cpu_info *ci,
+ struct intrhand *,
+ int);
extern int debug_port;
extern int xen_debug_handler(void *);
Index: src/sys/arch/xen/include/hypervisor.h
diff -u src/sys/arch/xen/include/hypervisor.h:1.31 src/sys/arch/xen/include/hypervisor.h:1.31.10.1
--- src/sys/arch/xen/include/hypervisor.h:1.31 Mon Oct 19 18:41:10 2009
+++ src/sys/arch/xen/include/hypervisor.h Thu Aug 4 09:07:46 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: hypervisor.h,v 1.31 2009/10/19 18:41:10 bouyer Exp $ */
+/* $NetBSD: hypervisor.h,v 1.31.10.1 2011/08/04 09:07:46 cherry Exp $ */
/*
* Copyright (c) 2006 Manuel Bouyer.
@@ -91,6 +91,7 @@
#include <xen/xen3-public/io/netif.h>
#include <xen/xen3-public/io/blkif.h>
+#include <machine/cpu.h>
#include <machine/hypercalls.h>
#undef u8
@@ -136,7 +137,8 @@
void hypervisor_mask_event(unsigned int);
void hypervisor_clear_event(unsigned int);
void hypervisor_enable_ipl(unsigned int);
-void hypervisor_set_ipending(uint32_t, int, int);
+void hypervisor_set_ipending(struct cpu_info *,
+ uint32_t, int, int);
void hypervisor_machdep_attach(void);
/*
Index: src/sys/arch/xen/include/intr.h
diff -u src/sys/arch/xen/include/intr.h:1.31.10.2 src/sys/arch/xen/include/intr.h:1.31.10.3
--- src/sys/arch/xen/include/intr.h:1.31.10.2 Sun Jun 26 12:56:32 2011
+++ src/sys/arch/xen/include/intr.h Thu Aug 4 09:07:46 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: intr.h,v 1.31.10.2 2011/06/26 12:56:32 cherry Exp $ */
+/* $NetBSD: intr.h,v 1.31.10.3 2011/08/04 09:07:46 cherry Exp $ */
/* NetBSD intr.h,v 1.15 2004/10/31 10:39:34 yamt Exp */
/*-
@@ -58,13 +58,8 @@
struct intrhand *ev_handlers; /* handler chain */
struct evcnt ev_evcnt; /* interrupt counter */
char ev_evname[32]; /* event counter name */
+ struct cpu_info *ev_cpu; /* cpu on which this event is bound */
struct simplelock ev_lock; /* protects this structure */
-
- /*
- * XXX: The lock is quite coursegrained ( for the entire
- * handler list ), but contention is expected to be low. See
- * how this performs and revisit.
- */
};
/*
Index: src/sys/arch/xen/x86/hypervisor_machdep.c
diff -u src/sys/arch/xen/x86/hypervisor_machdep.c:1.14.2.1 src/sys/arch/xen/x86/hypervisor_machdep.c:1.14.2.2
--- src/sys/arch/xen/x86/hypervisor_machdep.c:1.14.2.1 Fri Jun 3 13:27:41 2011
+++ src/sys/arch/xen/x86/hypervisor_machdep.c Thu Aug 4 09:07:47 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: hypervisor_machdep.c,v 1.14.2.1 2011/06/03 13:27:41 cherry Exp $ */
+/* $NetBSD: hypervisor_machdep.c,v 1.14.2.2 2011/08/04 09:07:47 cherry Exp $ */
/*
*
@@ -54,7 +54,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: hypervisor_machdep.c,v 1.14.2.1 2011/06/03 13:27:41 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: hypervisor_machdep.c,v 1.14.2.2 2011/08/04 09:07:47 cherry Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -86,13 +86,101 @@
// #define PORT_DEBUG 4
// #define EARLY_DEBUG_EVENT
+static inline unsigned int
+evt_bitstr_to_port(unsigned long l1, unsigned long l2)
+{
+ unsigned int l1i, l2i, port;
+
+ l1i = xen_ffs(l1) - 1;
+ l2i = xen_ffs(l2) - 1;
+
+ port = (l1i << LONG_SHIFT) + l2i;
+ return port;
+}
+
+/* callback function type */
+typedef void (*iterate_func_t)(struct cpu_info *,
+ unsigned int,
+ unsigned int,
+ unsigned int,
+ void *);
+
+
+static inline void
+evt_iterate_pending(struct cpu_info *ci,
+ volatile unsigned long *pendingl1,
+ volatile unsigned long *pendingl2,
+ volatile unsigned long *mask,
+ iterate_func_t iterate_pending,
+ void *iterate_args)
+{
+
+ KASSERT(pendingl1 != NULL);
+ KASSERT(pendingl2 != NULL);
+
+ unsigned long l1, l2;
+ unsigned int l1i, l2i, port;
+
+ l1 = xen_atomic_xchg(pendingl1, 0);
+ while ((l1i = xen_ffs(l1)) != 0) {
+ l1i--;
+ l1 &= ~(1UL << l1i);
+
+ l2 = pendingl2[l1i] & (mask != NULL ? ~mask[l1i] : -1UL);
+
+ if (mask != NULL) xen_atomic_setbits_l(&mask[l1i], l2);
+ xen_atomic_clearbits_l(&pendingl2[l1i], l2);
+
+ while ((l2i = xen_ffs(l2)) != 0) {
+ l2i--;
+ l2 &= ~(1UL << l2i);
+
+ port = (l1i << LONG_SHIFT) + l2i;
+
+ iterate_pending(ci, port, l1i, l2i, iterate_args);
+ }
+ }
+}
+
+/*
+ * Set per-cpu "pending" information for outstanding events that
+ * cannot be processed now.
+ */
+
+static inline void
+evt_set_pending(struct cpu_info *ci,
+ unsigned int port,
+ unsigned int l1i,
+ unsigned int l2i,
+ void *args)
+{
+
+ KASSERT(args != NULL);
+ KASSERT(ci != NULL);
+
+ int *ret = args;
+
+ if (evtsource[port]) {
+ hypervisor_set_ipending(ci,
+ evtsource[port]->ev_imask,
+ l1i, l2i);
+ evtsource[port]->ev_evcnt.ev_count++;
+ if (*ret == 0 && ci->ci_ilevel <
+ evtsource[port]->ev_maxlevel)
+ *ret = 1;
+ }
+#ifdef DOM0OPS
+ else {
+ /* set pending event */
+ xenevt_setipending(l1i, l2i);
+ }
+#endif
+}
+
int stipending(void);
int
stipending(void)
{
- unsigned long l1;
- unsigned long l2;
- unsigned int l1i, l2i, port;
volatile shared_info_t *s = HYPERVISOR_shared_info;
struct cpu_info *ci;
volatile struct vcpu_info *vci;
@@ -120,45 +208,19 @@
* we're only called after STIC, so we know that we'll have to
* STI at the end
*/
+
while (vci->evtchn_upcall_pending) {
cli();
+
vci->evtchn_upcall_pending = 0;
- /* NB. No need for a barrier here -- XCHG is a barrier
- * on x86. */
- l1 = xen_atomic_xchg(&vci->evtchn_pending_sel, 0);
- while ((l1i = xen_ffs(l1)) != 0) {
- l1i--;
- l1 &= ~(1UL << l1i);
-
- l2 = s->evtchn_pending[l1i] & ~s->evtchn_mask[l1i];
- /*
- * mask and clear event. More efficient than calling
- * hypervisor_mask/clear_event for each event.
- */
- xen_atomic_setbits_l(&s->evtchn_mask[l1i], l2);
- xen_atomic_clearbits_l(&s->evtchn_pending[l1i], l2);
- while ((l2i = xen_ffs(l2)) != 0) {
- l2i--;
- l2 &= ~(1UL << l2i);
-
- port = (l1i << LONG_SHIFT) + l2i;
- if (evtsource[port]) {
- hypervisor_set_ipending(
- evtsource[port]->ev_imask,
- l1i, l2i);
- evtsource[port]->ev_evcnt.ev_count++;
- if (ret == 0 && ci->ci_ilevel <
- evtsource[port]->ev_maxlevel)
- ret = 1;
- }
-#ifdef DOM0OPS
- else {
- /* set pending event */
- xenevt_setipending(l1i, l2i);
- }
-#endif
- }
- }
+
+ evt_iterate_pending(ci,
+ &vci->evtchn_pending_sel,
+ s->evtchn_pending,
+ s->evtchn_mask,
+ evt_set_pending,
+ &ret);
+
sti();
}
@@ -173,12 +235,45 @@
return (ret);
}
+/* Iterate through pending events and call the event handler */
+
+static inline void
+evt_do_hypervisor_callback(struct cpu_info *ci,
+ unsigned int port,
+ unsigned int l1i,
+ unsigned int l2i,
+ void *args)
+{
+ KASSERT(args != NULL);
+ KASSERT(ci == curcpu());
+
+ struct intrframe *regs = args;
+
+#ifdef PORT_DEBUG
+ if (port == PORT_DEBUG)
+ printf("do_hypervisor_callback event %d\n", port);
+#endif
+ if (evtsource[port])
+ call_evtchn_do_event(port, regs);
+#ifdef DOM0OPS
+ else {
+ if (ci->ci_ilevel < IPL_HIGH) {
+ /* fast path */
+ int oipl = ci->ci_ilevel;
+ ci->ci_ilevel = IPL_HIGH;
+ call_xenevt_event(port);
+ ci->ci_ilevel = oipl;
+ } else {
+ /* set pending event */
+ xenevt_setipending(l1i, l2i);
+ }
+ }
+#endif
+}
+
void
do_hypervisor_callback(struct intrframe *regs)
{
- unsigned long l1;
- unsigned long l2;
- unsigned int l1i, l2i, port;
volatile shared_info_t *s = HYPERVISOR_shared_info;
struct cpu_info *ci;
volatile struct vcpu_info *vci;
@@ -199,51 +294,13 @@
while (vci->evtchn_upcall_pending) {
vci->evtchn_upcall_pending = 0;
- /* NB. No need for a barrier here -- XCHG is a barrier
- * on x86. */
- l1 = xen_atomic_xchg(&vci->evtchn_pending_sel, 0);
- while ((l1i = xen_ffs(l1)) != 0) {
- l1i--;
- l1 &= ~(1UL << l1i);
-
- l2 = s->evtchn_pending[l1i] & ~s->evtchn_mask[l1i];
- /*
- * mask and clear the pending events.
- * Doing it here for all event that will be processed
- * avoids a race with stipending (which can be called
- * though evtchn_do_event->splx) that could cause an
- * event to be both processed and marked pending.
- */
- xen_atomic_setbits_l(&s->evtchn_mask[l1i], l2);
- xen_atomic_clearbits_l(&s->evtchn_pending[l1i], l2);
-
- while ((l2i = xen_ffs(l2)) != 0) {
- l2i--;
- l2 &= ~(1UL << l2i);
- port = (l1i << LONG_SHIFT) + l2i;
-#ifdef PORT_DEBUG
- if (port == PORT_DEBUG)
- printf("do_hypervisor_callback event %d\n", port);
-#endif
- if (evtsource[port])
- call_evtchn_do_event(port, regs);
-#ifdef DOM0OPS
- else {
- if (ci->ci_ilevel < IPL_HIGH) {
- /* fast path */
- int oipl = ci->ci_ilevel;
- ci->ci_ilevel = IPL_HIGH;
- call_xenevt_event(port);
- ci->ci_ilevel = oipl;
- } else {
- /* set pending event */
- xenevt_setipending(l1i, l2i);
- }
- }
-#endif
- }
- }
+ evt_iterate_pending(ci,
+ &vci->evtchn_pending_sel,
+ s->evtchn_pending,
+ s->evtchn_mask,
+ evt_do_hypervisor_callback,
+ regs);
}
#ifdef DIAGNOSTIC
@@ -303,11 +360,21 @@
xen_atomic_clear_bit(&s->evtchn_pending[0], ev);
}
+static inline void
+evt_enable_event(struct cpu_info *ci,
+ unsigned int port,
+ unsigned int l1i,
+ unsigned int l2i,
+ void *args)
+{
+ KASSERT(ci != NULL);
+ KASSERT(args == NULL);
+ hypervisor_enable_event(port);
+}
+
void
hypervisor_enable_ipl(unsigned int ipl)
{
- u_long l1, l2;
- int l1i, l2i;
struct cpu_info *ci = curcpu();
/*
@@ -316,30 +383,21 @@
* we know that all callback for this event have been processed.
*/
- l1 = ci->ci_isources[ipl]->ipl_evt_mask1;
- ci->ci_isources[ipl]->ipl_evt_mask1 = 0;
- while ((l1i = xen_ffs(l1)) != 0) {
- l1i--;
- l1 &= ~(1UL << l1i);
- l2 = ci->ci_isources[ipl]->ipl_evt_mask2[l1i];
- ci->ci_isources[ipl]->ipl_evt_mask2[l1i] = 0;
- while ((l2i = xen_ffs(l2)) != 0) {
- int evtch;
+ evt_iterate_pending(ci,
+ &ci->ci_isources[ipl]->ipl_evt_mask1,
+ ci->ci_isources[ipl]->ipl_evt_mask2,
+ NULL,
+ evt_enable_event,
+ NULL);
- l2i--;
- l2 &= ~(1UL << l2i);
-
- evtch = (l1i << LONG_SHIFT) + l2i;
- hypervisor_enable_event(evtch);
- }
- }
}
void
-hypervisor_set_ipending(uint32_t iplmask, int l1, int l2)
+hypervisor_set_ipending(struct cpu_info *ci,
+ uint32_t iplmask,
+ int l1, int l2)
{
int ipl;
- struct cpu_info *ci = curcpu();
/* set pending bit for the appropriate IPLs */
ci->ci_ipending |= iplmask;
Index: src/sys/arch/xen/xen/evtchn.c
diff -u src/sys/arch/xen/xen/evtchn.c:1.47.6.1 src/sys/arch/xen/xen/evtchn.c:1.47.6.2
--- src/sys/arch/xen/xen/evtchn.c:1.47.6.1 Fri Jun 3 13:27:42 2011
+++ src/sys/arch/xen/xen/evtchn.c Thu Aug 4 09:07:47 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: evtchn.c,v 1.47.6.1 2011/06/03 13:27:42 cherry Exp $ */
+/* $NetBSD: evtchn.c,v 1.47.6.2 2011/08/04 09:07:47 cherry Exp $ */
/*
* Copyright (c) 2006 Manuel Bouyer.
@@ -54,7 +54,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: evtchn.c,v 1.47.6.1 2011/06/03 13:27:42 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: evtchn.c,v 1.47.6.2 2011/08/04 09:07:47 cherry Exp $");
#include "opt_xen.h"
#include "isa.h"
@@ -234,14 +234,17 @@
ci->ci_data.cpu_nintr++;
evtsource[evtch]->ev_evcnt.ev_count++;
ilevel = ci->ci_ilevel;
- if (evtsource[evtch]->ev_maxlevel <= ilevel) {
+ if (evtsource[evtch]->ev_maxlevel <= ilevel ||
+ evtsource[evtch]->ev_cpu != ci /* XXX: get stats */) {
#ifdef IRQ_DEBUG
if (evtch == IRQ_DEBUG)
printf("evtsource[%d]->ev_maxlevel %d <= ilevel %d\n",
evtch, evtsource[evtch]->ev_maxlevel, ilevel);
#endif
- hypervisor_set_ipending(evtsource[evtch]->ev_imask,
- evtch >> LONG_SHIFT, evtch & LONG_MASK);
+ hypervisor_set_ipending(evtsource[evtch]->ev_cpu,
+ evtsource[evtch]->ev_imask,
+ evtch >> LONG_SHIFT,
+ evtch & LONG_MASK);
/* leave masked */
return 0;
}
@@ -251,13 +254,14 @@
simple_lock(&evtsource[evtch]->ev_lock);
ih = evtsource[evtch]->ev_handlers;
while (ih != NULL) {
- if (ih->ih_level <= ilevel) {
+ if (ih->ih_level <= ilevel ||
+ ih->ih_cpu != ci) {
#ifdef IRQ_DEBUG
if (evtch == IRQ_DEBUG)
printf("ih->ih_level %d <= ilevel %d\n", ih->ih_level, ilevel);
#endif
cli();
- hypervisor_set_ipending(iplmask,
+ hypervisor_set_ipending(ih->ih_cpu, iplmask,
evtch >> LONG_SHIFT, evtch & LONG_MASK);
/* leave masked */
simple_unlock(&evtsource[evtch]->ev_lock);
@@ -287,6 +291,7 @@
ci->ci_ilevel = i;
for (ih = ci->ci_isources[i]->ipl_handlers;
ih != NULL; ih = ih->ih_ipl_next) {
+ KASSERT(ih->ih_cpu == ci);
sti();
ih_fun = (void *)ih->ih_fun;
ih_fun(ih->ih_arg, regs);
@@ -582,6 +587,7 @@
ih->ih_arg = ih->ih_realarg = arg;
ih->ih_evt_next = NULL;
ih->ih_ipl_next = NULL;
+ ih->ih_cpu = ci;
#ifdef MULTIPROCESSOR
if (!mpsafe) {
ih->ih_fun = intr_biglock_wrapper;
@@ -591,8 +597,8 @@
s = splhigh();
- /* register handler for spllower() */
- event_set_iplhandler(ih, level);
+ /* register per-cpu handler for spllower() */
+ event_set_iplhandler(ci, ih, level);
/* register handler for event channel */
if (evtsource[evtch] == NULL) {
@@ -602,6 +608,13 @@
panic("can't allocate fixed interrupt source");
evts->ev_handlers = ih;
+ /*
+ * XXX: We're assuming here that ci is the same cpu as
+ * the one on which this event/port is bound on. The
+ * api needs to be reshuffled so that this assumption
+ * is more explicitly implemented.
+ */
+ evts->ev_cpu = ci;
simple_lock_init(&evts->ev_lock);
evtsource[evtch] = evts;
if (evname)
@@ -638,11 +651,13 @@
}
void
-event_set_iplhandler(struct intrhand *ih, int level)
+event_set_iplhandler(struct cpu_info *ci,
+ struct intrhand *ih,
+ int level)
{
- struct cpu_info *ci = curcpu();
struct iplsource *ipls;
+ KASSERT(ci == ih->ih_cpu);
if (ci->ci_isources[level] == NULL) {
ipls = malloc(sizeof (struct iplsource),
M_DEVBUF, M_WAITOK|M_ZERO);
Index: src/sys/arch/xen/xen/xenevt.c
diff -u src/sys/arch/xen/xen/xenevt.c:1.37 src/sys/arch/xen/xen/xenevt.c:1.37.2.1
--- src/sys/arch/xen/xen/xenevt.c:1.37 Sun May 22 04:27:15 2011
+++ src/sys/arch/xen/xen/xenevt.c Thu Aug 4 09:07:47 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: xenevt.c,v 1.37 2011/05/22 04:27:15 rmind Exp $ */
+/* $NetBSD: xenevt.c,v 1.37.2.1 2011/08/04 09:07:47 cherry Exp $ */
/*
* Copyright (c) 2005 Manuel Bouyer.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xenevt.c,v 1.37 2011/05/22 04:27:15 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xenevt.c,v 1.37.2.1 2011/08/04 09:07:47 cherry Exp $");
#include "opt_xen.h"
#include <sys/param.h>
@@ -160,6 +160,7 @@
ih->ih_fun = ih->ih_realfun = xenevt_processevt;
ih->ih_arg = ih->ih_realarg = NULL;
ih->ih_ipl_next = NULL;
+ ih->ih_cpu = curcpu();
#ifdef MULTIPROCESSOR
if (!mpsafe) {
ih->ih_fun = intr_biglock_wrapper;
@@ -168,7 +169,7 @@
#endif /* MULTIPROCESSOR */
s = splhigh();
- event_set_iplhandler(ih, level);
+ event_set_iplhandler(ih->ih_cpu, ih, level);
splx(s);
}
@@ -178,7 +179,7 @@
{
xenevt_ev1 |= 1UL << l1;
xenevt_ev2[l1] |= 1UL << l2;
- curcpu()->ci_ipending |= 1 << IPL_HIGH;
+ curcpu()/*XXX*/->ci_ipending |= 1 << IPL_HIGH;
}
/* process pending events */
