Changes in interrupt controller infrastructure to support smp.
Add an API that allows a processor to direct an interrupt to multiple cores
enable support for ampintc to serice MPSAFE interrupts.
bcm does not yet have support enabled, but are updated to be compatible
for single processor support.
This is needed for local timer interrupt support.
diff --git a/sys/arch/arm64/arm64/intr.c b/sys/arch/arm64/arm64/intr.c
index 24efa304017..f41cc8fbda9 100644
--- a/sys/arch/arm64/arm64/intr.c
+++ b/sys/arch/arm64/arm64/intr.c
@@ -53,7 +53,8 @@ struct arm_intr_func arm_intr_func = {
arm_dflt_splx,
arm_dflt_setipl,
arm_dflt_intr_establish,
- arm_dflt_intr_disestablish
+ arm_dflt_intr_disestablish,
+ NULL
};
void (*arm_intr_dispatch)(void *) = arm_dflt_intr;
@@ -71,16 +72,26 @@ arm_dflt_intr(void *frame)
}
-void *arm_intr_establish(int irqno, int level, int (*func)(void *),
+void *
+arm_intr_establish(int irqno, int level, int (*func)(void *),
void *cookie, char *name)
{
return arm_intr_func.intr_establish(irqno, level, func, cookie, name);
}
-void arm_intr_disestablish(void *cookie)
+
+void
+arm_intr_disestablish(void *cookie)
{
arm_intr_func.intr_disestablish(cookie);
}
+void
+arm_intr_route_ih(void *cookie, int enable, int cpu_idx)
+{
+ if (arm_intr_func.route_ih)
+ arm_intr_func.route_ih(cookie, enable, cpu_idx);
+}
+
/*
* Find the interrupt parent by walking up the tree.
*/
@@ -224,11 +235,6 @@ arm_intr_register_fdt(struct interrupt_controller *ic)
}
}
-struct arm_intr_handle {
- struct interrupt_controller *ih_ic;
- void *ih_ih;
-};
-
void *
arm_intr_establish_fdt(int node, int level, int (*func)(void *),
void *cookie, char *name)
@@ -457,7 +463,8 @@ void arm_set_intr_handler(int (*raise)(int), int
(*lower)(int),
void *(*intr_establish)(int irqno, int level, int (*func)(void *),
void *cookie, char *name),
void (*intr_disestablish)(void *cookie),
- void (*intr_handle)(void *))
+ void (*intr_handle)(void *),
+ void (*route_ih)(void*, int, int))
{
arm_intr_func.raise = raise;
arm_intr_func.lower = lower;
@@ -466,6 +473,7 @@ void arm_set_intr_handler(int (*raise)(int), int
(*lower)(int),
arm_intr_func.intr_establish = intr_establish;
arm_intr_func.intr_disestablish = intr_disestablish;
arm_intr_dispatch = intr_handle;
+ arm_intr_func.route_ih = route_ih;
}
void
@@ -586,6 +594,15 @@ cpu_initclocks(void)
}
void
+cpu_startclock(void)
+{
+ if (arm_clock_func.mpstartclock == NULL)
+ panic("mpstartclock function not initialized yet");
+
+ arm_clock_func.mpstartclock();
+}
+
+void
arm_dflt_delay(u_int usecs)
{
int j;
diff --git a/sys/arch/arm64/dev/agtimer.c b/sys/arch/arm64/dev/agtimer.c
index 278ad9a0e19..4517283adc7 100644
--- a/sys/arch/arm64/dev/agtimer.c
+++ b/sys/arch/arm64/dev/agtimer.c
@@ -65,6 +65,7 @@ struct agtimer_softc {
u_int32_t sc_ticks_per_intr;
u_int32_t sc_statvar;
u_int32_t sc_statmin;
+ void *sc_ih;
#ifdef AMPTIMER_DEBUG
struct evcount sc_clk_count;
@@ -297,10 +298,10 @@ agtimer_cpu_initclocks()
pc->pc_ticks_err_sum = 0;
/* Setup secure and non-secure timer IRQs. */
- arm_intr_establish_fdt_idx(sc->sc_node, 0, IPL_CLOCK,
- agtimer_intr, NULL, "tick");
- arm_intr_establish_fdt_idx(sc->sc_node, 1, IPL_CLOCK,
- agtimer_intr, NULL, "tick");
+ // irq 0 is cntv, irq 1 is cntp, since this only supports
+ // cntp, only register the one interrupt.
+ sc->sc_ih = arm_intr_establish_fdt_idx(sc->sc_node, 1,
+ IPL_CLOCK | IPL_MPSAFE, agtimer_intr, NULL, "tick");
next = agtimer_readcnt64() + sc->sc_ticks_per_intr;
pc->pc_nexttickevent = pc->pc_nextstatevent = next;
@@ -380,13 +381,14 @@ agtimer_startclock(void)
nextevent = agtimer_readcnt64() + sc->sc_ticks_per_intr;
pc->pc_nexttickevent = pc->pc_nextstatevent = nextevent;
+ arm_intr_route_ih(sc->sc_ih, 1, cpu_number());
+
reg = agtimer_get_ctrl();
reg &= ~GTIMER_CNTP_CTL_IMASK;
reg |= GTIMER_CNTP_CTL_ENABLE;
agtimer_set_tval(sc->sc_ticks_per_second);
agtimer_set_ctrl(reg);
}
-
void
agtimer_init(void)
{
diff --git a/sys/arch/arm64/dev/ampintc.c b/sys/arch/arm64/dev/ampintc.c
index aa15de5798b..58d92079ce8 100644
--- a/sys/arch/arm64/dev/ampintc.c
+++ b/sys/arch/arm64/dev/ampintc.c
@@ -141,6 +141,7 @@ struct intrhand {
void *ih_arg; /* arg for handler */
int ih_ipl; /* IPL_* */
int ih_irq; /* IRQ number */
+ int ih_flags; /* flags: eg IPL_MPSAFE */
struct evcount ih_count;
char *ih_name;
};
@@ -150,6 +151,7 @@ struct intrq {
int iq_irq; /* IRQ to mask while handling */
int iq_levels; /* IPL_*'s this IRQ has */
int iq_ist; /* share type */
+ int iq_route; /* cpu route mask */
};
@@ -164,6 +166,7 @@ void *ampintc_intr_establish(int, int, int
(*)(void *), void *,
char *);
void *ampintc_intr_establish_ext(int, int, int (*)(void *), void *,
char *);
+void ampintc_route_ih(void *, int, int);
void *ampintc_intr_establish_fdt(void *, int *, int,
int (*)(void *), void *, char *);
void ampintc_intr_disestablish(void *);
@@ -174,7 +177,7 @@ void ampintc_eoi(uint32_t);
void ampintc_set_priority(int, int);
void ampintc_intr_enable(int);
void ampintc_intr_disable(int);
-void ampintc_route(int, int , int);
+void ampintc_route(int, int);
struct cfattach ampintc_ca = {
sizeof (struct ampintc_softc), ampintc_match, ampintc_attach
@@ -263,6 +266,7 @@ ampintc_attach(struct device *parent, struct device *self,
void *aux)
sizeof(*sc->sc_ampintc_handler), M_DEVBUF, M_ZERO | M_NOWAIT);
for (i = 0; i < nintr; i++) {
TAILQ_INIT(&sc->sc_ampintc_handler[i].iq_list);
+ sc->sc_ampintc_handler[i].iq_route = 0;
}
ampintc_setipl(IPL_HIGH); /* XXX ??? */
@@ -271,7 +275,7 @@ ampintc_attach(struct device *parent, struct device *self,
void *aux)
/* insert self as interrupt handler */
arm_set_intr_handler(ampintc_splraise, ampintc_spllower, ampintc_splx,
ampintc_setipl, ampintc_intr_establish_ext,
- ampintc_intr_disestablish, ampintc_irq_handler);
+ ampintc_intr_disestablish, ampintc_irq_handler, ampintc_route_ih);
/* enable interrupts */
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, ICD_DCR, 3);
@@ -375,12 +379,12 @@ ampintc_calc_mask(void)
if (min != IPL_NONE) {
ampintc_set_priority(irq, min);
ampintc_intr_enable(irq);
- ampintc_route(irq, IRQ_ENABLE, 0);
+ sc->sc_ampintc_handler[irq].iq_route |= (1 << 0);
} else {
ampintc_intr_disable(irq);
- ampintc_route(irq, IRQ_DISABLE, 0);
-
+ sc->sc_ampintc_handler[irq].iq_route &= ~(1 << 0);
}
+ ampintc_route(irq, sc->sc_ampintc_handler[irq].iq_route);
}
ampintc_setipl(ci->ci_cpl);
}
@@ -448,17 +452,29 @@ ampintc_eoi(uint32_t eoi)
}
void
-ampintc_route(int irq, int enable, int cpu)
+ampintc_route_ih(void *vih, int enable, int cpuid)
{
- uint8_t val;
+ struct arm_intr_handle *aih = vih;
+ struct intrhand *ih = aih->ih_ih;
struct ampintc_softc *sc = ampintc;
- val = bus_space_read_1(sc->sc_iot, sc->sc_d_ioh, ICD_IPTRn(irq));
- if (enable == IRQ_ENABLE)
- val |= (1 << cpu);
- else
- val &= ~(1 << cpu);
- bus_space_write_1(sc->sc_iot, sc->sc_d_ioh, ICD_IPTRn(irq), val);
+ bus_space_write_4(sc->sc_iot, sc->sc_p_ioh, ICPICR, 1);
+ bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, ICD_ICRn(ih->ih_irq), 0);
+ ampintc_set_priority(ih->ih_irq,
+ sc->sc_ampintc_handler[ih->ih_irq].iq_irq);
+ ampintc_intr_enable(ih->ih_irq);
+
+ /* should be value in MPIDR, is the same for early processors. */
+ sc->sc_ampintc_handler[ih->ih_irq].iq_route |= (1 << cpuid);
+ ampintc_route(ih->ih_irq, sc->sc_ampintc_handler[ih->ih_irq].iq_route);
+}
+
+void
+ampintc_route(int irq, int cpumask)
+{
+ struct ampintc_softc *sc = ampintc;
+
+ bus_space_write_1(sc->sc_iot, sc->sc_d_ioh, ICD_IPTRn(irq), cpumask);
}
void
@@ -499,6 +515,16 @@ ampintc_irq_handler(void *frame)
pri = sc->sc_ampintc_handler[irq].iq_irq;
s = ampintc_splraise(pri);
TAILQ_FOREACH(ih, &sc->sc_ampintc_handler[irq].iq_list, ih_list) {
+#ifdef MULTIPROCESSOR
+ int need_lock;
+
+ if (ih->ih_flags & IPL_MPSAFE)
+ need_lock = 0;
+ else
+ need_lock = s < IPL_SCHED;
+ if (need_lock)
+ KERNEL_LOCK();
+#endif
if (ih->ih_arg != 0)
arg = ih->ih_arg;
else
@@ -507,9 +533,15 @@ ampintc_irq_handler(void *frame)
if (ih->ih_func(arg))
ih->ih_count.ec_count++;
+#ifdef MULTIPROCESSOR
+ if (need_lock)
+ KERNEL_UNLOCK();
+#endif
}
ampintc_eoi(iack_val);
+
+
ampintc_splx(s);
}
@@ -556,7 +588,8 @@ ampintc_intr_establish(int irqno, int level, int
(*func)(void *),
ih = malloc(sizeof(*ih), M_DEVBUF, M_WAITOK);
ih->ih_func = func;
ih->ih_arg = arg;
- ih->ih_ipl = level;
+ ih->ih_ipl = level & IPL_IRQMASK;
+ ih->ih_flags = level & IPL_FLAGMASK;
ih->ih_irq = irqno;
ih->ih_name = name;
@@ -572,7 +605,7 @@ ampintc_intr_establish(int irqno, int level, int
(*func)(void *),
name);
#endif
ampintc_calc_mask();
-
+
restore_interrupts(psw);
return (ih);
}
@@ -594,6 +627,10 @@ ampintc_intr_disestablish(void *cookie)
TAILQ_REMOVE(&sc->sc_ampintc_handler[ih->ih_irq].iq_list, ih, ih_list);
if (ih->ih_name != NULL)
evcount_detach(&ih->ih_count);
+
+ if (TAILQ_EMPTY(&sc->sc_ampintc_handler[ih->ih_irq].iq_list))
+ sc->sc_ampintc_handler[ih->ih_irq].iq_route = 0;
+
free(ih, M_DEVBUF, sizeof(*ih));
ampintc_calc_mask();
diff --git a/sys/arch/arm64/dev/bcm2836_intr.c
b/sys/arch/arm64/dev/bcm2836_intr.c
index 16d6e1daa50..09fe633ee73 100644
--- a/sys/arch/arm64/dev/bcm2836_intr.c
+++ b/sys/arch/arm64/dev/bcm2836_intr.c
@@ -80,6 +80,7 @@ struct intrhand {
void *ih_arg; /* arg for handler */
int ih_ipl; /* IPL_* */
int ih_irq; /* IRQ number */
+ int ih_flags; /* flags, eg MPSAFE */
struct evcount ih_count; /* interrupt counter */
char *ih_name; /* device name */
};
@@ -204,7 +205,7 @@ bcm_intc_attach(struct device *parent, struct device *self,
void *aux)
arm_set_intr_handler(bcm_intc_splraise, bcm_intc_spllower,
bcm_intc_splx, bcm_intc_setipl,
bcm_intc_intr_establish, bcm_intc_intr_disestablish,
- bcm_intc_irq_handler);
+ bcm_intc_irq_handler, NULL);
sc->sc_intc.ic_node = faa->fa_node;
sc->sc_intc.ic_cookie = sc;
@@ -522,7 +523,8 @@ bcm_intc_intr_establish(int irqno, int level, int
(*func)(void *),
ih = malloc(sizeof *ih, M_DEVBUF, M_WAITOK);
ih->ih_fun = func;
ih->ih_arg = arg;
- ih->ih_ipl = level;
+ ih->ih_ipl = level & IPL_IRQMASK;
+ ih->ih_flags = level & IPL_FLAGMASK;
ih->ih_irq = irqno;
ih->ih_name = name;
diff --git a/sys/arch/arm64/include/intr.h b/sys/arch/arm64/include/intr.h
index 10cbe793a93..85c0335255c 100644
--- a/sys/arch/arm64/include/intr.h
+++ b/sys/arch/arm64/include/intr.h
@@ -60,7 +60,9 @@
#define NIPL 13 /* number of levels */
/* Interrupt priority 'flags'. */
-#define IPL_MPSAFE 0 /* no "mpsafe" interrupts */
+#define IPL_IRQMASK 0x0f /* priority only */
+#define IPL_FLAGMASK 0xf00 /* flags only */
+#define IPL_MPSAFE 0x100 /* no "mpsafe" interrupts */
/* Interrupt sharing types. */
#define IST_NONE 0 /* none */
@@ -88,7 +90,8 @@ void arm_set_intr_handler(int (*raise)(int), int
(*lower)(int),
void *(*intr_establish)(int irqno, int level, int (*func)(void *),
void *cookie, char *name),
void (*intr_disestablish)(void *cookie),
- void (*intr_handle)(void *));
+ void (*intr_handle)(void *),
+ void (*route_ih)(void*, int, int));
struct arm_intr_func {
int (*raise)(int);
@@ -98,6 +101,7 @@ struct arm_intr_func {
void *(*intr_establish)(int irqno, int level, int (*func)(void *),
void *cookie, char *name);
void (*intr_disestablish)(void *cookie);
+ void (*route_ih)(void *cookie, int enable, int cpuid);
};
extern struct arm_intr_func arm_intr_func;
@@ -131,6 +135,11 @@ extern uint32_t arm_smask[NIPL];
#include <machine/softintr.h>
+struct arm_intr_handle {
+ struct interrupt_controller *ih_ic;
+ void *ih_ih;
+};
+
void *arm_intr_establish(int irqno, int level, int (*func)(void *),
void *cookie, char *name);
void arm_intr_disestablish(void *cookie);
@@ -153,6 +162,7 @@ struct interrupt_controller {
void arm_intr_init_fdt(void);
void arm_intr_register_fdt(struct interrupt_controller *);
+void arm_intr_route_ih(void *, int, int);
void *arm_intr_establish_fdt(int, int, int (*)(void *),
void *, char *);
void *arm_intr_establish_fdt_idx(int, int, int, int (*)(void *),
Dale Rahn dr...@dalerahn.com
