Module Name: src
Committed By: jmcneill
Date: Sat Mar 27 12:15:09 UTC 2021
Modified Files:
src/sys/arch/aarch64/include: cpu.h
src/sys/arch/arm/include: cpu.h
src/sys/arch/arm/pic: pic.c pic_splfuncs.c picvar.h
Log Message:
Revert recent pic optimizations until I have more time to work on this.
To generate a diff of this commit:
cvs rdiff -u -r1.33 -r1.34 src/sys/arch/aarch64/include/cpu.h
cvs rdiff -u -r1.116 -r1.117 src/sys/arch/arm/include/cpu.h
cvs rdiff -u -r1.69 -r1.70 src/sys/arch/arm/pic/pic.c
cvs rdiff -u -r1.19 -r1.20 src/sys/arch/arm/pic/pic_splfuncs.c
cvs rdiff -u -r1.33 -r1.34 src/sys/arch/arm/pic/picvar.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/aarch64/include/cpu.h
diff -u src/sys/arch/aarch64/include/cpu.h:1.33 src/sys/arch/aarch64/include/cpu.h:1.34
--- src/sys/arch/aarch64/include/cpu.h:1.33 Sun Feb 21 17:07:06 2021
+++ src/sys/arch/aarch64/include/cpu.h Sat Mar 27 12:15:09 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.33 2021/02/21 17:07:06 jmcneill Exp $ */
+/* $NetBSD: cpu.h,v 1.34 2021/03/27 12:15:09 jmcneill Exp $ */
/*-
* Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
@@ -104,9 +104,6 @@ struct cpu_info {
int ci_hwpl; /* current hardware priority */
volatile u_int ci_softints;
volatile u_int ci_intr_depth;
- volatile uint32_t ci_blocked_pics;
- volatile uint32_t ci_pending_pics;
- volatile uint32_t ci_pending_ipls;
int ci_kfpu_spl;
Index: src/sys/arch/arm/include/cpu.h
diff -u src/sys/arch/arm/include/cpu.h:1.116 src/sys/arch/arm/include/cpu.h:1.117
--- src/sys/arch/arm/include/cpu.h:1.116 Sun Feb 21 15:00:04 2021
+++ src/sys/arch/arm/include/cpu.h Sat Mar 27 12:15:08 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.116 2021/02/21 15:00:04 jmcneill Exp $ */
+/* $NetBSD: cpu.h,v 1.117 2021/03/27 12:15:08 jmcneill Exp $ */
/*
* Copyright (c) 1994-1996 Mark Brinicombe.
@@ -190,9 +190,6 @@ struct cpu_info {
volatile u_int ci_intr_depth; /* */
volatile u_int ci_softints;
- volatile uint32_t ci_blocked_pics;
- volatile uint32_t ci_pending_pics;
- volatile uint32_t ci_pending_ipls;
lwp_t * ci_lastlwp; /* last lwp */
Index: src/sys/arch/arm/pic/pic.c
diff -u src/sys/arch/arm/pic/pic.c:1.69 src/sys/arch/arm/pic/pic.c:1.70
--- src/sys/arch/arm/pic/pic.c:1.69 Sun Feb 21 17:07:45 2021
+++ src/sys/arch/arm/pic/pic.c Sat Mar 27 12:15:09 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: pic.c,v 1.69 2021/02/21 17:07:45 jmcneill Exp $ */
+/* $NetBSD: pic.c,v 1.70 2021/03/27 12:15:09 jmcneill Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -33,7 +33,7 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.69 2021/02/21 17:07:45 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.70 2021/03/27 12:15:09 jmcneill Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -59,20 +59,43 @@ __KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.69
#include <arm/pic/picvar.h>
#if defined(__HAVE_PIC_PENDING_INTRS)
-
/*
* This implementation of pending interrupts on a MULTIPROCESSOR system makes
* the assumption that a PIC (pic_softc) shall only have all its interrupts
* come from the same CPU. In other words, interrupts from a single PIC will
* not be distributed among multiple CPUs.
*/
+struct pic_pending {
+ volatile uint32_t blocked_pics;
+ volatile uint32_t pending_pics;
+ volatile uint32_t pending_ipls;
+};
static uint32_t
pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int);
static struct pic_softc *
- pic_list_find_pic_by_pending_ipl(struct cpu_info *, uint32_t);
+ pic_list_find_pic_by_pending_ipl(struct pic_pending *, uint32_t);
+static void
+ pic_deliver_irqs(struct pic_pending *, struct pic_softc *, int, void *);
static void
- pic_deliver_irqs(struct cpu_info *, struct pic_softc *, int, void *);
+ pic_list_deliver_irqs(struct pic_pending *, register_t, int, void *);
+#ifdef MULTIPROCESSOR
+percpu_t *pic_pending_percpu;
+static struct pic_pending *
+pic_pending_get(void)
+{
+ return percpu_getref(pic_pending_percpu);
+}
+static void
+pic_pending_put(struct pic_pending *pend)
+{
+ percpu_putref(pic_pending_percpu);
+}
+#else
+struct pic_pending pic_pending;
+#define pic_pending_get() (&pic_pending)
+#define pic_pending_put(pend) __nothing
+#endif /* MULTIPROCESSOR */
#endif /* __HAVE_PIC_PENDING_INTRS */
struct pic_softc *pic_list[PIC_MAXPICS];
@@ -95,6 +118,20 @@ EVCNT_ATTACH_STATIC(pic_deferral_ev);
static int pic_init(void);
+#ifdef __HAVE_PIC_SET_PRIORITY
+void
+pic_set_priority(struct cpu_info *ci, int newipl)
+{
+ register_t psw = DISABLE_INTERRUPT_SAVE();
+ if (pic_list[0] != NULL)
+ (pic_list[0]->pic_ops->pic_set_priority)(pic_list[0], newipl);
+ ci->ci_cpl = newipl;
+ if ((psw & I32_bit) == 0) {
+ ENABLE_INTERRUPT();
+ }
+}
+#endif
+
#ifdef MULTIPROCESSOR
int
pic_ipi_ast(void *arg)
@@ -138,9 +175,7 @@ pic_ipi_ddb(void *arg)
int
pic_ipi_kpreempt(void *arg)
{
- struct lwp *l = curlwp;
-
- l->l_md.md_astpending |= __BIT(1);
+ atomic_or_uint(&curcpu()->ci_astpending, __BIT(1));
return 1;
}
#endif /* __HAVE_PREEMPTION */
@@ -229,14 +264,15 @@ void
pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is)
{
const uint32_t ipl_mask = __BIT(is->is_ipl);
- struct cpu_info * const ci = curcpu();
atomic_or_32(&pic->pic_pending_irqs[is->is_irq >> 5],
__BIT(is->is_irq & 0x1f));
atomic_or_32(&pic->pic_pending_ipls, ipl_mask);
- atomic_or_32(&ci->ci_pending_ipls, ipl_mask);
- atomic_or_32(&ci->ci_pending_pics, __BIT(pic->pic_id));
+ struct pic_pending *pend = pic_pending_get();
+ atomic_or_32(&pend->pending_ipls, ipl_mask);
+ atomic_or_32(&pend->pending_pics, __BIT(pic->pic_id));
+ pic_pending_put(pend);
}
void
@@ -257,7 +293,6 @@ pic_mark_pending_sources(struct pic_soft
struct intrsource ** const isbase = &pic->pic_sources[irq_base];
struct intrsource *is;
volatile uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5];
- struct cpu_info * const ci = curcpu();
uint32_t ipl_mask = 0;
if (pending == 0)
@@ -280,8 +315,10 @@ pic_mark_pending_sources(struct pic_soft
}
atomic_or_32(&pic->pic_pending_ipls, ipl_mask);
- atomic_or_32(&ci->ci_pending_ipls, ipl_mask);
- atomic_or_32(&ci->ci_pending_pics, __BIT(pic->pic_id));
+ struct pic_pending *pend = pic_pending_get();
+ atomic_or_32(&pend->pending_ipls, ipl_mask);
+ atomic_or_32(&pend->pending_pics, __BIT(pic->pic_id));
+ pic_pending_put(pend);
return ipl_mask;
}
@@ -350,7 +387,7 @@ pic_dispatch(struct intrsource *is, void
#if defined(__HAVE_PIC_PENDING_INTRS)
void
-pic_deliver_irqs(struct cpu_info *ci, struct pic_softc *pic, int ipl,
+pic_deliver_irqs(struct pic_pending *pend, struct pic_softc *pic, int ipl,
void *frame)
{
const uint32_t ipl_mask = __BIT(ipl);
@@ -431,7 +468,7 @@ pic_deliver_irqs(struct cpu_info *ci, st
} while (pending_irqs);
if (blocked_irqs) {
atomic_or_32(iblocked, blocked_irqs);
- atomic_or_32(&ci->ci_blocked_pics, __BIT(pic->pic_id));
+ atomic_or_32(&pend->blocked_pics, __BIT(pic->pic_id));
}
}
@@ -441,15 +478,15 @@ pic_deliver_irqs(struct cpu_info *ci, st
* about these.
*/
if (atomic_and_32_nv(&pic->pic_pending_ipls, ~ipl_mask) == 0)
- atomic_and_32(&ci->ci_pending_pics, ~__BIT(pic->pic_id));
+ atomic_and_32(&pend->pending_pics, ~__BIT(pic->pic_id));
}
-void
-pic_list_unblock_irqs(struct cpu_info *ci)
+static void
+pic_list_unblock_irqs(struct pic_pending *pend)
{
- uint32_t blocked_pics = ci->ci_blocked_pics;
+ uint32_t blocked_pics = pend->blocked_pics;
- ci->ci_blocked_pics = 0;
+ pend->blocked_pics = 0;
for (;;) {
struct pic_softc *pic;
@@ -486,9 +523,9 @@ pic_list_unblock_irqs(struct cpu_info *c
}
struct pic_softc *
-pic_list_find_pic_by_pending_ipl(struct cpu_info *ci, uint32_t ipl_mask)
+pic_list_find_pic_by_pending_ipl(struct pic_pending *pend, uint32_t ipl_mask)
{
- uint32_t pending_pics = ci->ci_pending_pics;
+ uint32_t pending_pics = pend->pending_pics;
struct pic_softc *pic;
for (;;) {
@@ -505,17 +542,17 @@ pic_list_find_pic_by_pending_ipl(struct
}
void
-pic_list_deliver_irqs(struct cpu_info *ci, register_t psw, int ipl,
+pic_list_deliver_irqs(struct pic_pending *pend, register_t psw, int ipl,
void *frame)
{
const uint32_t ipl_mask = __BIT(ipl);
struct pic_softc *pic;
- while ((pic = pic_list_find_pic_by_pending_ipl(ci, ipl_mask)) != NULL) {
- pic_deliver_irqs(ci, pic, ipl, frame);
+ while ((pic = pic_list_find_pic_by_pending_ipl(pend, ipl_mask)) != NULL) {
+ pic_deliver_irqs(pend, pic, ipl, frame);
KASSERT((pic->pic_pending_ipls & ipl_mask) == 0);
}
- atomic_and_32(&ci->ci_pending_ipls, ~ipl_mask);
+ atomic_and_32(&pend->pending_ipls, ~ipl_mask);
}
#endif /* __HAVE_PIC_PENDING_INTRS */
@@ -528,23 +565,24 @@ pic_do_pending_ints(register_t psw, int
return;
}
#if defined(__HAVE_PIC_PENDING_INTRS)
- while ((ci->ci_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) {
- KASSERT(ci->ci_pending_ipls < __BIT(NIPL));
+ struct pic_pending *pend = pic_pending_get();
+ while ((pend->pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) {
+ KASSERT(pend->pending_ipls < __BIT(NIPL));
for (;;) {
- int ipl = 31 - __builtin_clz(ci->ci_pending_ipls);
+ int ipl = 31 - __builtin_clz(pend->pending_ipls);
KASSERT(ipl < NIPL);
if (ipl <= newipl)
break;
pic_set_priority(ci, ipl);
- pic_list_deliver_irqs(ci, psw, ipl, frame);
- pic_list_unblock_irqs(ci);
+ pic_list_deliver_irqs(pend, psw, ipl, frame);
+ pic_list_unblock_irqs(pend);
}
}
+ pic_pending_put(pend);
#endif /* __HAVE_PIC_PENDING_INTRS */
#ifdef __HAVE_PREEMPTION
- struct lwp *l = curlwp;
- if (newipl == IPL_NONE && (l->l_md.md_astpending & __BIT(1))) {
+ if (newipl == IPL_NONE && (ci->ci_astpending & __BIT(1))) {
pic_set_priority(ci, IPL_SCHED);
kpreempt(0);
}
@@ -604,6 +642,11 @@ pic_add(struct pic_softc *pic, int irqba
KASSERT(strlen(pic->pic_name) > 0);
+#if defined(__HAVE_PIC_PENDING_INTRS) && defined(MULTIPROCESSOR)
+ if (__predict_false(pic_pending_percpu == NULL))
+ pic_pending_percpu = percpu_alloc(sizeof(struct pic_pending));
+#endif /* __HAVE_PIC_PENDING_INTRS && MULTIPROCESSOR */
+
mutex_enter(&pic_lock);
if (irqbase == PIC_IRQBASE_ALLOC) {
irqbase = pic_lastbase;
Index: src/sys/arch/arm/pic/pic_splfuncs.c
diff -u src/sys/arch/arm/pic/pic_splfuncs.c:1.19 src/sys/arch/arm/pic/pic_splfuncs.c:1.20
--- src/sys/arch/arm/pic/pic_splfuncs.c:1.19 Mon Mar 1 11:29:14 2021
+++ src/sys/arch/arm/pic/pic_splfuncs.c Sat Mar 27 12:15:09 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: pic_splfuncs.c,v 1.19 2021/03/01 11:29:14 jmcneill Exp $ */
+/* $NetBSD: pic_splfuncs.c,v 1.20 2021/03/27 12:15:09 jmcneill Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -28,7 +28,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pic_splfuncs.c,v 1.19 2021/03/01 11:29:14 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pic_splfuncs.c,v 1.20 2021/03/27 12:15:09 jmcneill Exp $");
#define _INTR_PRIVATE
#include <sys/param.h>
@@ -46,42 +46,31 @@ __KERNEL_RCSID(0, "$NetBSD: pic_splfuncs
#include <arm/pic/picvar.h>
-#if defined(__HAVE_CPU_DOSOFTINTS_CI)
-#define CPU_DOSOFTINTS(ci) cpu_dosoftints_ci((ci))
-#else
-#define CPU_DOSOFTINTS(ci) cpu_dosoftints()
-#endif
-
-#if defined(__HAVE_PIC_PENDING_INTRS)
-static void splx_dopendingints(struct cpu_info *, const int);
-#endif
int
_splraise(int newipl)
{
struct cpu_info * const ci = curcpu();
const int oldipl = ci->ci_cpl;
- KDASSERT(newipl < NIPL);
+ KASSERT(newipl < NIPL);
if (newipl > ci->ci_cpl) {
pic_set_priority(ci, newipl);
}
return oldipl;
}
-
int
_spllower(int newipl)
{
struct cpu_info * const ci = curcpu();
const int oldipl = ci->ci_cpl;
- KDASSERT(panicstr || newipl <= ci->ci_cpl);
+ KASSERT(panicstr || newipl <= ci->ci_cpl);
if (newipl < ci->ci_cpl) {
register_t psw = DISABLE_INTERRUPT_SAVE();
ci->ci_intr_depth++;
pic_do_pending_ints(psw, newipl, NULL);
ci->ci_intr_depth--;
- if ((psw & I32_bit) == 0 || newipl == IPL_NONE) {
+ if ((psw & I32_bit) == 0 || newipl == IPL_NONE)
ENABLE_INTERRUPT();
- }
cpu_dosoftints();
}
return oldipl;
@@ -91,48 +80,24 @@ void
splx(int savedipl)
{
struct cpu_info * const ci = curcpu();
- KDASSERT(savedipl < NIPL);
+ KASSERT(savedipl < NIPL);
if (__predict_false(savedipl == ci->ci_cpl)) {
return;
}
-#if defined(__HAVE_PIC_PENDING_INTRS)
- if (__predict_false(ci->ci_pending_ipls != 0)) {
- splx_dopendingints(ci, savedipl);
- }
-#endif
-
- pic_set_priority(ci, savedipl);
- CPU_DOSOFTINTS(ci);
-
- KDASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d",
- ci->ci_cpl, savedipl);
-}
+ register_t psw = DISABLE_INTERRUPT_SAVE();
+ KASSERTMSG(panicstr != NULL || savedipl < ci->ci_cpl,
+ "splx(%d) to a higher ipl than %d", savedipl, ci->ci_cpl);
-#if defined(__HAVE_PIC_PENDING_INTRS)
-static void __noinline
-splx_dopendingints(struct cpu_info *ci, const int savedipl)
-{
- const register_t psw = DISABLE_INTERRUPT_SAVE();
ci->ci_intr_depth++;
- while ((ci->ci_pending_ipls & ~__BIT(savedipl)) > __BIT(savedipl)) {
- KASSERT(ci->ci_pending_ipls < __BIT(NIPL));
- for (;;) {
- int ipl = 31 - __builtin_clz(ci->ci_pending_ipls);
- KASSERT(ipl < NIPL);
- if (ipl <= savedipl) {
- break;
- }
-
- pic_set_priority(ci, ipl);
- pic_list_deliver_irqs(ci, psw, ipl, NULL);
- pic_list_unblock_irqs(ci);
- }
- }
+ pic_do_pending_ints(psw, savedipl, NULL);
ci->ci_intr_depth--;
- if ((psw & I32_bit) == 0) {
+ KASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d",
+ ci->ci_cpl, savedipl);
+ if ((psw & I32_bit) == 0)
ENABLE_INTERRUPT();
- }
+ cpu_dosoftints();
+ KASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d",
+ ci->ci_cpl, savedipl);
}
-#endif
Index: src/sys/arch/arm/pic/picvar.h
diff -u src/sys/arch/arm/pic/picvar.h:1.33 src/sys/arch/arm/pic/picvar.h:1.34
--- src/sys/arch/arm/pic/picvar.h:1.33 Sat Feb 27 14:22:07 2021
+++ src/sys/arch/arm/pic/picvar.h Sat Mar 27 12:15:09 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: picvar.h,v 1.33 2021/02/27 14:22:07 jmcneill Exp $ */
+/* $NetBSD: picvar.h,v 1.34 2021/03/27 12:15:09 jmcneill Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -73,8 +73,6 @@ void pic_mark_pending(struct pic_softc *
void pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is);
uint32_t pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base,
uint32_t pending);
-void pic_list_deliver_irqs(struct cpu_info *, register_t, int, void *);
-void pic_list_unblock_irqs(struct cpu_info *);
#endif /* __HAVE_PIC_PENDING_INTRS */
void *pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type,
int (*func)(void *), void *arg, const char *);
@@ -107,8 +105,6 @@ void intr_ipi_send(const kcpuset_t *, u_
#include <sys/evcnt.h>
#include <sys/percpu.h>
-#include <arm/cpufunc.h>
-
#ifndef PIC_MAXPICS
#define PIC_MAXPICS 32
#endif
@@ -178,30 +174,22 @@ struct pic_ops {
#endif
};
-/* Using an inline causes catch-22 problems with cpu.h */
#ifdef __HAVE_PIC_SET_PRIORITY
/*
* This is used to update a hardware pic with a value corresponding
* to the ipl being set.
*/
-#define pic_set_priority(ci, newipl) \
- do { \
- register_t __psw = cpsid(I32_bit); \
- (ci)->ci_cpl = (newipl); \
- if (__predict_true(pic_list[0] != NULL)) { \
- (pic_list[0]->pic_ops->pic_set_priority)(pic_list[0], newipl); \
- } \
- if ((__psw & I32_bit) == 0) { \
- cpsie(I32_bit); \
- } \
- } while (0)
+struct cpu_info;
+void pic_set_priority(struct cpu_info *, int);
#else
-#define pic_set_priority(ci, newipl) ((void)((ci)->ci_cpl = (newipl)))
+/* Using an inline causes catch-22 problems with cpu.h */
+#define pic_set_priority(ci, newipl) ((void)((ci)->ci_cpl = (newipl)))
#endif
#define PIC_IRQBASE_ALLOC (-2)
int pic_add(struct pic_softc *, int);
+void pic_do_pending_int(void);
#ifdef MULTIPROCESSOR
int pic_ipi_ast(void *);
int pic_ipi_nop(void *);
