Module Name: src
Committed By: rmind
Date: Tue Jun 22 18:29:03 UTC 2010
Modified Files:
src/sys/arch/alpha/alpha: ipifuncs.c
src/sys/arch/alpha/include: intr.h
src/sys/arch/powerpc/pic: ipi.c ipivar.h
src/sys/arch/sparc/sparc: cpu.c
src/sys/arch/sparc64/sparc64: ipifuncs.c
src/sys/arch/vax/include: cpu.h
src/sys/arch/vax/vax: multicpu.c
src/sys/arch/x86/include: intrdefs.h
src/sys/arch/x86/x86: ipi.c
src/sys/kern: subr_xcall.c
src/sys/sys: xcall.h
Log Message:
Implement high priority (XC_HIGHPRI) xcall(9) mechanism - a facility
to execute functions from software interrupt context, at SOFTINT_CLOCK.
Functions must be lightweight. Will be used for passive serialization.
OK a...@.
To generate a diff of this commit:
cvs rdiff -u -r1.41 -r1.42 src/sys/arch/alpha/alpha/ipifuncs.c
cvs rdiff -u -r1.67 -r1.68 src/sys/arch/alpha/include/intr.h
cvs rdiff -u -r1.4 -r1.5 src/sys/arch/powerpc/pic/ipi.c
cvs rdiff -u -r1.3 -r1.4 src/sys/arch/powerpc/pic/ipivar.h
cvs rdiff -u -r1.222 -r1.223 src/sys/arch/sparc/sparc/cpu.c
cvs rdiff -u -r1.38 -r1.39 src/sys/arch/sparc64/sparc64/ipifuncs.c
cvs rdiff -u -r1.87 -r1.88 src/sys/arch/vax/include/cpu.h
cvs rdiff -u -r1.28 -r1.29 src/sys/arch/vax/vax/multicpu.c
cvs rdiff -u -r1.15 -r1.16 src/sys/arch/x86/include/intrdefs.h
cvs rdiff -u -r1.17 -r1.18 src/sys/arch/x86/x86/ipi.c
cvs rdiff -u -r1.11 -r1.12 src/sys/kern/subr_xcall.c
cvs rdiff -u -r1.3 -r1.4 src/sys/sys/xcall.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/alpha/alpha/ipifuncs.c
diff -u src/sys/arch/alpha/alpha/ipifuncs.c:1.41 src/sys/arch/alpha/alpha/ipifuncs.c:1.42
--- src/sys/arch/alpha/alpha/ipifuncs.c:1.41 Mon Oct 26 03:51:42 2009
+++ src/sys/arch/alpha/alpha/ipifuncs.c Tue Jun 22 18:29:01 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: ipifuncs.c,v 1.41 2009/10/26 03:51:42 thorpej Exp $ */
+/* $NetBSD: ipifuncs.c,v 1.42 2010/06/22 18:29:01 rmind Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.41 2009/10/26 03:51:42 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.42 2010/06/22 18:29:01 rmind Exp $");
/*
* Interprocessor interrupt handlers.
@@ -64,6 +64,7 @@
void alpha_ipi_synch_fpu(struct cpu_info *, struct trapframe *);
void alpha_ipi_discard_fpu(struct cpu_info *, struct trapframe *);
void alpha_ipi_pause(struct cpu_info *, struct trapframe *);
+void alpha_ipi_xcall(struct cpu_info *, struct trapframe *);
/*
* NOTE: This table must be kept in order with the bit definitions
@@ -78,6 +79,7 @@
alpha_ipi_synch_fpu,
alpha_ipi_discard_fpu,
alpha_ipi_pause,
+ alpha_ipi_xcall
};
const char *ipinames[ALPHA_NIPIS] = {
@@ -89,6 +91,7 @@
"synch fpu ipi",
"discard fpu ipi",
"pause ipi",
+ "xcall ipi"
};
/*
@@ -309,3 +312,30 @@
/* Do an IMB on the way out, in case the kernel text was changed. */
alpha_pal_imb();
}
+
+/*
+ * MD support for xcall(9) interface.
+ */
+
+static void
+alpha_ipi_xcall(struct cpu_info *ci, struct trapframe *framep)
+{
+
+ xc_ipi_handler();
+}
+
+void
+xc_send_ipi(struct cpu_info *ci)
+{
+
+ KASSERT(kpreempt_disabled());
+ KASSERT(curcpu() != ci);
+
+ if (ci) {
+ /* Unicast: remote CPU. */
+ alpha_send_ipi(ci->ci_cpuid, ALPHA_IPI_XCALL);
+ } else {
+ /* Broadcast: all, but local CPU (caller will handle it). */
+ alpha_broadcast_ipi(ALPHA_IPI_XCALL);
+ }
+}
Index: src/sys/arch/alpha/include/intr.h
diff -u src/sys/arch/alpha/include/intr.h:1.67 src/sys/arch/alpha/include/intr.h:1.68
--- src/sys/arch/alpha/include/intr.h:1.67 Mon Oct 26 03:51:43 2009
+++ src/sys/arch/alpha/include/intr.h Tue Jun 22 18:29:02 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: intr.h,v 1.67 2009/10/26 03:51:43 thorpej Exp $ */
+/* $NetBSD: intr.h,v 1.68 2010/06/22 18:29:02 rmind Exp $ */
/*-
* Copyright (c) 2000, 2001, 2002 The NetBSD Foundation, Inc.
@@ -165,8 +165,9 @@
#define ALPHA_IPI_SYNCH_FPU (1UL << 5)
#define ALPHA_IPI_DISCARD_FPU (1UL << 6)
#define ALPHA_IPI_PAUSE (1UL << 7)
+#define ALPHA_IPI_XCALL (1UL << 8)
-#define ALPHA_NIPIS 8 /* must not exceed 64 */
+#define ALPHA_NIPIS 9 /* must not exceed 64 */
struct cpu_info;
struct trapframe;
Index: src/sys/arch/powerpc/pic/ipi.c
diff -u src/sys/arch/powerpc/pic/ipi.c:1.4 src/sys/arch/powerpc/pic/ipi.c:1.5
--- src/sys/arch/powerpc/pic/ipi.c:1.4 Mon Apr 28 20:23:32 2008
+++ src/sys/arch/powerpc/pic/ipi.c Tue Jun 22 18:29:02 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: ipi.c,v 1.4 2008/04/28 20:23:32 martin Exp $ */
+/* $NetBSD: ipi.c,v 1.5 2010/06/22 18:29:02 rmind Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.4 2008/04/28 20:23:32 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.5 2010/06/22 18:29:02 rmind Exp $");
#include "opt_multiprocessor.h"
#include "opt_pic.h"
@@ -76,6 +76,9 @@
save_vec_cpu();
#endif
+ if (ipi & PPC_IPI_XCALL)
+ xc_ipi_handler();
+
if (ipi & PPC_IPI_HALT) {
aprint_normal("halting CPU %d\n", cpu_id);
msr = (mfmsr() & ~PSL_EE) | PSL_POW;
@@ -84,7 +87,28 @@
mtmsr(msr);
}
}
+
return 1;
}
+/*
+ * MD support for xcall(9) interface.
+ */
+
+void
+xc_send_ipi(struct cpu_info *ci)
+{
+
+ KASSERT(kpreempt_disabled());
+ KASSERT(curcpu() != ci);
+
+ if (ci) {
+ /* Unicast: remote CPU. */
+ ppc_send_ipi(ci->ci_cpuid, PPC_IPI_XCALL);
+ } else {
+ /* Broadcast: all, but local CPU (caller will handle it). */
+ ppc_send_ipi(IPI_T_NOTME, PPC_IPI_XCALL);
+ }
+}
+
#endif /*MULTIPROCESSOR*/
Index: src/sys/arch/powerpc/pic/ipivar.h
diff -u src/sys/arch/powerpc/pic/ipivar.h:1.3 src/sys/arch/powerpc/pic/ipivar.h:1.4
--- src/sys/arch/powerpc/pic/ipivar.h:1.3 Mon Apr 28 20:23:32 2008
+++ src/sys/arch/powerpc/pic/ipivar.h Tue Jun 22 18:29:02 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: ipivar.h,v 1.3 2008/04/28 20:23:32 martin Exp $ */
+/* $NetBSD: ipivar.h,v 1.4 2010/06/22 18:29:02 rmind Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipivar.h,v 1.3 2008/04/28 20:23:32 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipivar.h,v 1.4 2010/06/22 18:29:02 rmind Exp $");
#ifndef _IPI_VAR_H_
#define _IPI_VAR_H_
@@ -51,6 +51,7 @@
#define PPC_IPI_HALT 0x0001
#define PPC_IPI_FLUSH_FPU 0x0002
#define PPC_IPI_FLUSH_VEC 0x0004
+#define PPC_IPI_XCALL 0x0008
/* OpenPIC */
void setup_openpic_ipi(void);
Index: src/sys/arch/sparc/sparc/cpu.c
diff -u src/sys/arch/sparc/sparc/cpu.c:1.222 src/sys/arch/sparc/sparc/cpu.c:1.223
--- src/sys/arch/sparc/sparc/cpu.c:1.222 Mon May 31 03:16:47 2010
+++ src/sys/arch/sparc/sparc/cpu.c Tue Jun 22 18:29:02 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.222 2010/05/31 03:16:47 mrg Exp $ */
+/* $NetBSD: cpu.c,v 1.223 2010/06/22 18:29:02 rmind Exp $ */
/*
* Copyright (c) 1996
@@ -52,7 +52,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.222 2010/05/31 03:16:47 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.223 2010/06/22 18:29:02 rmind Exp $");
#include "opt_multiprocessor.h"
#include "opt_lockdebug.h"
@@ -63,11 +63,8 @@
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/malloc.h>
-#include <sys/simplelock.h>
#include <sys/kernel.h>
-#if 0
#include <sys/xcall.h>
-#endif
#include <uvm/uvm.h>
@@ -640,9 +637,8 @@
mutex_spin_exit(&xpmsg_mutex);
}
-#if 0
/*
- * MI interface to call xc_ipi_handler() everywhere.
+ * MD support for MI xcall(9) interface.
*/
void
xc_send_ipi(struct cpu_info *target)
@@ -658,7 +654,6 @@
cpuset = CPUSET_ALL & ~(1 << cpuinfo.ci_cpuid);
XCALL0(xc_ipi_handler, cpuset);
}
-#endif
/*
* Tell all CPUs other than the current one to enter the PROM idle loop.
Index: src/sys/arch/sparc64/sparc64/ipifuncs.c
diff -u src/sys/arch/sparc64/sparc64/ipifuncs.c:1.38 src/sys/arch/sparc64/sparc64/ipifuncs.c:1.39
--- src/sys/arch/sparc64/sparc64/ipifuncs.c:1.38 Sat May 29 21:59:34 2010
+++ src/sys/arch/sparc64/sparc64/ipifuncs.c Tue Jun 22 18:29:02 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: ipifuncs.c,v 1.38 2010/05/29 21:59:34 martin Exp $ */
+/* $NetBSD: ipifuncs.c,v 1.39 2010/06/22 18:29:02 rmind Exp $ */
/*-
* Copyright (c) 2004 The NetBSD Foundation, Inc.
@@ -27,14 +27,14 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.38 2010/05/29 21:59:34 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.39 2010/06/22 18:29:02 rmind Exp $");
#include "opt_ddb.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
-#include <sys/simplelock.h>
+#include <sys/xcall.h>
#include <machine/db_machdep.h>
@@ -463,6 +463,10 @@
printf("\n");
}
+/*
+ * MD support for xcall(9) interface.
+ */
+
void
sparc64_generic_xcall(struct cpu_info *target, ipi_c_call_func_t func,
void *arg)
@@ -478,14 +482,9 @@
}
}
-#if 0
-/* XXX: remove once a public prototype is available */
-void xc_ipi_handler(void);
-void xc_send_ipi(struct cpu_info *);
-
void
xc_send_ipi(struct cpu_info *target)
{
+
sparc64_generic_xcall(target, (ipi_c_call_func_t)xc_ipi_handler, NULL);
}
-#endif
Index: src/sys/arch/vax/include/cpu.h
diff -u src/sys/arch/vax/include/cpu.h:1.87 src/sys/arch/vax/include/cpu.h:1.88
--- src/sys/arch/vax/include/cpu.h:1.87 Sat Dec 12 14:44:09 2009
+++ src/sys/arch/vax/include/cpu.h Tue Jun 22 18:29:02 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.87 2009/12/12 14:44:09 tsutsui Exp $ */
+/* $NetBSD: cpu.h,v 1.88 2010/06/22 18:29:02 rmind Exp $ */
/*
* Copyright (c) 1994 Ludd, University of Lule}, Sweden
@@ -101,6 +101,7 @@
#define IPI_RUNNING 3 /* This CPU just started to run */
#define IPI_TBIA 4 /* Flush the TLB */
#define IPI_DDB 5 /* Jump into the DDB loop */
+#define IPI_XCALL 6 /* Helper for xcall(9) */
#define IPI_DEST_MASTER -1 /* Destination is mastercpu */
#define IPI_DEST_ALL -2 /* Broadcast */
Index: src/sys/arch/vax/vax/multicpu.c
diff -u src/sys/arch/vax/vax/multicpu.c:1.28 src/sys/arch/vax/vax/multicpu.c:1.29
--- src/sys/arch/vax/vax/multicpu.c:1.28 Thu Nov 26 00:19:23 2009
+++ src/sys/arch/vax/vax/multicpu.c Tue Jun 22 18:29:02 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: multicpu.c,v 1.28 2009/11/26 00:19:23 matt Exp $ */
+/* $NetBSD: multicpu.c,v 1.29 2010/06/22 18:29:02 rmind Exp $ */
/*
* Copyright (c) 2000 Ludd, University of Lule}, Sweden. All rights reserved.
@@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: multicpu.c,v 1.28 2009/11/26 00:19:23 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: multicpu.c,v 1.29 2010/06/22 18:29:02 rmind Exp $");
#include "opt_multiprocessor.h"
@@ -44,6 +44,7 @@
#include <sys/malloc.h>
#include <sys/proc.h>
#include <sys/device.h>
+#include <sys/xcall.h>
#include <uvm/uvm_extern.h>
@@ -195,9 +196,32 @@
case IPI_DDB:
Debugger();
break;
+ case IPI_XCALL:
+ xc_ipi_handler();
+ break;
default:
panic("cpu_handle_ipi: bad bit %x", bitno);
}
}
splx(s);
}
+
+/*
+ * MD support for xcall(9) interface.
+ */
+
+void
+xc_send_ipi(struct cpu_info *ci)
+{
+
+ KASSERT(kpreempt_disabled());
+ KASSERT(curcpu() != ci);
+
+ if (ci) {
+ /* Unicast: remote CPU. */
+ cpu_send_ipi(ci->ci_cpuid, IPI_XCALL);
+ } else {
+ /* Broadcast: all, but local CPU (caller will handle it). */
+ cpu_send_ipi(IPI_DEST_ALL, IPI_XCALL);
+ }
+}
Index: src/sys/arch/x86/include/intrdefs.h
diff -u src/sys/arch/x86/include/intrdefs.h:1.15 src/sys/arch/x86/include/intrdefs.h:1.16
--- src/sys/arch/x86/include/intrdefs.h:1.15 Mon Oct 5 23:59:31 2009
+++ src/sys/arch/x86/include/intrdefs.h Tue Jun 22 18:29:02 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: intrdefs.h,v 1.15 2009/10/05 23:59:31 rmind Exp $ */
+/* $NetBSD: intrdefs.h,v 1.16 2010/06/22 18:29:02 rmind Exp $ */
#ifndef _X86_INTRDEFS_H_
#define _X86_INTRDEFS_H_
@@ -62,7 +62,7 @@
#define X86_IPI_SYNCH_FPU 0x00000008
#define X86_IPI_MTRR 0x00000010
#define X86_IPI_GDT 0x00000020
-#define X86_IPI__UNUSED2 0x00000040
+#define X86_IPI_XCALL 0x00000040
#define X86_IPI_ACPI_CPU_SLEEP 0x00000080
#define X86_IPI_KPREEMPT 0x00000100
@@ -70,7 +70,7 @@
#define X86_IPI_NAMES { "halt IPI", "timeset IPI", "unused", \
"FPU synch IPI", "MTRR update IPI", \
- "GDT update IPI", "unused", \
+ "GDT update IPI", "xcall IPI", \
"ACPI CPU sleep IPI", "kpreempt IPI" }
#define IREENT_MAGIC 0x18041969
Index: src/sys/arch/x86/x86/ipi.c
diff -u src/sys/arch/x86/x86/ipi.c:1.17 src/sys/arch/x86/x86/ipi.c:1.18
--- src/sys/arch/x86/x86/ipi.c:1.17 Sun Apr 25 16:10:51 2010
+++ src/sys/arch/x86/x86/ipi.c Tue Jun 22 18:29:03 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: ipi.c,v 1.17 2010/04/25 16:10:51 ad Exp $ */
+/* $NetBSD: ipi.c,v 1.18 2010/06/22 18:29:03 rmind Exp $ */
/*-
* Copyright (c) 2000, 2008, 2009 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.17 2010/04/25 16:10:51 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.18 2010/06/22 18:29:03 rmind Exp $");
#include "opt_mtrr.h"
@@ -42,6 +42,7 @@
#include <sys/atomic.h>
#include <sys/intr.h>
#include <sys/cpu.h>
+#include <sys/xcall.h>
#ifdef MULTIPROCESSOR
@@ -71,6 +72,7 @@
static void x86_ipi_halt(struct cpu_info *);
static void x86_ipi_kpreempt(struct cpu_info *);
+static void x86_ipi_xcall(struct cpu_info *);
#ifdef MTRR
static void x86_ipi_reload_mtrr(struct cpu_info *);
@@ -92,7 +94,7 @@
x86_ipi_synch_fpu,
x86_ipi_reload_mtrr,
gdt_reload_cpu,
- NULL,
+ x86_ipi_xcall,
acpi_cpu_sleep,
x86_ipi_kpreempt
};
@@ -209,6 +211,33 @@
softint_trigger(1 << SIR_PREEMPT);
}
+/*
+ * MD support for xcall(9) interface.
+ */
+
+static void
+x86_ipi_xcall(struct cpu_info *ci)
+{
+
+ xc_ipi_handler();
+}
+
+void
+xc_send_ipi(struct cpu_info *ci)
+{
+
+ KASSERT(kpreempt_disabled());
+ KASSERT(curcpu() != ci);
+
+ if (ci) {
+ /* Unicast: remote CPU. */
+ x86_send_ipi(ci, X86_IPI_XCALL);
+ } else {
+ /* Broadcast: all, but local CPU (caller will handle it). */
+ x86_broadcast_ipi(X86_IPI_XCALL);
+ }
+}
+
#else
int
Index: src/sys/kern/subr_xcall.c
diff -u src/sys/kern/subr_xcall.c:1.11 src/sys/kern/subr_xcall.c:1.12
--- src/sys/kern/subr_xcall.c:1.11 Mon Nov 30 15:37:56 2009
+++ src/sys/kern/subr_xcall.c Tue Jun 22 18:29:01 2010
@@ -1,11 +1,11 @@
-/* $NetBSD: subr_xcall.c,v 1.11 2009/11/30 15:37:56 pooka Exp $ */
+/* $NetBSD: subr_xcall.c,v 1.12 2010/06/22 18:29:01 rmind Exp $ */
/*-
- * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc.
+ * Copyright (c) 2007-2010 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
- * by Andrew Doran.
+ * by Andrew Doran and Mindaugas Rasiukevicius.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -67,14 +67,14 @@
* Cross calls must not allocate memory, as the pagedaemon uses
* them (and memory allocation may need to wait on the pagedaemon).
*
- * Future directions
- *
- * Add a low-overhead mechanism to run cross calls in interrupt
- * context (XC_HIGHPRI).
+ * A low-overhead mechanism for high priority calls (XC_HIGHPRI) is
+ * also provided. The function to be executed runs on a software
+ * interrupt context, at SOFTINT_CLOCK level, and is expected to be
+ * very lightweight, e.g. avoid blocking.
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.11 2009/11/30 15:37:56 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.12 2010/06/22 18:29:01 rmind Exp $");
#include <sys/types.h>
#include <sys/param.h>
@@ -85,19 +85,66 @@
#include <sys/kthread.h>
#include <sys/cpu.h>
-static void xc_thread(void *);
-static uint64_t xc_lowpri(u_int, xcfunc_t, void *, void *, struct cpu_info *);
+/* Cross-call state box. */
+typedef struct {
+ kmutex_t xc_lock;
+ kcondvar_t xc_busy;
+ xcfunc_t xc_func;
+ void * xc_arg1;
+ void * xc_arg2;
+ uint64_t xc_headp;
+ uint64_t xc_donep;
+} xc_state_t;
+
+/* Bit indicating high (1) or low (0) priority. */
+#define XC_PRI_BIT (1ULL << 63)
+
+/* Low priority xcall structures. */
+static xc_state_t xc_low_pri;
+static uint64_t xc_tailp;
+
+/* High priority xcall structures. */
+static xc_state_t xc_high_pri;
+static void * xc_sih;
-static kmutex_t xc_lock;
-static xcfunc_t xc_func;
-static void *xc_arg1;
-static void *xc_arg2;
-static kcondvar_t xc_busy;
+/* Event counters. */
static struct evcnt xc_unicast_ev;
static struct evcnt xc_broadcast_ev;
-static uint64_t xc_headp;
-static uint64_t xc_tailp;
-static uint64_t xc_donep;
+
+static void xc_init(void);
+static void xc_thread(void *);
+static void xc_highpri_intr(void *);
+
+static inline uint64_t xc_highpri(xcfunc_t, void *, void *, struct cpu_info *);
+static inline uint64_t xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *);
+
+/*
+ * xc_init:
+ *
+ * Initialize low and high priority cross-call structures.
+ */
+static void
+xc_init(void)
+{
+ xc_state_t *xclo = &xc_low_pri, *xchi = &xc_high_pri;
+
+ memset(xclo, 0, sizeof(xc_state_t));
+ mutex_init(&xclo->xc_lock, MUTEX_DEFAULT, IPL_NONE);
+ cv_init(&xclo->xc_busy, "xclocv");
+ xc_tailp = 0;
+
+ memset(xchi, 0, sizeof(xc_state_t));
+ mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTCLOCK);
+ cv_init(&xchi->xc_busy, "xchicv");
+ xc_sih = softint_establish(SOFTINT_CLOCK | SOFTINT_MPSAFE,
+ xc_highpri_intr, NULL);
+ KASSERT(xc_sih != NULL);
+
+ evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL,
+ "crosscall", "unicast");
+ evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL,
+ "crosscall", "broadcast");
+}
/*
* xc_init_cpu:
@@ -113,20 +160,13 @@
if (!again) {
/* Autoconfiguration will prevent re-entry. */
+ xc_init();
again = true;
- mutex_init(&xc_lock, MUTEX_DEFAULT, IPL_NONE);
- cv_init(&xc_busy, "xcallbsy");
- evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL,
- "crosscall", "unicast");
- evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL,
- "crosscall", "broadcast");
}
-
cv_init(&ci->ci_data.cpu_xcall, "xcall");
error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread,
NULL, NULL, "xcall/%u", ci->ci_index);
- if (error != 0)
- panic("xc_init_cpu: error %d", error);
+ KASSERT(error == 0);
}
/*
@@ -138,10 +178,12 @@
xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2)
{
+ KASSERT(!cpu_intr_p() && !cpu_softintr_p());
+
if ((flags & XC_HIGHPRI) != 0) {
- panic("xc_broadcast: no high priority crosscalls yet");
+ return xc_highpri(func, arg1, arg2, NULL);
} else {
- return xc_lowpri(flags, func, arg1, arg2, NULL);
+ return xc_lowpri(func, arg1, arg2, NULL);
}
}
@@ -155,12 +197,47 @@
struct cpu_info *ci)
{
+ KASSERT(ci != NULL);
+ KASSERT(!cpu_intr_p() && !cpu_softintr_p());
+
if ((flags & XC_HIGHPRI) != 0) {
- panic("xc_unicast: no high priority crosscalls yet");
+ return xc_highpri(func, arg1, arg2, ci);
+ } else {
+ return xc_lowpri(func, arg1, arg2, ci);
+ }
+}
+
+/*
+ * xc_wait:
+ *
+ * Wait for a cross call to complete.
+ */
+void
+xc_wait(uint64_t where)
+{
+ xc_state_t *xc;
+
+ KASSERT(!cpu_intr_p() && !cpu_softintr_p());
+
+ /* Determine whether it is high or low priority cross-call. */
+ if ((where & XC_PRI_BIT) != 0) {
+ xc = &xc_high_pri;
+ where &= ~XC_PRI_BIT;
} else {
- KASSERT(ci != NULL);
- return xc_lowpri(flags, func, arg1, arg2, ci);
+ xc = &xc_low_pri;
+ }
+
+ /* Fast path, if already done. */
+ if (xc->xc_donep >= where) {
+ return;
+ }
+
+ /* Slow path: block until awoken. */
+ mutex_enter(&xc->xc_lock);
+ while (xc->xc_donep < where) {
+ cv_wait(&xc->xc_busy, &xc->xc_lock);
}
+ mutex_exit(&xc->xc_lock);
}
/*
@@ -168,60 +245,45 @@
*
* Trigger a low priority call on one or more CPUs.
*/
-static uint64_t
-xc_lowpri(u_int flags, xcfunc_t func, void *arg1, void *arg2,
- struct cpu_info *ci)
+static inline uint64_t
+xc_lowpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci)
{
+ xc_state_t *xc = &xc_low_pri;
CPU_INFO_ITERATOR cii;
uint64_t where;
- mutex_enter(&xc_lock);
- while (xc_headp != xc_tailp)
- cv_wait(&xc_busy, &xc_lock);
- xc_arg1 = arg1;
- xc_arg2 = arg2;
- xc_func = func;
+ mutex_enter(&xc->xc_lock);
+ while (xc->xc_headp != xc_tailp) {
+ cv_wait(&xc->xc_busy, &xc->xc_lock);
+ }
+ xc->xc_arg1 = arg1;
+ xc->xc_arg2 = arg2;
+ xc->xc_func = func;
if (ci == NULL) {
xc_broadcast_ev.ev_count++;
for (CPU_INFO_FOREACH(cii, ci)) {
if ((ci->ci_schedstate.spc_flags & SPCF_RUNNING) == 0)
continue;
- xc_headp += 1;
+ xc->xc_headp += 1;
ci->ci_data.cpu_xcall_pending = true;
cv_signal(&ci->ci_data.cpu_xcall);
}
} else {
xc_unicast_ev.ev_count++;
- xc_headp += 1;
+ xc->xc_headp += 1;
ci->ci_data.cpu_xcall_pending = true;
cv_signal(&ci->ci_data.cpu_xcall);
}
- KASSERT(xc_tailp < xc_headp);
- where = xc_headp;
- mutex_exit(&xc_lock);
+ KASSERT(xc_tailp < xc->xc_headp);
+ where = xc->xc_headp;
+ mutex_exit(&xc->xc_lock);
+ /* Return a low priority ticket. */
+ KASSERT((where & XC_PRI_BIT) == 0);
return where;
}
/*
- * xc_wait:
- *
- * Wait for a cross call to complete.
- */
-void
-xc_wait(uint64_t where)
-{
-
- if (xc_donep >= where)
- return;
-
- mutex_enter(&xc_lock);
- while (xc_donep < where)
- cv_wait(&xc_busy, &xc_lock);
- mutex_exit(&xc_lock);
-}
-
-/*
* xc_thread:
*
* One thread per-CPU to dispatch low priority calls.
@@ -229,31 +291,130 @@
static void
xc_thread(void *cookie)
{
+ struct cpu_info *ci = curcpu();
+ xc_state_t *xc = &xc_low_pri;
void *arg1, *arg2;
- struct cpu_info *ci;
xcfunc_t func;
- ci = curcpu();
-
- mutex_enter(&xc_lock);
+ mutex_enter(&xc->xc_lock);
for (;;) {
while (!ci->ci_data.cpu_xcall_pending) {
- if (xc_headp == xc_tailp)
- cv_broadcast(&xc_busy);
- cv_wait(&ci->ci_data.cpu_xcall, &xc_lock);
+ if (xc->xc_headp == xc_tailp) {
+ cv_broadcast(&xc->xc_busy);
+ }
+ cv_wait(&ci->ci_data.cpu_xcall, &xc->xc_lock);
KASSERT(ci == curcpu());
}
ci->ci_data.cpu_xcall_pending = false;
- func = xc_func;
- arg1 = xc_arg1;
- arg2 = xc_arg2;
+ func = xc->xc_func;
+ arg1 = xc->xc_arg1;
+ arg2 = xc->xc_arg2;
xc_tailp++;
- mutex_exit(&xc_lock);
+ mutex_exit(&xc->xc_lock);
+ KASSERT(func != NULL);
(*func)(arg1, arg2);
- mutex_enter(&xc_lock);
- xc_donep++;
+ mutex_enter(&xc->xc_lock);
+ xc->xc_donep++;
}
/* NOTREACHED */
}
+
+/*
+ * xc_ipi_handler:
+ *
+ * Handler of cross-call IPI.
+ */
+void
+xc_ipi_handler(void)
+{
+ /* Executes xc_highpri_intr() via software interrupt. */
+ softint_schedule(xc_sih);
+}
+
+/*
+ * xc_highpri_intr:
+ *
+ * A software interrupt handler for high priority calls.
+ */
+static void
+xc_highpri_intr(void *dummy)
+{
+ xc_state_t *xc = &xc_high_pri;
+ void *arg1, *arg2;
+ xcfunc_t func;
+
+ KASSERT(cpu_softintr_p());
+ /*
+ * Lock-less fetch of function and its arguments.
+ * Safe since it cannot change at this point.
+ */
+ KASSERT(xc->xc_donep < xc->xc_headp);
+ func = xc->xc_func;
+ arg1 = xc->xc_arg1;
+ arg2 = xc->xc_arg2;
+
+ KASSERT(func != NULL);
+ (*func)(arg1, arg2);
+
+ /*
+ * Note the request as done, and if we have reached the head,
+ * cross-call has been processed - notify waiters, if any.
+ */
+ mutex_enter(&xc->xc_lock);
+ if (++xc->xc_donep == xc->xc_headp) {
+ cv_broadcast(&xc->xc_busy);
+ }
+ mutex_exit(&xc->xc_lock);
+}
+
+/*
+ * xc_highpri:
+ *
+ * Trigger a high priority call on one or more CPUs.
+ */
+static inline uint64_t
+xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci)
+{
+ xc_state_t *xc = &xc_high_pri;
+ uint64_t where;
+
+ mutex_enter(&xc->xc_lock);
+ while (xc->xc_headp != xc->xc_donep) {
+ cv_wait(&xc->xc_busy, &xc->xc_lock);
+ }
+ xc->xc_func = func;
+ xc->xc_arg1 = arg1;
+ xc->xc_arg2 = arg2;
+ xc->xc_headp += (ci ? 1 : ncpu);
+ where = xc->xc_headp;
+ mutex_exit(&xc->xc_lock);
+
+ /*
+ * Send the IPI once lock is released.
+ * Note: it will handle the local CPU case.
+ */
+
+#ifdef MULTIPROCESSOR
+ kpreempt_disable();
+ if (curcpu() == ci) {
+ /* Unicast: local CPU. */
+ xc_ipi_handler();
+ } else if (ci) {
+ /* Unicast: remote CPU. */
+ xc_send_ipi(ci);
+ } else {
+ /* Broadcast: all, including local. */
+ xc_send_ipi(NULL);
+ xc_ipi_handler();
+ }
+ kpreempt_enable();
+#else
+ KASSERT(curcpu() == ci);
+ xc_ipi_handler();
+#endif
+
+ /* Indicate a high priority ticket. */
+ return (where | XC_PRI_BIT);
+}
Index: src/sys/sys/xcall.h
diff -u src/sys/sys/xcall.h:1.3 src/sys/sys/xcall.h:1.4
--- src/sys/sys/xcall.h:1.3 Mon Apr 28 20:24:11 2008
+++ src/sys/sys/xcall.h Tue Jun 22 18:29:01 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: xcall.h,v 1.3 2008/04/28 20:24:11 martin Exp $ */
+/* $NetBSD: xcall.h,v 1.4 2010/06/22 18:29:01 rmind Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -40,6 +40,9 @@
struct cpu_info;
void xc_init_cpu(struct cpu_info *);
+void xc_send_ipi(struct cpu_info *);
+void xc_ipi_handler(void);
+
uint64_t xc_broadcast(u_int, xcfunc_t, void *, void *);
uint64_t xc_unicast(u_int, xcfunc_t, void *, void *, struct cpu_info *);
void xc_wait(uint64_t);
