Module Name: src
Committed By: ad
Date: Sun Dec 1 15:34:47 UTC 2019
Modified Files:
src/sys/arch/alpha/alpha: ipifuncs.c
src/sys/arch/alpha/include: cpu.h
src/sys/arch/arm/include: cpu.h
src/sys/arch/hppa/include: cpu.h
src/sys/arch/ia64/include: cpu.h
src/sys/arch/m68k/include: cpu.h
src/sys/arch/mips/include: cpu.h
src/sys/arch/mips/mips: ipifuncs.c
src/sys/arch/mips/rmi: rmixl_cpu.c
src/sys/arch/or1k/include: cpu.h
src/sys/arch/powerpc/booke: e500_intr.c
src/sys/arch/powerpc/include: cpu.h
src/sys/arch/powerpc/pic: ipi.c
src/sys/arch/riscv/include: cpu.h
src/sys/arch/sh3/include: cpu.h
src/sys/arch/sparc/include: cpu.h
src/sys/arch/sparc64/include: cpu.h
src/sys/arch/sparc64/sparc64: clock.c
src/sys/arch/usermode/include: cpu.h
src/sys/arch/vax/include: cpu.h
src/sys/arch/vax/vax: ka6400.c ka820.c ka88.c
src/sys/arch/x86/include: cpu.h
src/sys/arch/x86/x86: ipi.c
src/sys/arch/xen/x86: xen_ipi.c
src/sys/kern: kern_clock.c kern_cpu.c kern_idle.c kern_runq.c
kern_softint.c kern_synch.c sched_4bsd.c sched_m2.c
src/sys/sys: cpu_data.h lwp.h
Log Message:
Fix false sharing problems with cpu_info. Identified with tprof(8).
This was a very nice win in my tests on a 48 CPU box.
- Reorganise cpu_data slightly according to usage.
- Put cpu_onproc into struct cpu_info alongside ci_curlwp (now is ci_onproc).
- On x86, put some items in their own cache lines according to usage, like
the IPI bitmask and ci_want_resched.
To generate a diff of this commit:
cvs rdiff -u -r1.49 -r1.50 src/sys/arch/alpha/alpha/ipifuncs.c
cvs rdiff -u -r1.85 -r1.86 src/sys/arch/alpha/include/cpu.h
cvs rdiff -u -r1.102 -r1.103 src/sys/arch/arm/include/cpu.h
cvs rdiff -u -r1.5 -r1.6 src/sys/arch/hppa/include/cpu.h
cvs rdiff -u -r1.19 -r1.20 src/sys/arch/ia64/include/cpu.h
cvs rdiff -u -r1.16 -r1.17 src/sys/arch/m68k/include/cpu.h
cvs rdiff -u -r1.127 -r1.128 src/sys/arch/mips/include/cpu.h
cvs rdiff -u -r1.12 -r1.13 src/sys/arch/mips/mips/ipifuncs.c
cvs rdiff -u -r1.9 -r1.10 src/sys/arch/mips/rmi/rmixl_cpu.c
cvs rdiff -u -r1.3 -r1.4 src/sys/arch/or1k/include/cpu.h
cvs rdiff -u -r1.39 -r1.40 src/sys/arch/powerpc/booke/e500_intr.c
cvs rdiff -u -r1.109 -r1.110 src/sys/arch/powerpc/include/cpu.h
cvs rdiff -u -r1.13 -r1.14 src/sys/arch/powerpc/pic/ipi.c
cvs rdiff -u -r1.6 -r1.7 src/sys/arch/riscv/include/cpu.h
cvs rdiff -u -r1.58 -r1.59 src/sys/arch/sh3/include/cpu.h
cvs rdiff -u -r1.102 -r1.103 src/sys/arch/sparc/include/cpu.h
cvs rdiff -u -r1.127 -r1.128 src/sys/arch/sparc64/include/cpu.h
cvs rdiff -u -r1.121 -r1.122 src/sys/arch/sparc64/sparc64/clock.c
cvs rdiff -u -r1.12 -r1.13 src/sys/arch/usermode/include/cpu.h
cvs rdiff -u -r1.103 -r1.104 src/sys/arch/vax/include/cpu.h
cvs rdiff -u -r1.19 -r1.20 src/sys/arch/vax/vax/ka6400.c \
src/sys/arch/vax/vax/ka88.c
cvs rdiff -u -r1.56 -r1.57 src/sys/arch/vax/vax/ka820.c
cvs rdiff -u -r1.114 -r1.115 src/sys/arch/x86/include/cpu.h
cvs rdiff -u -r1.29 -r1.30 src/sys/arch/x86/x86/ipi.c
cvs rdiff -u -r1.34 -r1.35 src/sys/arch/xen/x86/xen_ipi.c
cvs rdiff -u -r1.138 -r1.139 src/sys/kern/kern_clock.c
cvs rdiff -u -r1.77 -r1.78 src/sys/kern/kern_cpu.c
cvs rdiff -u -r1.26 -r1.27 src/sys/kern/kern_idle.c
cvs rdiff -u -r1.51 -r1.52 src/sys/kern/kern_runq.c \
src/sys/kern/kern_softint.c
cvs rdiff -u -r1.326 -r1.327 src/sys/kern/kern_synch.c
cvs rdiff -u -r1.39 -r1.40 src/sys/kern/sched_4bsd.c
cvs rdiff -u -r1.35 -r1.36 src/sys/kern/sched_m2.c
cvs rdiff -u -r1.39 -r1.40 src/sys/sys/cpu_data.h
cvs rdiff -u -r1.191 -r1.192 src/sys/sys/lwp.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/alpha/alpha/ipifuncs.c
diff -u src/sys/arch/alpha/alpha/ipifuncs.c:1.49 src/sys/arch/alpha/alpha/ipifuncs.c:1.50
--- src/sys/arch/alpha/alpha/ipifuncs.c:1.49 Thu Nov 21 19:02:43 2019
+++ src/sys/arch/alpha/alpha/ipifuncs.c Sun Dec 1 15:34:44 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ipifuncs.c,v 1.49 2019/11/21 19:02:43 ad Exp $ */
+/* $NetBSD: ipifuncs.c,v 1.50 2019/12/01 15:34:44 ad Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.49 2019/11/21 19:02:43 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.50 2019/12/01 15:34:44 ad Exp $");
/*
* Interprocessor interrupt handlers.
@@ -263,8 +263,8 @@ static void
alpha_ipi_ast(struct cpu_info *ci, struct trapframe *framep)
{
- if (ci->ci_data.cpu_onproc != ci->ci_data.cpu_idlelwp)
- aston(ci->ci_data.cpu_onproc);
+ if (ci->ci_onproc != ci->ci_data.cpu_idlelwp)
+ aston(ci->ci_onproc);
}
static void
Index: src/sys/arch/alpha/include/cpu.h
diff -u src/sys/arch/alpha/include/cpu.h:1.85 src/sys/arch/alpha/include/cpu.h:1.86
--- src/sys/arch/alpha/include/cpu.h:1.85 Sun Nov 24 15:40:24 2019
+++ src/sys/arch/alpha/include/cpu.h Sun Dec 1 15:34:44 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.85 2019/11/24 15:40:24 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.86 2019/12/01 15:34:44 ad Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
@@ -108,6 +108,7 @@ struct cpu_info {
* Public members.
*/
struct lwp *ci_curlwp; /* current owner of the processor */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct cpu_data ci_data; /* MI per-cpu data */
#if !defined(_KMEMUSER)
struct cctr_state ci_cc; /* cycle counter state */
Index: src/sys/arch/arm/include/cpu.h
diff -u src/sys/arch/arm/include/cpu.h:1.102 src/sys/arch/arm/include/cpu.h:1.103
--- src/sys/arch/arm/include/cpu.h:1.102 Thu Nov 21 19:23:59 2019
+++ src/sys/arch/arm/include/cpu.h Sun Dec 1 15:34:44 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.102 2019/11/21 19:23:59 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.103 2019/12/01 15:34:44 ad Exp $ */
/*
* Copyright (c) 1994-1996 Mark Brinicombe.
@@ -152,6 +152,7 @@ struct cpu_info {
ci_softints;
lwp_t * ci_curlwp; /* current lwp */
+ lwp_t * ci_onproc; /* current user LWP / kthread */
lwp_t * ci_lastlwp; /* last lwp */
struct evcnt ci_arm700bugcount;
Index: src/sys/arch/hppa/include/cpu.h
diff -u src/sys/arch/hppa/include/cpu.h:1.5 src/sys/arch/hppa/include/cpu.h:1.6
--- src/sys/arch/hppa/include/cpu.h:1.5 Tue Apr 16 12:25:17 2019
+++ src/sys/arch/hppa/include/cpu.h Sun Dec 1 15:34:44 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.5 2019/04/16 12:25:17 skrll Exp $ */
+/* $NetBSD: cpu.h,v 1.6 2019/12/01 15:34:44 ad Exp $ */
/* $OpenBSD: cpu.h,v 1.55 2008/07/23 17:39:35 kettenis Exp $ */
@@ -295,6 +295,7 @@ struct cpu_info {
#define CPUF_PRIMARY 0x0001 /* ... is monarch/primary */
#define CPUF_RUNNING 0x0002 /* ... is running. */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
volatile u_long ci_ipi; /* IPIs pending */
struct cpu_softc *ci_softc;
Index: src/sys/arch/ia64/include/cpu.h
diff -u src/sys/arch/ia64/include/cpu.h:1.19 src/sys/arch/ia64/include/cpu.h:1.20
--- src/sys/arch/ia64/include/cpu.h:1.19 Sun Nov 24 15:45:41 2019
+++ src/sys/arch/ia64/include/cpu.h Sun Dec 1 15:34:44 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.19 2019/11/24 15:45:41 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.20 2019/12/01 15:34:44 ad Exp $ */
/*-
* Copyright (c) 2006 The NetBSD Foundation, Inc.
@@ -90,6 +90,7 @@ struct cpu_info {
struct cpu_data ci_data; /* MI per-cpu data */
device_t ci_dev; /* pointer to our device */
struct lwp *ci_curlwp; /* current owner of the processor */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct cctr_state ci_cc; /* cycle counter state */
struct cpu_info *ci_next; /* next cpu_info structure */
Index: src/sys/arch/m68k/include/cpu.h
diff -u src/sys/arch/m68k/include/cpu.h:1.16 src/sys/arch/m68k/include/cpu.h:1.17
--- src/sys/arch/m68k/include/cpu.h:1.16 Wed Aug 22 01:05:22 2018
+++ src/sys/arch/m68k/include/cpu.h Sun Dec 1 15:34:44 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.16 2018/08/22 01:05:22 msaitoh Exp $ */
+/* $NetBSD: cpu.h,v 1.17 2019/12/01 15:34:44 ad Exp $ */
/*
* Copyright (c) 1988 University of Utah.
@@ -177,6 +177,7 @@ struct cpu_info {
int ci_mtx_oldspl;
volatile int ci_want_resched;
volatile int ci_idepth;
+ struct lwp *ci_onproc; /* current user LWP / kthread */
};
#endif /* _KERNEL || _KMEMUSER */
Index: src/sys/arch/mips/include/cpu.h
diff -u src/sys/arch/mips/include/cpu.h:1.127 src/sys/arch/mips/include/cpu.h:1.128
--- src/sys/arch/mips/include/cpu.h:1.127 Thu Nov 21 19:24:00 2019
+++ src/sys/arch/mips/include/cpu.h Sun Dec 1 15:34:44 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.127 2019/11/21 19:24:00 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.128 2019/12/01 15:34:44 ad Exp $ */
/*-
* Copyright (c) 1992, 1993
@@ -95,6 +95,7 @@ struct cpu_info {
u_long ci_divisor_delay; /* for delay/DELAY */
u_long ci_divisor_recip; /* unused, for obsolete microtime(9) */
struct lwp *ci_curlwp; /* currently running lwp */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
volatile int ci_want_resched; /* user preemption pending */
int ci_mtx_count; /* negative count of held mutexes */
int ci_mtx_oldspl; /* saved SPL value */
Index: src/sys/arch/mips/mips/ipifuncs.c
diff -u src/sys/arch/mips/mips/ipifuncs.c:1.12 src/sys/arch/mips/mips/ipifuncs.c:1.13
--- src/sys/arch/mips/mips/ipifuncs.c:1.12 Sat Nov 23 19:40:35 2019
+++ src/sys/arch/mips/mips/ipifuncs.c Sun Dec 1 15:34:44 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ipifuncs.c,v 1.12 2019/11/23 19:40:35 ad Exp $ */
+/* $NetBSD: ipifuncs.c,v 1.13 2019/12/01 15:34:44 ad Exp $ */
/*-
* Copyright (c) 2010 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
#include "opt_ddb.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.12 2019/11/23 19:40:35 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.13 2019/12/01 15:34:44 ad Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@@ -76,7 +76,7 @@ ipi_nop(struct cpu_info *ci)
static void
ipi_ast(struct cpu_info *ci)
{
- ci->ci_data.cpu_onproc->l_md.md_astpending = 1;
+ ci->ci_onproc->l_md.md_astpending = 1;
}
static void
Index: src/sys/arch/mips/rmi/rmixl_cpu.c
diff -u src/sys/arch/mips/rmi/rmixl_cpu.c:1.9 src/sys/arch/mips/rmi/rmixl_cpu.c:1.10
--- src/sys/arch/mips/rmi/rmixl_cpu.c:1.9 Sun Jun 28 22:14:38 2015
+++ src/sys/arch/mips/rmi/rmixl_cpu.c Sun Dec 1 15:34:45 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: rmixl_cpu.c,v 1.9 2015/06/28 22:14:38 matt Exp $ */
+/* $NetBSD: rmixl_cpu.c,v 1.10 2019/12/01 15:34:45 ad Exp $ */
/*
* Copyright 2002 Wasabi Systems, Inc.
@@ -38,7 +38,7 @@
#include "locators.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: rmixl_cpu.c,v 1.9 2015/06/28 22:14:38 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: rmixl_cpu.c,v 1.10 2019/12/01 15:34:45 ad Exp $");
#include "opt_multiprocessor.h"
#include "opt_ddb.h"
@@ -419,7 +419,6 @@ rmixl_cpu_data_print(struct cpu_data *dp
printf("&cpu_schedstate %p\n", &dp->cpu_schedstate); /* TBD */
printf("&cpu_xcall %p\n", &dp->cpu_xcall); /* TBD */
printf("cpu_xcall_pending %d\n", dp->cpu_xcall_pending);
- printf("cpu_onproc %p\n", dp->cpu_onproc);
printf("cpu_idlelwp %p\n", dp->cpu_idlelwp);
printf("cpu_lockstat %p\n", dp->cpu_lockstat);
printf("cpu_index %d\n", dp->cpu_index);
@@ -462,6 +461,7 @@ rmixl_cpuinfo_print(u_int cpuindex)
printf("ci_divisor_delay %ld\n", ci->ci_divisor_delay);
printf("ci_divisor_recip %ld\n", ci->ci_divisor_recip);
printf("ci_curlwp %p\n", ci->ci_curlwp);
+ printf("ci_onproc %p\n", dp->ci_onproc);
printf("ci_want_resched %d\n", ci->ci_want_resched);
printf("ci_mtx_count %d\n", ci->ci_mtx_count);
printf("ci_mtx_oldspl %d\n", ci->ci_mtx_oldspl);
Index: src/sys/arch/or1k/include/cpu.h
diff -u src/sys/arch/or1k/include/cpu.h:1.3 src/sys/arch/or1k/include/cpu.h:1.4
--- src/sys/arch/or1k/include/cpu.h:1.3 Thu Nov 21 19:24:01 2019
+++ src/sys/arch/or1k/include/cpu.h Sun Dec 1 15:34:45 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.3 2019/11/21 19:24:01 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.4 2019/12/01 15:34:45 ad Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -52,6 +52,7 @@ struct cpu_info {
device_t ci_dev;
cpuid_t ci_cpuid;
struct lwp *ci_curlwp;
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct lwp *ci_softlwps[SOFTINT_COUNT];
uint64_t ci_lastintr;
Index: src/sys/arch/powerpc/booke/e500_intr.c
diff -u src/sys/arch/powerpc/booke/e500_intr.c:1.39 src/sys/arch/powerpc/booke/e500_intr.c:1.40
--- src/sys/arch/powerpc/booke/e500_intr.c:1.39 Sat Nov 23 19:40:36 2019
+++ src/sys/arch/powerpc/booke/e500_intr.c Sun Dec 1 15:34:45 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: e500_intr.c,v 1.39 2019/11/23 19:40:36 ad Exp $ */
+/* $NetBSD: e500_intr.c,v 1.40 2019/12/01 15:34:45 ad Exp $ */
/*-
* Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -41,7 +41,7 @@
#define __INTR_PRIVATE
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: e500_intr.c,v 1.39 2019/11/23 19:40:36 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: e500_intr.c,v 1.40 2019/12/01 15:34:45 ad Exp $");
#include <sys/param.h>
#include <sys/proc.h>
@@ -1338,7 +1338,7 @@ e500_ipi_suspend(void)
static void
e500_ipi_ast(void)
{
- curcpu()->ci_data.cpu_onproc->l_md.md_astpending = 1;
+ curcpu()->ci_onproc->l_md.md_astpending = 1;
}
static const ipifunc_t e500_ipifuncs[] = {
Index: src/sys/arch/powerpc/include/cpu.h
diff -u src/sys/arch/powerpc/include/cpu.h:1.109 src/sys/arch/powerpc/include/cpu.h:1.110
--- src/sys/arch/powerpc/include/cpu.h:1.109 Sat Nov 23 19:40:36 2019
+++ src/sys/arch/powerpc/include/cpu.h Sun Dec 1 15:34:45 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.109 2019/11/23 19:40:36 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.110 2019/12/01 15:34:45 ad Exp $ */
/*
* Copyright (C) 1999 Wolfgang Solfrank.
@@ -67,7 +67,7 @@ struct cpu_info {
device_t ci_dev; /* device of corresponding cpu */
struct cpu_softc *ci_softc; /* private cpu info */
struct lwp *ci_curlwp; /* current owner of the processor */
-
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct pcb *ci_curpcb;
struct pmap *ci_curpm;
struct lwp *ci_softlwps[SOFTINT_COUNT];
Index: src/sys/arch/powerpc/pic/ipi.c
diff -u src/sys/arch/powerpc/pic/ipi.c:1.13 src/sys/arch/powerpc/pic/ipi.c:1.14
--- src/sys/arch/powerpc/pic/ipi.c:1.13 Sun Nov 24 15:49:12 2019
+++ src/sys/arch/powerpc/pic/ipi.c Sun Dec 1 15:34:45 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ipi.c,v 1.13 2019/11/24 15:49:12 ad Exp $ */
+/* $NetBSD: ipi.c,v 1.14 2019/12/01 15:34:45 ad Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.13 2019/11/24 15:49:12 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.14 2019/12/01 15:34:45 ad Exp $");
#include "opt_multiprocessor.h"
#include "opt_pic.h"
@@ -79,7 +79,7 @@ ipi_intr(void *v)
cpu_pause(NULL);
if (ipi & IPI_AST)
- ci->ci_data.cpu_onproc->l_md.md_astpending = 1;
+ ci->ci_onproc->l_md.md_astpending = 1;
if (ipi & IPI_HALT) {
struct cpuset_info * const csi = &cpuset_info;
Index: src/sys/arch/riscv/include/cpu.h
diff -u src/sys/arch/riscv/include/cpu.h:1.6 src/sys/arch/riscv/include/cpu.h:1.7
--- src/sys/arch/riscv/include/cpu.h:1.6 Thu Nov 21 19:24:01 2019
+++ src/sys/arch/riscv/include/cpu.h Sun Dec 1 15:34:45 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.6 2019/11/21 19:24:01 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.7 2019/12/01 15:34:45 ad Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -54,6 +54,7 @@ struct cpu_info {
device_t ci_dev;
cpuid_t ci_cpuid;
struct lwp *ci_curlwp;
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct lwp *ci_softlwps[SOFTINT_COUNT];
struct trapframe *ci_ddb_regs;
Index: src/sys/arch/sh3/include/cpu.h
diff -u src/sys/arch/sh3/include/cpu.h:1.58 src/sys/arch/sh3/include/cpu.h:1.59
--- src/sys/arch/sh3/include/cpu.h:1.58 Sat Nov 23 19:40:36 2019
+++ src/sys/arch/sh3/include/cpu.h Sun Dec 1 15:34:45 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.58 2019/11/23 19:40:36 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.59 2019/12/01 15:34:45 ad Exp $ */
/*-
* Copyright (c) 2002, 2019 The NetBSD Foundation, Inc. All rights reserved.
@@ -60,6 +60,7 @@ struct cpu_info {
int ci_mtx_oldspl;
int ci_want_resched;
int ci_idepth;
+ struct lwp *ci_onproc; /* current user LWP / kthread */
};
extern struct cpu_info cpu_info_store;
Index: src/sys/arch/sparc/include/cpu.h
diff -u src/sys/arch/sparc/include/cpu.h:1.102 src/sys/arch/sparc/include/cpu.h:1.103
--- src/sys/arch/sparc/include/cpu.h:1.102 Sat Nov 23 19:40:36 2019
+++ src/sys/arch/sparc/include/cpu.h Sun Dec 1 15:34:45 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.102 2019/11/23 19:40:36 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.103 2019/12/01 15:34:45 ad Exp $ */
/*
* Copyright (c) 1992, 1993
@@ -240,6 +240,7 @@ struct cpu_info {
* etc.
*/
struct lwp *ci_curlwp; /* CPU owner */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct lwp *fplwp; /* FPU owner */
int ci_mtx_count;
Index: src/sys/arch/sparc64/include/cpu.h
diff -u src/sys/arch/sparc64/include/cpu.h:1.127 src/sys/arch/sparc64/include/cpu.h:1.128
--- src/sys/arch/sparc64/include/cpu.h:1.127 Sat Nov 23 19:40:36 2019
+++ src/sys/arch/sparc64/include/cpu.h Sun Dec 1 15:34:45 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.127 2019/11/23 19:40:36 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.128 2019/12/01 15:34:45 ad Exp $ */
/*
* Copyright (c) 1992, 1993
@@ -123,6 +123,7 @@ struct cpu_info {
/* Most important fields first */
struct lwp *ci_curlwp;
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct pcb *ci_cpcb;
struct cpu_info *ci_next;
Index: src/sys/arch/sparc64/sparc64/clock.c
diff -u src/sys/arch/sparc64/sparc64/clock.c:1.121 src/sys/arch/sparc64/sparc64/clock.c:1.122
--- src/sys/arch/sparc64/sparc64/clock.c:1.121 Fri Feb 8 20:14:51 2019
+++ src/sys/arch/sparc64/sparc64/clock.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: clock.c,v 1.121 2019/02/08 20:14:51 palle Exp $ */
+/* $NetBSD: clock.c,v 1.122 2019/12/01 15:34:46 ad Exp $ */
/*
* Copyright (c) 1992, 1993
@@ -55,7 +55,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: clock.c,v 1.121 2019/02/08 20:14:51 palle Exp $");
+__KERNEL_RCSID(0, "$NetBSD: clock.c,v 1.122 2019/12/01 15:34:46 ad Exp $");
#include "opt_multiprocessor.h"
@@ -760,7 +760,7 @@ int
schedintr(void *arg)
{
- schedclock(curcpu()->ci_data.cpu_onproc);
+ schedclock(curcpu()->ci_onproc);
return (1);
}
#endif
Index: src/sys/arch/usermode/include/cpu.h
diff -u src/sys/arch/usermode/include/cpu.h:1.12 src/sys/arch/usermode/include/cpu.h:1.13
--- src/sys/arch/usermode/include/cpu.h:1.12 Sat Nov 23 19:40:37 2019
+++ src/sys/arch/usermode/include/cpu.h Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.12 2019/11/23 19:40:37 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.13 2019/12/01 15:34:46 ad Exp $ */
/*-
* Copyright (c) 2007 Jared D. McNeill <jmcne...@invisible.ca>
@@ -58,6 +58,7 @@ struct cpu_info {
int ci_idepth;
volatile int ci_mtx_count;
volatile int ci_mtx_oldspl;
+ lwp_t *ci_onproc; /* current user LWP / kthread */
lwp_t *ci_curlwp;
lwp_t *ci_stash;
};
Index: src/sys/arch/vax/include/cpu.h
diff -u src/sys/arch/vax/include/cpu.h:1.103 src/sys/arch/vax/include/cpu.h:1.104
--- src/sys/arch/vax/include/cpu.h:1.103 Sat Nov 23 19:40:37 2019
+++ src/sys/arch/vax/include/cpu.h Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.103 2019/11/23 19:40:37 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.104 2019/12/01 15:34:46 ad Exp $ */
/*
* Copyright (c) 1994 Ludd, University of Lule}, Sweden
@@ -133,6 +133,7 @@ struct cpu_info {
vaddr_t ci_istack; /* Interrupt stack location */
const char *ci_cpustr;
int ci_slotid; /* cpu slot */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
#if defined(MULTIPROCESSOR)
struct lwp *ci_curlwp; /* current lwp (for other cpus) */
volatile int ci_flags; /* See below */
Index: src/sys/arch/vax/vax/ka6400.c
diff -u src/sys/arch/vax/vax/ka6400.c:1.19 src/sys/arch/vax/vax/ka6400.c:1.20
--- src/sys/arch/vax/vax/ka6400.c:1.19 Mon May 22 16:46:15 2017
+++ src/sys/arch/vax/vax/ka6400.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ka6400.c,v 1.19 2017/05/22 16:46:15 ragge Exp $ */
+/* $NetBSD: ka6400.c,v 1.20 2019/12/01 15:34:46 ad Exp $ */
/*
* Copyright (c) 2000 Ludd, University of Lule}, Sweden. All rights reserved.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ka6400.c,v 1.19 2017/05/22 16:46:15 ragge Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ka6400.c,v 1.20 2019/12/01 15:34:46 ad Exp $");
#include "opt_multiprocessor.h"
@@ -330,7 +330,7 @@ rxchar(void)
static void
ka6400_startslave(struct cpu_info *ci)
{
- const struct pcb *pcb = lwp_getpcb(ci->ci_data.cpu_onproc);
+ const struct pcb *pcb = lwp_getpcb(ci->ci_onproc);
const int id = ci->ci_slotid;
int i;
Index: src/sys/arch/vax/vax/ka88.c
diff -u src/sys/arch/vax/vax/ka88.c:1.19 src/sys/arch/vax/vax/ka88.c:1.20
--- src/sys/arch/vax/vax/ka88.c:1.19 Mon May 22 16:53:05 2017
+++ src/sys/arch/vax/vax/ka88.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ka88.c,v 1.19 2017/05/22 16:53:05 ragge Exp $ */
+/* $NetBSD: ka88.c,v 1.20 2019/12/01 15:34:46 ad Exp $ */
/*
* Copyright (c) 2000 Ludd, University of Lule}, Sweden. All rights reserved.
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ka88.c,v 1.19 2017/05/22 16:53:05 ragge Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ka88.c,v 1.20 2019/12/01 15:34:46 ad Exp $");
#include "opt_multiprocessor.h"
@@ -364,7 +364,7 @@ rxchar(void)
static void
ka88_startslave(struct cpu_info *ci)
{
- const struct pcb *pcb = lwp_getpcb(ci->ci_data.cpu_onproc);
+ const struct pcb *pcb = lwp_getpcb(ci->ci_onproc);
const int id = ci->ci_slotid;
int i;
Index: src/sys/arch/vax/vax/ka820.c
diff -u src/sys/arch/vax/vax/ka820.c:1.56 src/sys/arch/vax/vax/ka820.c:1.57
--- src/sys/arch/vax/vax/ka820.c:1.56 Wed Mar 26 08:01:21 2014
+++ src/sys/arch/vax/vax/ka820.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ka820.c,v 1.56 2014/03/26 08:01:21 christos Exp $ */
+/* $NetBSD: ka820.c,v 1.57 2019/12/01 15:34:46 ad Exp $ */
/*
* Copyright (c) 1988 Regents of the University of California.
* All rights reserved.
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ka820.c,v 1.56 2014/03/26 08:01:21 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ka820.c,v 1.57 2019/12/01 15:34:46 ad Exp $");
#include "opt_multiprocessor.h"
@@ -530,7 +530,7 @@ ka820_settime(struct timeval *tvp)
static void
ka820_startslave(struct cpu_info *ci)
{
- const struct pcb *pcb = lwp_getpcb(ci->ci_data.cpu_onproc);
+ const struct pcb *pcb = lwp_getpcb(ci->ci_onproc);
const int id = ci->ci_slotid;
int i;
Index: src/sys/arch/x86/include/cpu.h
diff -u src/sys/arch/x86/include/cpu.h:1.114 src/sys/arch/x86/include/cpu.h:1.115
--- src/sys/arch/x86/include/cpu.h:1.114 Wed Nov 27 06:24:33 2019
+++ src/sys/arch/x86/include/cpu.h Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.114 2019/11/27 06:24:33 maxv Exp $ */
+/* $NetBSD: cpu.h,v 1.115 2019/12/01 15:34:46 ad Exp $ */
/*
* Copyright (c) 1990 The Regents of the University of California.
@@ -116,15 +116,6 @@ struct cpu_info {
#endif
/*
- * Will be accessed by other CPUs.
- */
- struct cpu_info *ci_next; /* next cpu */
- struct lwp *ci_curlwp; /* current owner of the processor */
- cpuid_t ci_cpuid; /* our CPU ID */
- uint32_t ci_acpiid; /* our ACPI/MADT ID */
- uint32_t ci_initapicid; /* our initial APIC ID */
-
- /*
* Private members.
*/
struct pmap *ci_pmap; /* current pmap */
@@ -166,11 +157,8 @@ struct cpu_info {
uint32_t ci_imask[NIPL];
uint32_t ci_iunmask[NIPL];
- uint32_t ci_flags; /* flags; see below */
- uint32_t ci_ipis; /* interprocessor interrupts pending */
-
- uint32_t ci_signature; /* X86 cpuid type (cpuid.1.%eax) */
- uint32_t ci_vendor[4]; /* vendor string */
+ uint32_t ci_signature; /* X86 cpuid type (cpuid.1.%eax) */
+ uint32_t ci_vendor[4]; /* vendor string */
uint32_t ci_max_cpuid; /* cpuid.0:%eax */
uint32_t ci_max_ext_cpuid; /* cpuid.80000000:%eax */
volatile uint32_t ci_lapic_counter;
@@ -226,8 +214,31 @@ struct cpu_info {
uintptr_t ci_suspend_cr4;
uintptr_t ci_suspend_cr8;
- /* The following must be in a single cache line. */
+ /*
+ * The following must be in their own cache line, as they are
+ * stored to regularly by remote CPUs; when they were mixed with
+ * other fields we observed frequent cache misses.
+ */
int ci_want_resched __aligned(64);
+ uint32_t ci_ipis; /* interprocessor interrupts pending */
+
+ /*
+ * These are largely static, and will be frequently fetched by other
+ * CPUs. For that reason they get their own cache line, too.
+ */
+ uint32_t ci_flags __aligned(64);/* general flags */
+ uint32_t ci_acpiid; /* our ACPI/MADT ID */
+ uint32_t ci_initapicid; /* our initial APIC ID */
+ cpuid_t ci_cpuid; /* our CPU ID */
+ struct cpu_info *ci_next; /* next cpu */
+
+ /*
+ * This is stored frequently, and is fetched by remote CPUs.
+ */
+ struct lwp *ci_curlwp __aligned(64);/* general flags */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
+
+ /* Here ends the cachline-aligned sections. */
int ci_padout __aligned(64);
#ifndef __HAVE_DIRECT_MAP
Index: src/sys/arch/x86/x86/ipi.c
diff -u src/sys/arch/x86/x86/ipi.c:1.29 src/sys/arch/x86/x86/ipi.c:1.30
--- src/sys/arch/x86/x86/ipi.c:1.29 Sat Nov 23 19:40:37 2019
+++ src/sys/arch/x86/x86/ipi.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ipi.c,v 1.29 2019/11/23 19:40:37 ad Exp $ */
+/* $NetBSD: ipi.c,v 1.30 2019/12/01 15:34:46 ad Exp $ */
/*-
* Copyright (c) 2000, 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.29 2019/11/23 19:40:37 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.30 2019/12/01 15:34:46 ad Exp $");
#include "opt_mtrr.h"
@@ -216,7 +216,7 @@ static void
x86_ipi_ast(struct cpu_info *ci)
{
- aston(ci->ci_data.cpu_onproc);
+ aston(ci->ci_onproc);
}
/*
Index: src/sys/arch/xen/x86/xen_ipi.c
diff -u src/sys/arch/xen/x86/xen_ipi.c:1.34 src/sys/arch/xen/x86/xen_ipi.c:1.35
--- src/sys/arch/xen/x86/xen_ipi.c:1.34 Sat Nov 23 19:40:38 2019
+++ src/sys/arch/xen/x86/xen_ipi.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: xen_ipi.c,v 1.34 2019/11/23 19:40:38 ad Exp $ */
+/* $NetBSD: xen_ipi.c,v 1.35 2019/12/01 15:34:46 ad Exp $ */
/*-
* Copyright (c) 2011, 2019 The NetBSD Foundation, Inc.
@@ -33,10 +33,10 @@
/*
* Based on: x86/ipi.c
- * __KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.34 2019/11/23 19:40:38 ad Exp $");
+ * __KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.35 2019/12/01 15:34:46 ad Exp $");
*/
-__KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.34 2019/11/23 19:40:38 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.35 2019/12/01 15:34:46 ad Exp $");
#include "opt_ddb.h"
@@ -291,7 +291,7 @@ xen_ipi_ast(struct cpu_info *ci, struct
KASSERT(ci != NULL);
KASSERT(intrf != NULL);
- aston(ci->ci_data.cpu_onproc);
+ aston(ci->ci_onproc);
}
void
Index: src/sys/kern/kern_clock.c
diff -u src/sys/kern/kern_clock.c:1.138 src/sys/kern/kern_clock.c:1.139
--- src/sys/kern/kern_clock.c:1.138 Mon Sep 3 21:29:30 2018
+++ src/sys/kern/kern_clock.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_clock.c,v 1.138 2018/09/03 21:29:30 riastradh Exp $ */
+/* $NetBSD: kern_clock.c,v 1.139 2019/12/01 15:34:46 ad Exp $ */
/*-
* Copyright (c) 2000, 2004, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -69,7 +69,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_clock.c,v 1.138 2018/09/03 21:29:30 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_clock.c,v 1.139 2019/12/01 15:34:46 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_dtrace.h"
@@ -219,7 +219,7 @@ hardclock(struct clockframe *frame)
struct cpu_info *ci;
ci = curcpu();
- l = ci->ci_data.cpu_onproc;
+ l = ci->ci_onproc;
timer_tick(l, CLKF_USERMODE(frame));
@@ -340,7 +340,7 @@ statclock(struct clockframe *frame)
setstatclockrate(profhz);
}
}
- l = ci->ci_data.cpu_onproc;
+ l = ci->ci_onproc;
if ((l->l_flag & LW_IDLE) != 0) {
/*
* don't account idle lwps as swapper.
Index: src/sys/kern/kern_cpu.c
diff -u src/sys/kern/kern_cpu.c:1.77 src/sys/kern/kern_cpu.c:1.78
--- src/sys/kern/kern_cpu.c:1.77 Wed Nov 13 01:31:47 2019
+++ src/sys/kern/kern_cpu.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_cpu.c,v 1.77 2019/11/13 01:31:47 mrg Exp $ */
+/* $NetBSD: kern_cpu.c,v 1.78 2019/12/01 15:34:46 ad Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc.
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.77 2019/11/13 01:31:47 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.78 2019/12/01 15:34:46 ad Exp $");
#include "opt_cpu_ucode.h"
@@ -185,9 +185,9 @@ mi_cpu_attach(struct cpu_info *ci)
}
if (ci == curcpu())
- ci->ci_data.cpu_onproc = curlwp;
+ ci->ci_onproc = curlwp;
else
- ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp;
+ ci->ci_onproc = ci->ci_data.cpu_idlelwp;
percpu_init_cpu(ci);
softint_init(ci);
Index: src/sys/kern/kern_idle.c
diff -u src/sys/kern/kern_idle.c:1.26 src/sys/kern/kern_idle.c:1.27
--- src/sys/kern/kern_idle.c:1.26 Sat Nov 23 19:42:52 2019
+++ src/sys/kern/kern_idle.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_idle.c,v 1.26 2019/11/23 19:42:52 ad Exp $ */
+/* $NetBSD: kern_idle.c,v 1.27 2019/12/01 15:34:46 ad Exp $ */
/*-
* Copyright (c)2002, 2006, 2007 YAMAMOTO Takashi,
@@ -28,7 +28,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_idle.c,v 1.26 2019/11/23 19:42:52 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_idle.c,v 1.27 2019/12/01 15:34:46 ad Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@@ -51,7 +51,7 @@ idle_loop(void *dummy)
kcpuset_atomic_set(kcpuset_running, cpu_index(ci));
spc = &ci->ci_schedstate;
- ci->ci_data.cpu_onproc = l;
+ ci->ci_onproc = l;
/* Update start time for this thread. */
lwp_lock(l);
Index: src/sys/kern/kern_runq.c
diff -u src/sys/kern/kern_runq.c:1.51 src/sys/kern/kern_runq.c:1.52
--- src/sys/kern/kern_runq.c:1.51 Sun Dec 1 13:20:42 2019
+++ src/sys/kern/kern_runq.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_runq.c,v 1.51 2019/12/01 13:20:42 ad Exp $ */
+/* $NetBSD: kern_runq.c,v 1.52 2019/12/01 15:34:46 ad Exp $ */
/*-
* Copyright (c) 2019 The NetBSD Foundation, Inc.
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_runq.c,v 1.51 2019/12/01 13:20:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_runq.c,v 1.52 2019/12/01 15:34:46 ad Exp $");
#include "opt_dtrace.h"
@@ -378,7 +378,7 @@ sched_resched_cpu(struct cpu_info *ci, p
/*
* Figure out what kind of preemption we should do.
*/
- l = ci->ci_data.cpu_onproc;
+ l = ci->ci_onproc;
if ((l->l_flag & LW_IDLE) != 0) {
f = RESCHED_IDLE | RESCHED_UPREEMPT;
} else if (pri >= sched_kpreempt_pri && (l->l_pflag & LP_INTR) == 0) {
Index: src/sys/kern/kern_softint.c
diff -u src/sys/kern/kern_softint.c:1.51 src/sys/kern/kern_softint.c:1.52
--- src/sys/kern/kern_softint.c:1.51 Mon Nov 25 17:24:59 2019
+++ src/sys/kern/kern_softint.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_softint.c,v 1.51 2019/11/25 17:24:59 ad Exp $ */
+/* $NetBSD: kern_softint.c,v 1.52 2019/12/01 15:34:46 ad Exp $ */
/*-
* Copyright (c) 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -170,7 +170,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.51 2019/11/25 17:24:59 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.52 2019/12/01 15:34:46 ad Exp $");
#include <sys/param.h>
#include <sys/proc.h>
@@ -691,7 +691,7 @@ softint_trigger(uintptr_t machdep)
ci = curcpu();
ci->ci_data.cpu_softints |= machdep;
- l = ci->ci_data.cpu_onproc;
+ l = ci->ci_onproc;
if (l == ci->ci_data.cpu_idlelwp) {
atomic_or_uint(&ci->ci_want_resched, RESCHED_UPREEMPT);
} else {
Index: src/sys/kern/kern_synch.c
diff -u src/sys/kern/kern_synch.c:1.326 src/sys/kern/kern_synch.c:1.327
--- src/sys/kern/kern_synch.c:1.326 Sat Nov 23 19:42:52 2019
+++ src/sys/kern/kern_synch.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_synch.c,v 1.326 2019/11/23 19:42:52 ad Exp $ */
+/* $NetBSD: kern_synch.c,v 1.327 2019/12/01 15:34:46 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009, 2019
@@ -69,7 +69,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.326 2019/11/23 19:42:52 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.327 2019/12/01 15:34:46 ad Exp $");
#include "opt_kstack.h"
#include "opt_dtrace.h"
@@ -599,12 +599,12 @@ mi_switch(lwp_t *l)
/*
* ci_curlwp changes when a fast soft interrupt occurs.
- * We use cpu_onproc to keep track of which kernel or
+ * We use ci_onproc to keep track of which kernel or
* user thread is running 'underneath' the software
* interrupt. This is important for time accounting,
* itimers and forcing user threads to preempt (aston).
*/
- ci->ci_data.cpu_onproc = newl;
+ ci->ci_onproc = newl;
}
/*
@@ -825,12 +825,12 @@ lwp_exit_switchaway(lwp_t *l)
/*
* ci_curlwp changes when a fast soft interrupt occurs.
- * We use cpu_onproc to keep track of which kernel or
+ * We use ci_onproc to keep track of which kernel or
* user thread is running 'underneath' the software
* interrupt. This is important for time accounting,
* itimers and forcing user threads to preempt (aston).
*/
- ci->ci_data.cpu_onproc = newl;
+ ci->ci_onproc = newl;
/* Unlock the run queue. */
spc_unlock(ci);
Index: src/sys/kern/sched_4bsd.c
diff -u src/sys/kern/sched_4bsd.c:1.39 src/sys/kern/sched_4bsd.c:1.40
--- src/sys/kern/sched_4bsd.c:1.39 Sun Dec 1 13:20:42 2019
+++ src/sys/kern/sched_4bsd.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: sched_4bsd.c,v 1.39 2019/12/01 13:20:42 ad Exp $ */
+/* $NetBSD: sched_4bsd.c,v 1.40 2019/12/01 15:34:46 ad Exp $ */
/*
* Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2019
@@ -69,7 +69,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sched_4bsd.c,v 1.39 2019/12/01 13:20:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sched_4bsd.c,v 1.40 2019/12/01 15:34:46 ad Exp $");
#include "opt_ddb.h"
#include "opt_lockdebug.h"
@@ -113,7 +113,7 @@ sched_tick(struct cpu_info *ci)
RESCHED_IDLE | RESCHED_UPREEMPT);
return;
}
- l = ci->ci_data.cpu_onproc;
+ l = ci->ci_onproc;
if (l == NULL) {
return;
}
Index: src/sys/kern/sched_m2.c
diff -u src/sys/kern/sched_m2.c:1.35 src/sys/kern/sched_m2.c:1.36
--- src/sys/kern/sched_m2.c:1.35 Sun Dec 1 13:20:42 2019
+++ src/sys/kern/sched_m2.c Sun Dec 1 15:34:46 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: sched_m2.c,v 1.35 2019/12/01 13:20:42 ad Exp $ */
+/* $NetBSD: sched_m2.c,v 1.36 2019/12/01 15:34:46 ad Exp $ */
/*
* Copyright (c) 2007, 2008 Mindaugas Rasiukevicius <rmind at NetBSD org>
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sched_m2.c,v 1.35 2019/12/01 13:20:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sched_m2.c,v 1.36 2019/12/01 15:34:46 ad Exp $");
#include <sys/param.h>
@@ -288,7 +288,7 @@ void
sched_tick(struct cpu_info *ci)
{
struct schedstate_percpu *spc = &ci->ci_schedstate;
- struct lwp *l = ci->ci_data.cpu_onproc;
+ struct lwp *l = ci->ci_onproc;
struct proc *p;
if (__predict_false(CURCPU_IDLE_P()))
Index: src/sys/sys/cpu_data.h
diff -u src/sys/sys/cpu_data.h:1.39 src/sys/sys/cpu_data.h:1.40
--- src/sys/sys/cpu_data.h:1.39 Thu Jan 3 09:09:39 2019
+++ src/sys/sys/cpu_data.h Sun Dec 1 15:34:47 2019
@@ -1,7 +1,7 @@
-/* $NetBSD: cpu_data.h,v 1.39 2019/01/03 09:09:39 skrll Exp $ */
+/* $NetBSD: cpu_data.h,v 1.40 2019/12/01 15:34:47 ad Exp $ */
/*-
- * Copyright (c) 2004, 2006, 2007, 2008 The NetBSD Foundation, Inc.
+ * Copyright (c) 2004, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -64,28 +64,18 @@ struct cpu_data {
* The first section is likely to be touched by other CPUs -
* it is cache hot.
*/
+ u_int cpu_index; /* CPU index */
lwp_t *cpu_biglock_wanted; /* LWP spinning on biglock */
- void *cpu_callout; /* per-CPU callout state */
- void *cpu_unused1; /* unused */
- u_int cpu_unused2; /* unused */
- struct schedstate_percpu cpu_schedstate; /* scheduler state */
kcondvar_t cpu_xcall; /* cross-call support */
int cpu_xcall_pending; /* cross-call support */
- lwp_t *cpu_onproc; /* bottom level LWP */
uint32_t cpu_ipipend[IPI_BITWORDS]; /* pending IPIs */
-
- cpuid_t cpu_package_id;
- cpuid_t cpu_core_id;
- cpuid_t cpu_smt_id;
-
- struct lwp * volatile cpu_pcu_curlwp[PCU_UNIT_COUNT];
+ struct schedstate_percpu cpu_schedstate; /* scheduler state */
/*
* This section is mostly CPU-private.
*/
- lwp_t *cpu_idlelwp; /* idle lwp */
+ lwp_t *cpu_idlelwp __aligned(64);/* idle lwp */
void *cpu_lockstat; /* lockstat private tables */
- u_int cpu_index; /* CPU index */
u_int cpu_biglock_count; /* # recursive holds */
u_int cpu_spin_locks; /* # of spinlockmgr locks */
u_int cpu_simple_locks; /* # of simple locks held */
@@ -99,6 +89,7 @@ struct cpu_data {
uint64_t cpu_nsoft; /* soft interrupt count */
uint64_t cpu_nfault; /* pagefault counter */
struct uvm_cpu *cpu_uvm; /* uvm per-cpu data */
+ void *cpu_callout; /* per-CPU callout state */
void *cpu_softcpu; /* soft interrupt table */
TAILQ_HEAD(,buf) cpu_biodone; /* finished block xfers */
percpu_cpu_t cpu_percpu; /* per-cpu data */
@@ -110,6 +101,10 @@ struct cpu_data {
int64_t cpu_cc_skew; /* counter skew vs cpu0 */
char cpu_name[8]; /* eg, "cpu4" */
kcpuset_t *cpu_kcpuset; /* kcpuset_t of this cpu only */
+ cpuid_t cpu_package_id;
+ cpuid_t cpu_core_id;
+ cpuid_t cpu_smt_id;
+ struct lwp * volatile cpu_pcu_curlwp[PCU_UNIT_COUNT];
};
#define ci_schedstate ci_data.cpu_schedstate
Index: src/sys/sys/lwp.h
diff -u src/sys/sys/lwp.h:1.191 src/sys/sys/lwp.h:1.192
--- src/sys/sys/lwp.h:1.191 Sat Nov 30 17:45:54 2019
+++ src/sys/sys/lwp.h Sun Dec 1 15:34:47 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: lwp.h,v 1.191 2019/11/30 17:45:54 ad Exp $ */
+/* $NetBSD: lwp.h,v 1.192 2019/12/01 15:34:47 ad Exp $ */
/*
* Copyright (c) 2001, 2006, 2007, 2008, 2009, 2010, 2019
@@ -500,7 +500,7 @@ static __inline bool
CURCPU_IDLE_P(void)
{
struct cpu_info *ci = curcpu();
- return ci->ci_data.cpu_onproc == ci->ci_data.cpu_idlelwp;
+ return ci->ci_onproc == ci->ci_data.cpu_idlelwp;
}
/*
