Module Name: src
Committed By: rmind
Date: Sun Aug 7 13:33:03 UTC 2011
Modified Files:
src/common/lib/libc/sys: cpuset.c
src/distrib/sets/lists/comp: mi
src/sys/conf: files
src/sys/kern: init_main.c kern_cpu.c kern_runq.c subr_pserialize.c
sys_pset.c sys_sched.c
src/sys/sys: Makefile cpu.h lwp.h sched.h
Added Files:
src/sys/kern: subr_kcpuset.c
src/sys/sys: kcpuset.h
Log Message:
Add kcpuset(9) - a reworked dynamic CPU set implementation for kernel.
Suitable for use during the early boot. MD and other implementations
should be replaced with this interface.
Discussed on: tech-kern@
To generate a diff of this commit:
cvs rdiff -u -r1.16 -r1.17 src/common/lib/libc/sys/cpuset.c
cvs rdiff -u -r1.1657 -r1.1658 src/distrib/sets/lists/comp/mi
cvs rdiff -u -r1.1022 -r1.1023 src/sys/conf/files
cvs rdiff -u -r1.434 -r1.435 src/sys/kern/init_main.c
cvs rdiff -u -r1.47 -r1.48 src/sys/kern/kern_cpu.c
cvs rdiff -u -r1.30 -r1.31 src/sys/kern/kern_runq.c
cvs rdiff -u -r0 -r1.1 src/sys/kern/subr_kcpuset.c
cvs rdiff -u -r1.2 -r1.3 src/sys/kern/subr_pserialize.c
cvs rdiff -u -r1.15 -r1.16 src/sys/kern/sys_pset.c
cvs rdiff -u -r1.35 -r1.36 src/sys/kern/sys_sched.c
cvs rdiff -u -r1.136 -r1.137 src/sys/sys/Makefile
cvs rdiff -u -r1.32 -r1.33 src/sys/sys/cpu.h
cvs rdiff -u -r0 -r1.1 src/sys/sys/kcpuset.h
cvs rdiff -u -r1.152 -r1.153 src/sys/sys/lwp.h
cvs rdiff -u -r1.72 -r1.73 src/sys/sys/sched.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/common/lib/libc/sys/cpuset.c
diff -u src/common/lib/libc/sys/cpuset.c:1.16 src/common/lib/libc/sys/cpuset.c:1.17
--- src/common/lib/libc/sys/cpuset.c:1.16 Tue Sep 21 02:03:29 2010
+++ src/common/lib/libc/sys/cpuset.c Sun Aug 7 13:33:02 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: cpuset.c,v 1.16 2010/09/21 02:03:29 rmind Exp $ */
+/* $NetBSD: cpuset.c,v 1.17 2011/08/07 13:33:02 rmind Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
#ifndef _STANDALONE
#include <sys/cdefs.h>
#if defined(LIBC_SCCS) && !defined(lint)
-__RCSID("$NetBSD: cpuset.c,v 1.16 2010/09/21 02:03:29 rmind Exp $");
+__RCSID("$NetBSD: cpuset.c,v 1.17 2011/08/07 13:33:02 rmind Exp $");
#endif /* LIBC_SCCS and not lint */
#include <sys/param.h>
@@ -64,24 +64,11 @@
uint32_t bits[0];
};
-#ifdef _KERNEL
-struct _kcpuset {
- unsigned int nused;
- struct _kcpuset *next;
- uint32_t bits[0];
-};
-#define KCPUSET_SIZE(n) (sizeof( \
- struct { \
- unsigned int nused; \
- struct _kcpuset *next; \
- uint32_t bits[0]; \
- }) + sizeof(uint32_t) * (n))
-#endif
+#ifndef _KERNEL
static size_t cpuset_size = 0;
static size_t cpuset_nentries = 0;
-#ifndef _KERNEL
size_t
/*ARGSUSED*/
_cpuset_size(const cpuset_t *c)
@@ -161,150 +148,5 @@
free(c);
}
-#else
-
-kcpuset_t *
-kcpuset_create(void)
-{
- kcpuset_t *c;
-
- if (cpuset_size == 0) {
- cpuset_nentries = CPUSET_NENTRIES(MAXCPUS);
- cpuset_size = KCPUSET_SIZE(cpuset_nentries);
- }
- c = kmem_zalloc(cpuset_size, KM_SLEEP);
- c->next = NULL;
- c->nused = 1;
- return c;
-}
-
-void
-kcpuset_destroy(kcpuset_t *c)
-{
- kcpuset_t *nc;
-
- while (c) {
- KASSERT(c->nused == 0);
- nc = c->next;
- kmem_free(c, cpuset_size);
- c = nc;
- }
-}
-
-void
-kcpuset_copy(kcpuset_t *d, const kcpuset_t *s)
-{
-
- KASSERT(d->nused == 1);
- memcpy(d->bits, s->bits, cpuset_nentries * sizeof(s->bits[0]));
-}
-
-void
-kcpuset_use(kcpuset_t *c)
-{
-
- atomic_inc_uint(&c->nused);
-}
-
-void
-kcpuset_unuse(kcpuset_t *c, kcpuset_t **lst)
-{
-
- if (atomic_dec_uint_nv(&c->nused) != 0)
- return;
- KASSERT(c->nused == 0);
- KASSERT(c->next == NULL);
- if (lst == NULL) {
- kcpuset_destroy(c);
- return;
- }
- c->next = *lst;
- *lst = c;
-}
-
-int
-kcpuset_copyin(const cpuset_t *u, kcpuset_t *k, size_t len)
-{
-
- KASSERT(k->nused > 0);
- KASSERT(k->next == NULL);
- if (len != CPUSET_SIZE(cpuset_nentries))
- return EINVAL;
- return copyin(u->bits, k->bits, cpuset_nentries * sizeof(k->bits[0]));
-}
-
-int
-kcpuset_copyout(const kcpuset_t *k, cpuset_t *u, size_t len)
-{
-
- KASSERT(k->nused > 0);
- KASSERT(k->next == NULL);
- if (len != CPUSET_SIZE(cpuset_nentries))
- return EINVAL;
- return copyout(k->bits, u->bits, cpuset_nentries * sizeof(u->bits[0]));
-}
-
-void
-kcpuset_zero(kcpuset_t *c)
-{
-
- KASSERT(c->nused > 0);
- KASSERT(c->next == NULL);
- memset(c->bits, 0, cpuset_nentries * sizeof(c->bits[0]));
-}
-
-void
-kcpuset_fill(kcpuset_t *c)
-{
-
- KASSERT(c->nused > 0);
- KASSERT(c->next == NULL);
- memset(c->bits, ~0, cpuset_nentries * sizeof(c->bits[0]));
-}
-
-void
-kcpuset_set(cpuid_t i, kcpuset_t *c)
-{
- const unsigned long j = i >> CPUSET_SHIFT;
-
- KASSERT(c->next == NULL);
- KASSERT(j < cpuset_nentries);
- c->bits[j] |= 1 << (i & CPUSET_MASK);
-}
-
-int
-kcpuset_isset(cpuid_t i, const kcpuset_t *c)
-{
- const unsigned long j = i >> CPUSET_SHIFT;
-
- KASSERT(c != NULL);
- KASSERT(c->nused > 0);
- KASSERT(c->next == NULL);
- KASSERT(j < cpuset_nentries);
- return ((1 << (i & CPUSET_MASK)) & c->bits[j]) != 0;
-}
-
-bool
-kcpuset_iszero(const kcpuset_t *c)
-{
- unsigned long j;
-
- for (j = 0; j < cpuset_nentries; j++)
- if (c->bits[j] != 0)
- return false;
- return true;
-}
-
-bool
-kcpuset_match(const kcpuset_t *c1, const kcpuset_t *c2)
-{
- unsigned long j;
-
- for (j = 0; j < cpuset_nentries; j++)
- if ((c1->bits[j] & c2->bits[j]) != c2->bits[j])
- return false;
- return true;
-}
-
#endif
#endif
Index: src/distrib/sets/lists/comp/mi
diff -u src/distrib/sets/lists/comp/mi:1.1657 src/distrib/sets/lists/comp/mi:1.1658
--- src/distrib/sets/lists/comp/mi:1.1657 Sat Aug 6 17:17:39 2011
+++ src/distrib/sets/lists/comp/mi Sun Aug 7 13:33:02 2011
@@ -1,4 +1,4 @@
-# $NetBSD: mi,v 1.1657 2011/08/06 17:17:39 jruoho Exp $
+# $NetBSD: mi,v 1.1658 2011/08/07 13:33:02 rmind Exp $
#
# Note: don't delete entries from here - mark them as "obsolete" instead.
#
@@ -2385,6 +2385,7 @@
./usr/include/sys/ipc.h comp-c-include
./usr/include/sys/joystick.h comp-c-include
./usr/include/sys/kcore.h comp-c-include
+./usr/include/sys/kcpuset.h comp-c-include
./usr/include/sys/kernel.h comp-obsolete obsolete
./usr/include/sys/keylock.h comp-obsolete obsolete
./usr/include/sys/kgdb.h comp-c-include
Index: src/sys/conf/files
diff -u src/sys/conf/files:1.1022 src/sys/conf/files:1.1023
--- src/sys/conf/files:1.1022 Sat Jul 30 17:01:04 2011
+++ src/sys/conf/files Sun Aug 7 13:33:01 2011
@@ -1,4 +1,4 @@
-# $NetBSD: files,v 1.1022 2011/07/30 17:01:04 christos Exp $
+# $NetBSD: files,v 1.1023 2011/08/07 13:33:01 rmind Exp $
# @(#)files.newconf 7.5 (Berkeley) 5/10/93
version 20100430
@@ -1530,6 +1530,7 @@
file kern/subr_hash.c
file kern/subr_humanize.c
file kern/subr_iostat.c
+file kern/subr_kcpuset.c
file kern/subr_kmem.c
file kern/subr_kobj.c
file kern/subr_kobj_vfs.c
Index: src/sys/kern/init_main.c
diff -u src/sys/kern/init_main.c:1.434 src/sys/kern/init_main.c:1.435
--- src/sys/kern/init_main.c:1.434 Sat Jul 30 17:01:04 2011
+++ src/sys/kern/init_main.c Sun Aug 7 13:33:01 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: init_main.c,v 1.434 2011/07/30 17:01:04 christos Exp $ */
+/* $NetBSD: init_main.c,v 1.435 2011/08/07 13:33:01 rmind Exp $ */
/*-
* Copyright (c) 2008, 2009 The NetBSD Foundation, Inc.
@@ -97,7 +97,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.434 2011/07/30 17:01:04 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.435 2011/08/07 13:33:01 rmind Exp $");
#include "opt_ddb.h"
#include "opt_ipsec.h"
@@ -167,6 +167,7 @@
#include <sys/event.h>
#include <sys/lockf.h>
#include <sys/once.h>
+#include <sys/kcpuset.h>
#include <sys/ksyms.h>
#include <sys/uidinfo.h>
#include <sys/kprintf.h>
@@ -310,6 +311,7 @@
evcnt_init();
uvm_init();
+ kcpuset_sysinit();
prop_kern_init();
Index: src/sys/kern/kern_cpu.c
diff -u src/sys/kern/kern_cpu.c:1.47 src/sys/kern/kern_cpu.c:1.48
--- src/sys/kern/kern_cpu.c:1.47 Wed Jun 29 06:22:21 2011
+++ src/sys/kern/kern_cpu.c Sun Aug 7 13:33:01 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_cpu.c,v 1.47 2011/06/29 06:22:21 matt Exp $ */
+/* $NetBSD: kern_cpu.c,v 1.48 2011/08/07 13:33:01 rmind Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc.
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.47 2011/06/29 06:22:21 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.48 2011/08/07 13:33:01 rmind Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -107,6 +107,9 @@
int ncpu __read_mostly;
int ncpuonline __read_mostly;
bool mp_online __read_mostly;
+
+kcpuset_t * kcpuset_attached __read_mostly;
+
struct cpuqueue cpu_queue __cacheline_aligned
= CIRCLEQ_HEAD_INITIALIZER(cpu_queue);
@@ -131,8 +134,11 @@
if (__predict_false(cpu_infos == NULL)) {
cpu_infos =
kmem_zalloc(sizeof(cpu_infos[0]) * maxcpus, KM_SLEEP);
+ kcpuset_create(&kcpuset_attached);
+ kcpuset_zero(kcpuset_attached);
}
cpu_infos[cpu_index(ci)] = ci;
+ kcpuset_set(kcpuset_attached, ci->ci_index);
sched_cpuattach(ci);
@@ -315,7 +321,7 @@
for (CPU_INFO_FOREACH(cii, mci)) {
mspc = &mci->ci_schedstate;
if ((mspc->spc_flags & SPCF_OFFLINE) == 0 &&
- kcpuset_isset(cpu_index(mci), l->l_affinity))
+ kcpuset_isset(l->l_affinity, cpu_index(mci)))
break;
}
if (mci == NULL) {
Index: src/sys/kern/kern_runq.c
diff -u src/sys/kern/kern_runq.c:1.30 src/sys/kern/kern_runq.c:1.31
--- src/sys/kern/kern_runq.c:1.30 Wed Mar 3 00:47:30 2010
+++ src/sys/kern/kern_runq.c Sun Aug 7 13:33:01 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_runq.c,v 1.30 2010/03/03 00:47:30 yamt Exp $ */
+/* $NetBSD: kern_runq.c,v 1.31 2011/08/07 13:33:01 rmind Exp $ */
/*
* Copyright (c) 2007, 2008 Mindaugas Rasiukevicius <rmind at NetBSD org>
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_runq.c,v 1.30 2010/03/03 00:47:30 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_runq.c,v 1.31 2011/08/07 13:33:01 rmind Exp $");
#include <sys/param.h>
#include <sys/kernel.h>
@@ -352,7 +352,7 @@
/* Affinity bind */
if (__predict_false(l->l_flag & LW_AFFINITY))
- return kcpuset_isset(cpu_index(ci), l->l_affinity);
+ return kcpuset_isset(l->l_affinity, cpu_index(ci));
/* Processor-set */
return (spc->spc_psid == l->l_psid);
Index: src/sys/kern/subr_pserialize.c
diff -u src/sys/kern/subr_pserialize.c:1.2 src/sys/kern/subr_pserialize.c:1.3
--- src/sys/kern/subr_pserialize.c:1.2 Mon Aug 1 15:26:31 2011
+++ src/sys/kern/subr_pserialize.c Sun Aug 7 13:33:01 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: subr_pserialize.c,v 1.2 2011/08/01 15:26:31 he Exp $ */
+/* $NetBSD: subr_pserialize.c,v 1.3 2011/08/07 13:33:01 rmind Exp $ */
/*-
* Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
@@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.2 2011/08/01 15:26:31 he Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.3 2011/08/07 13:33:01 rmind Exp $");
#include <sys/param.h>
@@ -103,8 +103,8 @@
psz = kmem_zalloc(sizeof(struct pserialize), KM_SLEEP);
cv_init(&psz->psz_notifier, "psrlz");
- psz->psz_target = kcpuset_create();
- psz->psz_pass = kcpuset_create();
+ kcpuset_create(&psz->psz_target);
+ kcpuset_create(&psz->psz_pass);
psz->psz_owner = NULL;
return psz;
@@ -147,7 +147,6 @@
return;
}
KASSERT(psz->psz_owner == NULL);
- KASSERT(kcpuset_iszero(psz->psz_target));
KASSERT(ncpu > 0);
/*
@@ -157,7 +156,7 @@
* other processors.
*/
psz->psz_owner = curlwp;
- kcpuset_fill(psz->psz_target);
+ kcpuset_copy(psz->psz_target, kcpuset_attached);
kcpuset_zero(psz->psz_pass);
mutex_spin_enter(&psz_lock);
@@ -234,7 +233,7 @@
for (psz = TAILQ_FIRST(&psz_queue1); psz != NULL; psz = next) {
next = TAILQ_NEXT(psz, psz_chain);
if (!kcpuset_match(psz->psz_pass, psz->psz_target)) {
- kcpuset_set(cid, psz->psz_pass);
+ kcpuset_set(psz->psz_pass, cid);
continue;
}
kcpuset_zero(psz->psz_pass);
@@ -248,7 +247,7 @@
for (psz = TAILQ_FIRST(&psz_queue0); psz != NULL; psz = next) {
next = TAILQ_NEXT(psz, psz_chain);
if (!kcpuset_match(psz->psz_pass, psz->psz_target)) {
- kcpuset_set(cid, psz->psz_pass);
+ kcpuset_set(psz->psz_pass, cid);
continue;
}
kcpuset_zero(psz->psz_pass);
Index: src/sys/kern/sys_pset.c
diff -u src/sys/kern/sys_pset.c:1.15 src/sys/kern/sys_pset.c:1.16
--- src/sys/kern/sys_pset.c:1.15 Thu Jul 1 02:38:31 2010
+++ src/sys/kern/sys_pset.c Sun Aug 7 13:33:01 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: sys_pset.c,v 1.15 2010/07/01 02:38:31 rmind Exp $ */
+/* $NetBSD: sys_pset.c,v 1.16 2011/08/07 13:33:01 rmind Exp $ */
/*
* Copyright (c) 2008, Mindaugas Rasiukevicius <rmind at NetBSD org>
@@ -36,7 +36,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sys_pset.c,v 1.15 2010/07/01 02:38:31 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sys_pset.c,v 1.16 2011/08/07 13:33:01 rmind Exp $");
#include <sys/param.h>
@@ -373,7 +373,7 @@
lwp_unlock(t);
continue;
}
- if (kcpuset_isset(cpu_index(ci), t->l_affinity)) {
+ if (kcpuset_isset(t->l_affinity, cpu_index(ci))) {
lwp_unlock(t);
mutex_exit(proc_lock);
mutex_exit(&cpu_lock);
Index: src/sys/kern/sys_sched.c
diff -u src/sys/kern/sys_sched.c:1.35 src/sys/kern/sys_sched.c:1.36
--- src/sys/kern/sys_sched.c:1.35 Thu Jul 1 02:38:31 2010
+++ src/sys/kern/sys_sched.c Sun Aug 7 13:33:01 2011
@@ -1,7 +1,7 @@
-/* $NetBSD: sys_sched.c,v 1.35 2010/07/01 02:38:31 rmind Exp $ */
+/* $NetBSD: sys_sched.c,v 1.36 2011/08/07 13:33:01 rmind Exp $ */
/*
- * Copyright (c) 2008, Mindaugas Rasiukevicius <rmind at NetBSD org>
+ * Copyright (c) 2008, 2011 Mindaugas Rasiukevicius <rmind at NetBSD org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -42,7 +42,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sys_sched.c,v 1.35 2010/07/01 02:38:31 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sys_sched.c,v 1.36 2011/08/07 13:33:01 rmind Exp $");
#include <sys/param.h>
@@ -298,12 +298,16 @@
static int
genkcpuset(kcpuset_t **dset, const cpuset_t *sset, size_t size)
{
+ kcpuset_t *kset;
int error;
- *dset = kcpuset_create();
- error = kcpuset_copyin(sset, *dset, size);
- if (error != 0)
- kcpuset_unuse(*dset, NULL);
+ kcpuset_create(&kset);
+ error = kcpuset_copyin(sset, kset, size);
+ if (error) {
+ kcpuset_unuse(kset, NULL);
+ } else {
+ *dset = kset;
+ }
return error;
}
@@ -320,7 +324,7 @@
syscallarg(size_t) size;
syscallarg(const cpuset_t *) cpuset;
} */
- kcpuset_t *cpuset, *cpulst = NULL;
+ kcpuset_t *kcset, *kcpulst = NULL;
struct cpu_info *ici, *ci;
struct proc *p;
struct lwp *t;
@@ -330,7 +334,7 @@
u_int lcnt;
int error;
- error = genkcpuset(&cpuset, SCARG(uap, cpuset), SCARG(uap, size));
+ error = genkcpuset(&kcset, SCARG(uap, cpuset), SCARG(uap, size));
if (error)
return error;
@@ -349,7 +353,7 @@
for (CPU_INFO_FOREACH(cii, ici)) {
struct schedstate_percpu *ispc;
- if (kcpuset_isset(cpu_index(ici), cpuset) == 0)
+ if (kcpuset_isset(kcset, cpu_index(ici)) == 0)
continue;
ispc = &ici->ci_schedstate;
@@ -375,8 +379,8 @@
goto out;
}
/* Empty set */
- kcpuset_unuse(cpuset, &cpulst);
- cpuset = NULL;
+ kcpuset_unuse(kcset, &kcpulst);
+ kcset = NULL;
}
if (SCARG(uap, pid) != 0) {
@@ -433,33 +437,42 @@
lwp_unlock(t);
continue;
}
- if (cpuset) {
+ if (kcset) {
/* Set the affinity flag and new CPU set */
t->l_flag |= LW_AFFINITY;
- kcpuset_use(cpuset);
+ kcpuset_use(kcset);
if (t->l_affinity != NULL)
- kcpuset_unuse(t->l_affinity, &cpulst);
- t->l_affinity = cpuset;
+ kcpuset_unuse(t->l_affinity, &kcpulst);
+ t->l_affinity = kcset;
/* Migrate to another CPU, unlocks LWP */
lwp_migrate(t, ci);
} else {
/* Unset the affinity flag */
t->l_flag &= ~LW_AFFINITY;
if (t->l_affinity != NULL)
- kcpuset_unuse(t->l_affinity, &cpulst);
+ kcpuset_unuse(t->l_affinity, &kcpulst);
t->l_affinity = NULL;
lwp_unlock(t);
}
lcnt++;
}
mutex_exit(p->p_lock);
- if (lcnt == 0)
+ if (lcnt == 0) {
error = ESRCH;
+ }
out:
mutex_exit(&cpu_lock);
- if (cpuset != NULL)
- kcpuset_unuse(cpuset, &cpulst);
- kcpuset_destroy(cpulst);
+
+ /*
+ * Drop the initial reference (LWPs, if any, have the ownership now),
+ * and destroy whatever is in the G/C list, if filled.
+ */
+ if (kcset) {
+ kcpuset_unuse(kcset, &kcpulst);
+ }
+ if (kcpulst) {
+ kcpuset_destroy(kcpulst);
+ }
return error;
}
@@ -477,10 +490,10 @@
syscallarg(cpuset_t *) cpuset;
} */
struct lwp *t;
- kcpuset_t *cpuset;
+ kcpuset_t *kcset;
int error;
- error = genkcpuset(&cpuset, SCARG(uap, cpuset), SCARG(uap, size));
+ error = genkcpuset(&kcset, SCARG(uap, cpuset), SCARG(uap, size));
if (error)
return error;
@@ -500,15 +513,16 @@
lwp_lock(t);
if (t->l_flag & LW_AFFINITY) {
KASSERT(t->l_affinity != NULL);
- kcpuset_copy(cpuset, t->l_affinity);
- } else
- kcpuset_zero(cpuset);
+ kcpuset_copy(kcset, t->l_affinity);
+ } else {
+ kcpuset_zero(kcset);
+ }
lwp_unlock(t);
mutex_exit(t->l_proc->p_lock);
- error = kcpuset_copyout(cpuset, SCARG(uap, cpuset), SCARG(uap, size));
+ error = kcpuset_copyout(kcset, SCARG(uap, cpuset), SCARG(uap, size));
out:
- kcpuset_unuse(cpuset, NULL);
+ kcpuset_unuse(kcset, NULL);
return error;
}
Index: src/sys/sys/Makefile
diff -u src/sys/sys/Makefile:1.136 src/sys/sys/Makefile:1.137
--- src/sys/sys/Makefile:1.136 Sun Jul 17 20:54:54 2011
+++ src/sys/sys/Makefile Sun Aug 7 13:33:02 2011
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.136 2011/07/17 20:54:54 joerg Exp $
+# $NetBSD: Makefile,v 1.137 2011/08/07 13:33:02 rmind Exp $
.include <bsd.sys.mk>
@@ -19,7 +19,7 @@
flashio.h float_ieee754.h fstypes.h gcq.h gmon.h gpio.h hash.h \
ieee754.h inttypes.h ioccom.h ioctl.h ioctl_compat.h iostat.h ipc.h \
joystick.h \
- kcore.h kgdb.h kmem.h ksem.h ksyms.h ktrace.h \
+ kcore.h kcpuset.h kgdb.h kmem.h ksem.h ksyms.h ktrace.h \
localedef.h lock.h lockf.h lwp.h lwpctl.h \
malloc.h mallocvar.h mbuf.h md4.h md5.h midiio.h \
mman.h module.h mount.h mqueue.h msg.h msgbuf.h mtio.h mutex.h \
Index: src/sys/sys/cpu.h
diff -u src/sys/sys/cpu.h:1.32 src/sys/sys/cpu.h:1.33
--- src/sys/sys/cpu.h:1.32 Mon Dec 20 04:27:35 2010
+++ src/sys/sys/cpu.h Sun Aug 7 13:33:02 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.32 2010/12/20 04:27:35 christos Exp $ */
+/* $NetBSD: cpu.h,v 1.33 2011/08/07 13:33:02 rmind Exp $ */
/*-
* Copyright (c) 2007 YAMAMOTO Takashi,
@@ -89,6 +89,7 @@
extern kmutex_t cpu_lock;
extern u_int maxcpus;
extern struct cpuqueue cpu_queue;
+extern kcpuset_t *kcpuset_attached;
static inline u_int
cpu_index(struct cpu_info *ci)
Index: src/sys/sys/lwp.h
diff -u src/sys/sys/lwp.h:1.152 src/sys/sys/lwp.h:1.153
--- src/sys/sys/lwp.h:1.152 Thu May 19 03:07:29 2011
+++ src/sys/sys/lwp.h Sun Aug 7 13:33:02 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: lwp.h,v 1.152 2011/05/19 03:07:29 rmind Exp $ */
+/* $NetBSD: lwp.h,v 1.153 2011/08/07 13:33:02 rmind Exp $ */
/*-
* Copyright (c) 2001, 2006, 2007, 2008, 2009, 2010
@@ -37,6 +37,7 @@
#include <sys/time.h>
#include <sys/queue.h>
#include <sys/callout.h>
+#include <sys/kcpuset.h>
#include <sys/mutex.h>
#include <sys/condvar.h>
#include <sys/signalvar.h>
Index: src/sys/sys/sched.h
diff -u src/sys/sys/sched.h:1.72 src/sys/sys/sched.h:1.73
--- src/sys/sys/sched.h:1.72 Fri Apr 16 03:21:49 2010
+++ src/sys/sys/sched.h Sun Aug 7 13:33:02 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: sched.h,v 1.72 2010/04/16 03:21:49 rmind Exp $ */
+/* $NetBSD: sched.h,v 1.73 2011/08/07 13:33:02 rmind Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2007, 2008 The NetBSD Foundation, Inc.
@@ -96,25 +96,8 @@
* Interface of CPU-sets.
*/
typedef struct _cpuset cpuset_t;
-typedef struct _kcpuset kcpuset_t; /* XXX: lwp.h included from userland */
-#ifdef _KERNEL
-
-kcpuset_t *kcpuset_create(void);
-void kcpuset_destroy(kcpuset_t *);
-void kcpuset_copy(kcpuset_t *, const kcpuset_t *);
-void kcpuset_use(kcpuset_t *);
-void kcpuset_unuse(kcpuset_t *, kcpuset_t **);
-int kcpuset_copyin(const cpuset_t *, kcpuset_t *, size_t);
-int kcpuset_copyout(const kcpuset_t *, cpuset_t *, size_t);
-void kcpuset_zero(kcpuset_t *);
-void kcpuset_fill(kcpuset_t *);
-void kcpuset_set(cpuid_t, kcpuset_t *);
-int kcpuset_isset(cpuid_t, const kcpuset_t *);
-bool kcpuset_iszero(const kcpuset_t *);
-bool kcpuset_match(const kcpuset_t *, const kcpuset_t *);
-
-#else
+#ifndef _KERNEL
#define cpuset_create() _cpuset_create()
#define cpuset_destroy(c) _cpuset_destroy(c)
Added files:
Index: src/sys/kern/subr_kcpuset.c
diff -u /dev/null src/sys/kern/subr_kcpuset.c:1.1
--- /dev/null Sun Aug 7 13:33:04 2011
+++ src/sys/kern/subr_kcpuset.c Sun Aug 7 13:33:01 2011
@@ -0,0 +1,386 @@
+/* $NetBSD: subr_kcpuset.c,v 1.1 2011/08/07 13:33:01 rmind Exp $ */
+
+/*-
+ * Copyright (c) 2011 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Mindaugas Rasiukevicius.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Kernel CPU set implementation.
+ *
+ * Interface can be used by kernel subsystems as a unified dynamic CPU
+ * bitset implementation handling many CPUs. Facility also supports early
+ * use by MD code on boot, as it fixups bitsets on further boot.
+ *
+ * TODO:
+ * - Handle "reverse" bitset on fixup/grow.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: subr_kcpuset.c,v 1.1 2011/08/07 13:33:01 rmind Exp $");
+
+#include <sys/param.h>
+#include <sys/types.h>
+
+#include <sys/atomic.h>
+#include <sys/sched.h>
+#include <sys/kcpuset.h>
+#include <sys/pool.h>
+
+/* Number of CPUs to support. */
+#define KC_MAXCPUS roundup2(MAXCPUS, 32)
+
+/*
+ * Structure of dynamic CPU set in the kernel.
+ */
+struct kcpuset {
+ uint32_t bits[0];
+};
+
+typedef struct kcpuset_impl {
+ /* Reference count. */
+ u_int kc_refcnt;
+ /* Next to free, if non-NULL (used when multiple references). */
+ struct kcpuset * kc_next;
+ /* Actual variable-sized field of bits. */
+ struct kcpuset kc_field;
+} kcpuset_impl_t;
+
+#define KC_BITS_OFF (offsetof(struct kcpuset_impl, kc_field))
+#define KC_GETSTRUCT(b) ((kcpuset_impl_t *)((char *)(b) - KC_BITS_OFF))
+
+/* Sizes of a single bitset. */
+#define KC_SHIFT 5
+#define KC_MASK 31
+
+/* An array of noted early kcpuset creations and data. */
+#define KC_SAVE_NITEMS 8
+
+/* Structures for early boot mechanism (must be statically initialised). */
+static kcpuset_t ** kc_noted_early[KC_SAVE_NITEMS];
+static uint32_t kc_bits_early[KC_SAVE_NITEMS];
+static int kc_last_idx = 0;
+static bool kc_initialised = false;
+
+#define KC_BITSIZE_EARLY sizeof(kc_bits_early[0])
+#define KC_NFIELDS_EARLY (KC_BITSIZE_EARLY >> KC_SHIFT)
+
+/*
+ * The size of whole bitset fields and amount of fields.
+ * The whole size must statically initialise for early case.
+ */
+static size_t kc_bitsize __read_mostly = KC_BITSIZE_EARLY;
+static size_t kc_nfields __read_mostly = KC_NFIELDS_EARLY;
+
+static pool_cache_t kc_cache __read_mostly;
+
+static kcpuset_t * kcpuset_create_raw(void);
+
+/*
+ * kcpuset_sysinit: initialize the subsystem, transfer early boot cases
+ * to dynamically allocated sets.
+ */
+void
+kcpuset_sysinit(void)
+{
+ kcpuset_t *kc_dynamic[KC_SAVE_NITEMS], *kcp;
+ int i, s;
+
+ /* Set a kcpuset_t sizes. */
+ kc_nfields = (KC_MAXCPUS >> KC_SHIFT);
+ kc_bitsize = sizeof(uint32_t) * kc_nfields;
+
+ kc_cache = pool_cache_init(sizeof(kcpuset_impl_t) + kc_bitsize,
+ coherency_unit, 0, 0, "kcpuset", NULL, IPL_NONE, NULL, NULL, NULL);
+
+ /* First, pre-allocate kcpuset entries. */
+ for (i = 0; i < kc_last_idx; i++) {
+ kcp = kcpuset_create_raw();
+ kcpuset_zero(kcp);
+ kc_dynamic[i] = kcp;
+ }
+
+ /*
+ * Prepare to convert all early noted kcpuset uses to dynamic sets.
+ * All processors, except the one we are currently running (primary),
+ * must not be spinned yet. Since MD facilities can use kcpuset,
+ * raise the IPL to high.
+ */
+ KASSERT(mp_online == false);
+
+ s = splhigh();
+ for (i = 0; i < kc_last_idx; i++) {
+ /*
+ * Transfer the bits from early static storage to the kcpuset.
+ */
+ KASSERT(kc_bitsize >= KC_BITSIZE_EARLY);
+ memcpy(kc_dynamic[i], &kc_bits_early[i], KC_BITSIZE_EARLY);
+
+ /*
+ * Store the new pointer, pointing to the allocated kcpuset.
+ * Note: we are not in an interrupt context and it is the only
+ * CPU running - thus store is safe (e.g. no need for pointer
+ * variable to be volatile).
+ */
+ *kc_noted_early[i] = kc_dynamic[i];
+ }
+ kc_initialised = true;
+ kc_last_idx = 0;
+ splx(s);
+}
+
+/*
+ * kcpuset_early_ptr: note an early boot use by saving the pointer and
+ * returning a pointer to a static, temporary bit field.
+ */
+static kcpuset_t *
+kcpuset_early_ptr(kcpuset_t **kcptr)
+{
+ kcpuset_t *kcp;
+ int s;
+
+ s = splhigh();
+ if (kc_last_idx < KC_SAVE_NITEMS) {
+ /*
+ * Save the pointer, return pointer to static early field.
+ * Need to zero it out.
+ */
+ kc_noted_early[kc_last_idx++] = kcptr;
+ kcp = (kcpuset_t *)&kc_bits_early[kc_last_idx];
+ memset(kcp, 0, KC_BITSIZE_EARLY);
+ KASSERT(kc_bitsize == KC_BITSIZE_EARLY);
+ } else {
+ panic("kcpuset(9): all early-use entries exhausted; "
+ "increase KC_SAVE_NITEMS\n");
+ }
+ splx(s);
+
+ return kcp;
+}
+
+/*
+ * Routines to create or destroy the CPU set.
+ * Early boot case is handled.
+ */
+
+static kcpuset_t *
+kcpuset_create_raw(void)
+{
+ kcpuset_impl_t *kc;
+
+ kc = pool_cache_get(kc_cache, PR_WAITOK);
+ kc->kc_refcnt = 1;
+ kc->kc_next = NULL;
+
+ /* Note: return pointer to the actual field of bits. */
+ KASSERT((uint8_t *)kc + KC_BITS_OFF == (uint8_t *)&kc->kc_field);
+ return &kc->kc_field;
+}
+
+void
+kcpuset_create(kcpuset_t **retkcp)
+{
+
+ if (__predict_false(!kc_initialised)) {
+ /* Early boot use - special case. */
+ *retkcp = kcpuset_early_ptr(retkcp);
+ return;
+ }
+ *retkcp = kcpuset_create_raw();
+}
+
+void
+kcpuset_destroy(kcpuset_t *kcp)
+{
+ kcpuset_impl_t *kc, *nkc;
+
+ KASSERT(kc_initialised);
+ KASSERT(kcp != NULL);
+
+ kc = KC_GETSTRUCT(kcp);
+ do {
+ nkc = KC_GETSTRUCT(kc->kc_next);
+ pool_cache_put(kc_cache, kc);
+ kc = nkc;
+ } while (kc);
+}
+
+/*
+ * Routines to copy or reference/unreference the CPU set.
+ * Note: early boot case is not supported by these routines.
+ */
+
+void
+kcpuset_copy(kcpuset_t *dkcp, kcpuset_t *skcp)
+{
+
+ KASSERT(kc_initialised);
+ KASSERT(KC_GETSTRUCT(dkcp)->kc_refcnt == 1);
+ memcpy(dkcp, skcp, kc_bitsize);
+}
+
+void
+kcpuset_use(kcpuset_t *kcp)
+{
+ kcpuset_impl_t *kc = KC_GETSTRUCT(kcp);
+
+ KASSERT(kc_initialised);
+ atomic_inc_uint(&kc->kc_refcnt);
+}
+
+void
+kcpuset_unuse(kcpuset_t *kcp, kcpuset_t **lst)
+{
+ kcpuset_impl_t *kc = KC_GETSTRUCT(kcp);
+
+ KASSERT(kc_initialised);
+ KASSERT(kc->kc_refcnt > 0);
+
+ if (atomic_dec_uint_nv(&kc->kc_refcnt) != 0) {
+ return;
+ }
+ KASSERT(kc->kc_next == NULL);
+ if (lst == NULL) {
+ kcpuset_destroy(kcp);
+ return;
+ }
+ kc->kc_next = *lst;
+ *lst = kcp;
+}
+
+/*
+ * Routines to transfer the CPU set from / to userspace.
+ * Note: early boot case is not supported by these routines.
+ */
+
+int
+kcpuset_copyin(const cpuset_t *ucp, kcpuset_t *kcp, size_t len)
+{
+ kcpuset_impl_t *kc = KC_GETSTRUCT(kcp);
+
+ KASSERT(kc_initialised);
+ KASSERT(kc->kc_refcnt > 0);
+ KASSERT(kc->kc_next == NULL);
+ (void)kc;
+
+ if (len != kc_bitsize) { /* XXX */
+ return EINVAL;
+ }
+ return copyin(ucp, kcp, kc_bitsize);
+}
+
+int
+kcpuset_copyout(kcpuset_t *kcp, cpuset_t *ucp, size_t len)
+{
+ kcpuset_impl_t *kc = KC_GETSTRUCT(kcp);
+
+ KASSERT(kc_initialised);
+ KASSERT(kc->kc_refcnt > 0);
+ KASSERT(kc->kc_next == NULL);
+ (void)kc;
+
+ if (len != kc_bitsize) { /* XXX */
+ return EINVAL;
+ }
+ return copyout(kcp, ucp, kc_bitsize);
+}
+
+/*
+ * Routines to change bit field - zero, fill, set, unset, etc.
+ */
+
+void
+kcpuset_zero(kcpuset_t *kcp)
+{
+
+ KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0);
+ KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL);
+ memset(kcp, 0, kc_bitsize);
+}
+
+void
+kcpuset_fill(kcpuset_t *kcp)
+{
+
+ KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0);
+ KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL);
+ memset(kcp, ~0, kc_bitsize);
+}
+
+void
+kcpuset_set(kcpuset_t *kcp, cpuid_t i)
+{
+ const size_t j = i >> KC_SHIFT;
+
+ KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL);
+ KASSERT(j < kc_nfields);
+
+ kcp->bits[j] |= 1 << (i & KC_MASK);
+}
+
+void
+kcpuset_clear(kcpuset_t *kcp, cpuid_t i)
+{
+ const size_t j = i >> KC_SHIFT;
+
+ KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL);
+ KASSERT(j < kc_nfields);
+
+ kcp->bits[j] &= ~(1 << (i & KC_MASK));
+}
+
+int
+kcpuset_isset(kcpuset_t *kcp, cpuid_t i)
+{
+ const size_t j = i >> KC_SHIFT;
+
+ KASSERT(kcp != NULL);
+ KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0);
+ KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL);
+ KASSERT(j < kc_nfields);
+
+ return ((1 << (i & KC_MASK)) & kcp->bits[j]) != 0;
+}
+
+bool
+kcpuset_iszero(kcpuset_t *kcp)
+{
+
+ for (size_t j = 0; j < kc_nfields; j++) {
+ if (kcp->bits[j] != 0) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool
+kcpuset_match(const kcpuset_t *kcp1, const kcpuset_t *kcp2)
+{
+
+ return memcmp(kcp1, kcp2, kc_bitsize) == 0;
+}
Index: src/sys/sys/kcpuset.h
diff -u /dev/null src/sys/sys/kcpuset.h:1.1
--- /dev/null Sun Aug 7 13:33:04 2011
+++ src/sys/sys/kcpuset.h Sun Aug 7 13:33:02 2011
@@ -0,0 +1,63 @@
+/* $NetBSD: kcpuset.h,v 1.1 2011/08/07 13:33:02 rmind Exp $ */
+
+/*-
+ * Copyright (c) 2008, 2011 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Christos Zoulas and Mindaugas Rasiukevicius.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SYS_KCPUSET_H_
+#define _SYS_KCPUSET_H_
+
+struct kcpuset;
+typedef struct kcpuset kcpuset_t;
+
+#ifdef _KERNEL
+
+void kcpuset_sysinit(void);
+
+void kcpuset_create(kcpuset_t **);
+void kcpuset_destroy(kcpuset_t *);
+void kcpuset_copy(kcpuset_t *, kcpuset_t *);
+
+void kcpuset_use(kcpuset_t *);
+void kcpuset_unuse(kcpuset_t *, kcpuset_t **);
+
+int kcpuset_copyin(const cpuset_t *, kcpuset_t *, size_t);
+int kcpuset_copyout(kcpuset_t *, cpuset_t *, size_t);
+
+void kcpuset_zero(kcpuset_t *);
+void kcpuset_fill(kcpuset_t *);
+void kcpuset_set(kcpuset_t *, cpuid_t);
+void kcpuset_clear(kcpuset_t *, cpuid_t);
+
+int kcpuset_isset(kcpuset_t *, cpuid_t);
+bool kcpuset_iszero(kcpuset_t *);
+bool kcpuset_match(const kcpuset_t *, const kcpuset_t *);
+
+#endif
+
+#endif /* _SYS_KCPUSET_H_ */
