Module Name: src
Committed By: pgoyette
Date: Thu Dec 12 22:55:20 UTC 2019
Modified Files:
src/sys/kern: files.kern init_main.c kern_module.c
src/sys/rump/librump/rumpkern: Makefile.rumpkern
src/sys/sys: module_hook.h param.h
Added Files:
src/sys/kern: kern_module_hook.c
Log Message:
Eliminate per-hook duplication of common code as suggested by
(and with major contributions from) riastradh@
Welcome to 9.99.23
To generate a diff of this commit:
cvs rdiff -u -r1.38 -r1.39 src/sys/kern/files.kern
cvs rdiff -u -r1.508 -r1.509 src/sys/kern/init_main.c
cvs rdiff -u -r1.140 -r1.141 src/sys/kern/kern_module.c
cvs rdiff -u -r0 -r1.1 src/sys/kern/kern_module_hook.c
cvs rdiff -u -r1.177 -r1.178 src/sys/rump/librump/rumpkern/Makefile.rumpkern
cvs rdiff -u -r1.5 -r1.6 src/sys/sys/module_hook.h
cvs rdiff -u -r1.623 -r1.624 src/sys/sys/param.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/files.kern
diff -u src/sys/kern/files.kern:1.38 src/sys/kern/files.kern:1.39
--- src/sys/kern/files.kern:1.38 Wed Nov 20 19:37:53 2019
+++ src/sys/kern/files.kern Thu Dec 12 22:55:20 2019
@@ -1,4 +1,4 @@
-# $NetBSD: files.kern,v 1.38 2019/11/20 19:37:53 pgoyette Exp $
+# $NetBSD: files.kern,v 1.39 2019/12/12 22:55:20 pgoyette Exp $
#
# kernel sources
@@ -54,6 +54,7 @@ file kern/kern_lock.c kern
file kern/kern_lwp.c kern
file kern/kern_malloc.c kern
file kern/kern_module.c kern
+file kern/kern_module_hook.c kern
file kern/kern_module_vfs.c kern
file kern/kern_mutex.c kern
file kern/kern_mutex_obj.c kern
Index: src/sys/kern/init_main.c
diff -u src/sys/kern/init_main.c:1.508 src/sys/kern/init_main.c:1.509
--- src/sys/kern/init_main.c:1.508 Mon Dec 2 23:22:43 2019
+++ src/sys/kern/init_main.c Thu Dec 12 22:55:20 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: init_main.c,v 1.508 2019/12/02 23:22:43 ad Exp $ */
+/* $NetBSD: init_main.c,v 1.509 2019/12/12 22:55:20 pgoyette Exp $ */
/*-
* Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -97,7 +97,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.508 2019/12/02 23:22:43 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.509 2019/12/12 22:55:20 pgoyette Exp $");
#include "opt_ddb.h"
#include "opt_inet.h"
@@ -170,6 +170,7 @@ extern void *_binary_splash_image_end;
#include <sys/disk.h>
#include <sys/msgbuf.h>
#include <sys/module.h>
+#include <sys/module_hook.h>
#include <sys/event.h>
#include <sys/lockf.h>
#include <sys/once.h>
@@ -354,6 +355,7 @@ main(void)
/* Start module system. */
module_init();
+ module_hook_init();
/*
* Initialize the kernel authorization subsystem and start the
Index: src/sys/kern/kern_module.c
diff -u src/sys/kern/kern_module.c:1.140 src/sys/kern/kern_module.c:1.141
--- src/sys/kern/kern_module.c:1.140 Thu Dec 12 16:36:25 2019
+++ src/sys/kern/kern_module.c Thu Dec 12 22:55:20 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_module.c,v 1.140 2019/12/12 16:36:25 pgoyette Exp $ */
+/* $NetBSD: kern_module.c,v 1.141 2019/12/12 22:55:20 pgoyette Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_module.c,v 1.140 2019/12/12 16:36:25 pgoyette Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_module.c,v 1.141 2019/12/12 22:55:20 pgoyette Exp $");
#define _MODULE_INTERNAL
@@ -128,12 +128,6 @@ static int sysctl_module_autotime(SYSCTL
static void module_callback_load(struct module *);
static void module_callback_unload(struct module *);
-/* Locking/synchronization stuff for module hooks */
-
-kmutex_t module_hook_mtx;
-kcondvar_t module_hook_cv;
-pserialize_t module_hook_psz;
-
#define MODULE_CLASS_MATCH(mi, modclass) \
((modclass) == MODULE_CLASS_ANY || (modclass) == (mi)->mi_class)
@@ -451,10 +445,6 @@ module_init(void)
module_netbsd = module_newmodule(MODULE_SOURCE_KERNEL);
module_netbsd->mod_refcnt = 1;
module_netbsd->mod_info = &module_netbsd_modinfo;
-
- mutex_init(&module_hook_mtx, MUTEX_DEFAULT, IPL_NONE);
- cv_init(&module_hook_cv, "mod_hook");
- module_hook_psz = pserialize_create();
}
/*
Index: src/sys/rump/librump/rumpkern/Makefile.rumpkern
diff -u src/sys/rump/librump/rumpkern/Makefile.rumpkern:1.177 src/sys/rump/librump/rumpkern/Makefile.rumpkern:1.178
--- src/sys/rump/librump/rumpkern/Makefile.rumpkern:1.177 Sun Oct 13 07:28:14 2019
+++ src/sys/rump/librump/rumpkern/Makefile.rumpkern Thu Dec 12 22:55:20 2019
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile.rumpkern,v 1.177 2019/10/13 07:28:14 mrg Exp $
+# $NetBSD: Makefile.rumpkern,v 1.178 2019/12/12 22:55:20 pgoyette Exp $
#
IOCONFDIR:= ${.PARSEDIR}
@@ -75,6 +75,7 @@ SRCS+= init_sysctl_base.c \
kern_ksyms.c \
kern_malloc.c \
kern_module.c \
+ kern_module_hook.c \
kern_mutex_obj.c \
kern_ntptime.c \
kern_proc.c \
Index: src/sys/sys/module_hook.h
diff -u src/sys/sys/module_hook.h:1.5 src/sys/sys/module_hook.h:1.6
--- src/sys/sys/module_hook.h:1.5 Thu Dec 12 02:15:43 2019
+++ src/sys/sys/module_hook.h Thu Dec 12 22:55:20 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: module_hook.h,v 1.5 2019/12/12 02:15:43 pgoyette Exp $ */
+/* $NetBSD: module_hook.h,v 1.6 2019/12/12 22:55:20 pgoyette Exp $ */
/*-
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -33,11 +33,14 @@
#define _SYS_MODULE_HOOK_H
#include <sys/param.h> /* for COHERENCY_UNIT, for __cacheline_aligned */
-#include <sys/mutex.h>
#include <sys/localcount.h>
-#include <sys/condvar.h>
-#include <sys/pserialize.h>
-#include <sys/atomic.h>
+#include <sys/stdbool.h>
+
+void module_hook_init(void);
+void module_hook_set(bool *, struct localcount *);
+void module_hook_unset(bool *, struct localcount *);
+bool module_hook_tryenter(bool *, struct localcount *);
+void module_hook_exit(struct localcount *);
/*
* Macros for creating MP-safe vectored function calls, where
@@ -48,14 +51,10 @@
#define MODULE_HOOK(hook, type, args) \
extern struct hook ## _t { \
struct localcount lc; \
- bool hooked; \
type (*f)args; \
+ bool hooked; \
} hook __cacheline_aligned;
-extern kmutex_t module_hook_mtx;
-extern kcondvar_t module_hook_cv;
-extern pserialize_t module_hook_psz;
-
/*
* We use pserialize_perform() to issue a memory barrier on the current
* CPU and on all other CPUs so that all prior memory operations on the
@@ -69,85 +68,32 @@ extern pserialize_t module_hook_psz;
#define MODULE_HOOK_SET(hook, func) \
do { \
- \
- KASSERT(kernconfig_is_held()); \
- KASSERT(!hook.hooked); \
- \
- localcount_init(&hook.lc); \
- hook.f = func; \
- \
- /* Make sure it's initialized before anyone uses it */ \
- pserialize_perform(module_hook_psz); \
- \
- /* Let them use it */ \
- atomic_store_relaxed(&hook.hooked, true); \
+ (hook).f = func; \
+ module_hook_set(&(hook).hooked, &(hook).lc); \
} while /* CONSTCOND */ (0)
#define MODULE_HOOK_UNSET(hook) \
do { \
- \
- KASSERT(kernconfig_is_held()); \
- KASSERT(hook.hooked); \
- KASSERT(hook.f); \
- \
- /* Grab the mutex */ \
- mutex_enter(&module_hook_mtx); \
- \
- /* Prevent new localcount_acquire calls. */ \
- atomic_store_relaxed(&hook.hooked, false); \
- \
- /* \
- * Wait for localcount_acquire calls already under way \
- * to finish. \
- */ \
- pserialize_perform(module_hook_psz); \
- \
- /* Wait for existing localcount references to drain. */\
- localcount_drain(&hook.lc, &module_hook_cv, \
- &module_hook_mtx); \
- \
- /* Release the mutex and clean up all resources */ \
- mutex_exit(&module_hook_mtx); \
- localcount_fini(&hook.lc); \
+ KASSERT((hook).f); \
+ module_hook_unset(&(hook).hooked, &(hook).lc); \
+ (hook).f = NULL; /* paranoia */ \
} while /* CONSTCOND */ (0)
#define MODULE_HOOK_CALL(hook, args, default, retval) \
do { \
- bool __hooked; \
- int __hook_s; \
- \
- __hook_s = pserialize_read_enter(); \
- __hooked = atomic_load_relaxed(&hook.hooked); \
- if (__hooked) { \
- localcount_acquire(&hook.lc); \
- } \
- pserialize_read_exit(__hook_s); \
- \
- if (__hooked) { \
- retval = (*hook.f)args; \
- localcount_release(&hook.lc, &module_hook_cv, \
- &module_hook_mtx); \
+ if (module_hook_tryenter(&(hook).hooked, &(hook).lc)) { \
+ (retval) = (*(hook).f)args; \
+ module_hook_exit(&(hook).lc); \
} else { \
- retval = default; \
+ (retval) = (default); \
} \
} while /* CONSTCOND */ (0)
#define MODULE_HOOK_CALL_VOID(hook, args, default) \
do { \
- bool __hooked; \
- int __hook_s; \
- \
- __hook_s = pserialize_read_enter(); \
- __hooked = atomic_load_relaxed(&hook.hooked); \
- if (__hooked) { \
- localcount_acquire(&hook.lc); \
- } \
- pserialize_read_exit(__hook_s); \
- \
- if (__hooked) { \
- (*hook.f)args; \
- localcount_release(&hook.lc, &module_hook_cv, \
- &module_hook_mtx); \
+ if (module_hook_tryenter(&(hook).hooked, &(hook).lc)) { \
+ (*(hook).f)args; \
+ module_hook_exit(&(hook).lc); \
} else { \
default; \
} \
Index: src/sys/sys/param.h
diff -u src/sys/sys/param.h:1.623 src/sys/sys/param.h:1.624
--- src/sys/sys/param.h:1.623 Thu Dec 12 02:15:43 2019
+++ src/sys/sys/param.h Thu Dec 12 22:55:20 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: param.h,v 1.623 2019/12/12 02:15:43 pgoyette Exp $ */
+/* $NetBSD: param.h,v 1.624 2019/12/12 22:55:20 pgoyette Exp $ */
/*-
* Copyright (c) 1982, 1986, 1989, 1993
@@ -67,7 +67,7 @@
* 2.99.9 (299000900)
*/
-#define __NetBSD_Version__ 999002200 /* NetBSD 9.99.22 */
+#define __NetBSD_Version__ 999002300 /* NetBSD 9.99.23 */
#define __NetBSD_Prereq__(M,m,p) (((((M) * 100000000) + \
(m) * 1000000) + (p) * 100) <= __NetBSD_Version__)
Added files:
Index: src/sys/kern/kern_module_hook.c
diff -u /dev/null src/sys/kern/kern_module_hook.c:1.1
--- /dev/null Thu Dec 12 22:55:20 2019
+++ src/sys/kern/kern_module_hook.c Thu Dec 12 22:55:20 2019
@@ -0,0 +1,132 @@
+/* $NetBSD: kern_module_hook.c,v 1.1 2019/12/12 22:55:20 pgoyette Exp $ */
+
+/*-
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Kernel module support.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: kern_module_hook.c,v 1.1 2019/12/12 22:55:20 pgoyette Exp $");
+
+#include <sys/param.h>
+#include <sys/module_hook.h>
+#include <sys/mutex.h>
+#include <sys/condvar.h>
+#include <sys/pserialize.h>
+
+#include <uvm/uvm_extern.h>
+
+/* Locking/synchronization stuff for module hooks */
+
+static struct {
+ kmutex_t mtx;
+ kcondvar_t cv;
+ pserialize_t psz;
+} module_hook __cacheline_aligned;
+
+/*
+ * We use pserialize_perform() to issue a memory barrier on the current
+ * CPU and on all other CPUs so that all prior memory operations on the
+ * current CPU globally happen before all subsequent memory operations
+ * on the current CPU, as perceived by any other CPU.
+ *
+ * pserialize_perform() might be rather heavy-weight here, but it only
+ * happens during module loading, and it allows MODULE_HOOK_CALL() to
+ * work without any other memory barriers.
+ */
+
+void
+module_hook_set(bool *hooked, struct localcount *lc)
+{
+
+ KASSERT(kernconfig_is_held());
+ KASSERT(!*hooked);
+
+ localcount_init(lc);
+
+ /* Wait until setup has been witnessed by all CPUs. */
+ pserialize_perform(module_hook.psz);
+
+ /* Let others use it */
+ atomic_store_relaxed(hooked, true);
+}
+
+void
+module_hook_unset(bool *hooked, struct localcount *lc)
+{
+
+ KASSERT(kernconfig_is_held());
+ KASSERT(*hooked);
+
+ /* Get exclusive with pserialize and localcount. */
+ mutex_enter(&module_hook.mtx);
+
+ /* Prevent new calls to module_hook_tryenter(). */
+ atomic_store_relaxed(hooked, false);
+
+ /* Wait for existing calls to module_hook_tryenter(). */
+ pserialize_perform(module_hook.psz);
+
+ /* Wait for module_hook_exit. */
+ localcount_drain(lc, &module_hook.cv, &module_hook.mtx);
+
+ /* All done! */
+ mutex_exit(&module_hook.mtx);
+ localcount_fini(lc);
+}
+
+bool
+module_hook_tryenter(bool *hooked, struct localcount *lc)
+{
+ bool call_hook;
+ int s;
+
+ s = pserialize_read_enter();
+ call_hook = atomic_load_relaxed(hooked);
+ if (call_hook)
+ localcount_acquire(lc);
+ pserialize_read_exit(s);
+
+ return call_hook;
+}
+
+void
+module_hook_exit(struct localcount *lc)
+{
+
+ localcount_release(lc, &module_hook.cv, &module_hook.mtx);
+}
+
+void
+module_hook_init(void)
+{
+
+ mutex_init(&module_hook.mtx, MUTEX_DEFAULT, IPL_NONE);
+ cv_init(&module_hook.cv, "mod_hook");
+ module_hook.psz = pserialize_create();
+}
