I had a situation the other day where I wanted to recover some data that
was sitting on an old decommissioned raidset. Since my current kernel
doesn't have raid installed, I had to build a new kernel (GENERIC was
fine), shutdown the production box, boot up the new kernel, install the
disk drive, mount the raidset, restore files, shutdown, and reboot
normally.
What a pain in the neck. Wouldn't it have been so much simpler if I
could have just attached the drive, mounted the raidset (including
autoload of the driver module, retrieved my data, and perhaps unloaded
the no-longer-needed module? :) I had looked at doing this previously,
but backed away because it "felt like" too much complex work.
It turns out not to be such a difficult task after all!
The attached diffs (plus 2 new files) convert our existing raidframe
driver into a loadable module. Most of the changes made were needed for
enabling module unload (detaching units and releasing data structures
dynamically, rather than waiting for system shutdown time).
I've done some fairly extensive testing, on both "bare-iron" and qemu
virtual machines, and it seems to work just fine. Including scanning
for auto-configurable raidsets when the module is loaded. I've also
tested the built-in variant of the module to ensure that that still
works, too.
I'm planning to commit this later this week. I'd appreciate any review
and (hopefully constructive) comments that the community might offer.
+------------------+--------------------------+------------------------+
| Paul Goyette | PGP Key fingerprint: | E-mail addresses: |
| (Retired) | FA29 0E3B 35AF E8AE 6651 | paul at whooppee.com |
| Kernel Developer | 0786 F758 55DE 53BA 7731 | pgoyette at netbsd.org |
+------------------+--------------------------+------------------------+
# $NetBSD$
ioconf raidframe
include "conf/files"
pseudo-device raid
# $NetBSD$
.include "../Makefile.inc"
IOCONF= raid.ioconf
.PATH: ${S}/dev/raidframe
KMOD= raid
SRCS+= rf_acctrace.c rf_alloclist.c rf_aselect.c
SRCS+= rf_callback.c rf_chaindecluster.c rf_copyback.c
SRCS+= rf_cvscan.c rf_dagdegrd.c rf_dagdegwr.c
SRCS+= rf_dagffrd.c rf_dagffwr.c rf_dagfuncs.c
SRCS+= rf_dagutils.c rf_debugMem.c rf_debugprint.c
SRCS+= rf_decluster.c rf_declusterPQ.c rf_diskqueue.c
SRCS+= rf_disks.c rf_driver.c rf_engine.c
SRCS+= rf_evenodd.c rf_evenodd_dagfuncs.c rf_evenodd_dags.c
SRCS+= rf_fifo.c rf_interdecluster.c rf_invertq.c
SRCS+= rf_layout.c rf_map.c rf_mcpair.c
SRCS+= rf_netbsdkintf.c rf_nwayxor.c rf_options.c
SRCS+= rf_paritylog.c rf_paritylogDiskMgr.c rf_paritylogging.c
SRCS+= rf_parityloggingdags.c rf_paritymap.c rf_parityscan.c
SRCS+= rf_pq.c rf_pqdeg.c rf_pqdegdags.c
SRCS+= rf_psstatus.c rf_raid0.c rf_raid1.c
SRCS+= rf_raid4.c rf_raid5.c rf_raid5_rotatedspare.c
SRCS+= rf_reconbuffer.c rf_reconmap.c rf_reconstruct.c
SRCS+= rf_reconutil.c rf_revent.c rf_shutdown.c
SRCS+= rf_sstf.c rf_states.c rf_stripelocks.c
SRCS+= rf_strutils.c rf_utils.c rf_compat50.c
CPPFLAGS+= -DRAID_AUTOCONFIG=1
# Include optional raid styles
CPPFLAGS+= -DRF_INCLUDE_EVENODD=1
CPPFLAGS+= -DRF_INCLUDE_RAID5_RS=1
CPPFLAGS+= -DRF_INCLUDE_PARITYLOGGING=1
CPPFLAGS+= -DRF_INCLUDE_CHAINDECLUSTER=1
CPPFLAGS+= -DRF_INCLUDE_INTERDECLUSTER=1
CPPFLAGS+= -DRF_INCLUDE_PARITY_DECLUSTERING=1
CPPFLAGS+= -DRF_INCLUDE_PARITY_DECLUSTERING_DS=1
CPPFLAGS+= -DCOMPAT_50
.include <bsd.kmodule.mk>
Index: distrib/sets/lists/modules/mi
===================================================================
RCS file: /cvsroot/src/distrib/sets/lists/modules/mi,v
retrieving revision 1.82
diff -u -p -r1.82 mi
--- distrib/sets/lists/modules/mi 7 Dec 2015 11:38:46 -0000 1.82
+++ distrib/sets/lists/modules/mi 20 Dec 2015 04:26:50 -0000
@@ -206,6 +206,8 @@
./@MODULEDIR@/puffs/puffs.kmod base-kernel-modules kmod
./@MODULEDIR@/putter base-kernel-modules kmod
./@MODULEDIR@/putter/putter.kmod base-kernel-modules kmod
+./@MODULEDIR@/raid base-kernel-modules kmod
+./@MODULEDIR@/raid/raid.kmod base-kernel-modules kmod
./@MODULEDIR@/scsiverbose base-kernel-modules kmod
./@MODULEDIR@/scsiverbose/scsiverbose.kmod base-kernel-modules kmod
./@MODULEDIR@/sdt base-obsolete obsolete
Index: sys/modules/Makefile
===================================================================
RCS file: /cvsroot/src/sys/modules/Makefile,v
retrieving revision 1.164
diff -u -p -r1.164 Makefile
--- sys/modules/Makefile 7 Dec 2015 11:38:46 -0000 1.164
+++ sys/modules/Makefile 20 Dec 2015 04:26:54 -0000
@@ -82,6 +82,7 @@ SUBDIR+= procfs
SUBDIR+= ptyfs
SUBDIR+= puffs
SUBDIR+= putter
+SUBDIR+= raid
SUBDIR+= scsiverbose
SUBDIR+= sdtemp
SUBDIR+= secmodel_bsd44
Index: sys/dev/raidframe/rf_driver.c
===================================================================
RCS file: /cvsroot/src/sys/dev/raidframe/rf_driver.c,v
retrieving revision 1.131
diff -u -p -r1.131 rf_driver.c
--- sys/dev/raidframe/rf_driver.c 10 Dec 2012 08:36:03 -0000 1.131
+++ sys/dev/raidframe/rf_driver.c 20 Dec 2015 04:26:58 -0000
@@ -158,16 +158,21 @@ static void rf_alloc_mutex_cond(RF_Raid_
/* called at system boot time */
int
-rf_BootRaidframe(void)
+rf_BootRaidframe(bool boot)
{
- if (raidframe_booted)
- return (EBUSY);
- raidframe_booted = 1;
- rf_init_mutex2(configureMutex, IPL_NONE);
- configureCount = 0;
- isconfigged = 0;
- globalShutdown = NULL;
+ if (boot) {
+ if (raidframe_booted)
+ return (EBUSY);
+ raidframe_booted = 1;
+ rf_init_mutex2(configureMutex, IPL_NONE);
+ configureCount = 0;
+ isconfigged = 0;
+ globalShutdown = NULL;
+ } else {
+ rf_destroy_mutex2(configureMutex);
+ raidframe_booted = 0;
+ }
return (0);
}
Index: sys/dev/raidframe/rf_driver.h
===================================================================
RCS file: /cvsroot/src/sys/dev/raidframe/rf_driver.h,v
retrieving revision 1.19
diff -u -p -r1.19 rf_driver.h
--- sys/dev/raidframe/rf_driver.h 30 Apr 2011 01:44:36 -0000 1.19
+++ sys/dev/raidframe/rf_driver.h 20 Dec 2015 04:27:02 -0000
@@ -42,7 +42,7 @@
#endif
extern rf_declare_mutex2(rf_printf_mutex);
-int rf_BootRaidframe(void);
+int rf_BootRaidframe(bool);
int rf_UnbootRaidframe(void);
int rf_Shutdown(RF_Raid_t *);
int rf_Configure(RF_Raid_t *, RF_Config_t *, RF_AutoConfig_t *);
Index: sys/dev/raidframe/rf_netbsdkintf.c
===================================================================
RCS file: /cvsroot/src/sys/dev/raidframe/rf_netbsdkintf.c,v
retrieving revision 1.326
diff -u -p -r1.326 rf_netbsdkintf.c
--- sys/dev/raidframe/rf_netbsdkintf.c 8 Dec 2015 20:36:15 -0000 1.326
+++ sys/dev/raidframe/rf_netbsdkintf.c 20 Dec 2015 04:27:06 -0000
@@ -126,6 +126,7 @@ __KERNEL_RCSID(0, "$NetBSD: rf_netbsdkin
#include <sys/bufq.h>
#include <sys/reboot.h>
#include <sys/kauth.h>
+#include <sys/module.h>
#include <prop/proplib.h>
@@ -258,6 +259,14 @@ struct raid_softc {
#define raidunit(x) DISKUNIT(x)
+MODULE(MODULE_CLASS_DRIVER, raid, "dk_subr");
+
+#ifdef _MODULE
+CFDRIVER_DECL(raid, DV_DISK, NULL);
+#endif
+
+static int raid_modcmd(modcmd_t, void *);
+
extern struct cfdriver raid_cd;
CFATTACH_DECL3_NEW(raid, sizeof(struct raid_softc),
raid_match, raid_attach, raid_detach, NULL, NULL, NULL,
@@ -360,7 +369,7 @@ raiddestroy(struct raid_softc *sc) {
}
static struct raid_softc *
-raidget(int unit) {
+raidget(int unit, bool create) {
struct raid_softc *sc;
if (unit < 0) {
#ifdef DIAGNOSTIC
@@ -376,6 +385,8 @@ raidget(int unit) {
}
}
mutex_exit(&raid_lock);
+ if (!create)
+ return NULL;
if ((sc = raidcreate(unit)) == NULL)
return NULL;
mutex_enter(&raid_lock);
@@ -395,6 +406,7 @@ raidput(struct raid_softc *sc) {
void
raidattach(int num)
{
+#if 0 /* moved to raid_modcmd() */
mutex_init(&raid_lock, MUTEX_DEFAULT, IPL_NONE);
/* This is where all the initialization stuff gets done. */
@@ -423,6 +435,7 @@ raidattach(int num)
*/
if (config_finalize_register(NULL, rf_autoconfig) != 0)
aprint_error("WARNING: unable to register RAIDframe
finalizer\n");
+#endif
}
int
@@ -606,7 +619,7 @@ raidsize(dev_t dev)
int part, unit, omask, size;
unit = raidunit(dev);
- if ((rs = raidget(unit)) == NULL)
+ if ((rs = raidget(unit, false)) == NULL)
return -1;
if ((rs->sc_flags & RAIDF_INITED) == 0)
return (-1);
@@ -643,7 +656,7 @@ raiddump(dev_t dev, daddr_t blkno, void
int part, c, sparecol, j, scol, dumpto;
int error = 0;
- if ((rs = raidget(unit)) == NULL)
+ if ((rs = raidget(unit, false)) == NULL)
return ENXIO;
raidPtr = &rs->sc_r;
@@ -656,7 +669,6 @@ raiddump(dev_t dev, daddr_t blkno, void
raidPtr->Layout.numParityCol != 1)
return EINVAL;
-
if ((error = raidlock(rs)) != 0)
return error;
@@ -779,7 +791,7 @@ raidopen(dev_t dev, int flags, int fmt,
int part, pmask;
int error = 0;
- if ((rs = raidget(unit)) == NULL)
+ if ((rs = raidget(unit, true)) == NULL)
return ENXIO;
if ((error = raidlock(rs)) != 0)
return (error);
@@ -863,7 +875,7 @@ raidclose(dev_t dev, int flags, int fmt,
int error = 0;
int part;
- if ((rs = raidget(unit)) == NULL)
+ if ((rs = raidget(unit, false)) == NULL)
return ENXIO;
if ((error = raidlock(rs)) != 0)
@@ -893,15 +905,31 @@ raidclose(dev_t dev, int flags, int fmt,
rf_update_component_labels(&rs->sc_r,
RF_FINAL_COMPONENT_UPDATE);
-
- /* If the kernel is shutting down, it will detach
- * this RAID set soon enough.
+ }
+ if ((rs->sc_dkdev.dk_openmask == 0) &&
+ ((rs->sc_flags & RAIDF_SHUTDOWN) != 0)) {
+ /*
+ * Detach this raid unit
*/
+ cfdata_t cf = NULL;
+ int retcode = 0;
+
+ if (rs->sc_dev != NULL) {
+ cf = device_cfdata(rs->sc_dev);
+
+ raidunlock(rs);
+ retcode = config_detach(rs->sc_dev, DETACH_QUIET);
+ if (retcode == 0)
+ /* free the pseudo device attach bits */
+ free(cf, M_RAIDFRAME);
+ } else {
+ LIST_REMOVE(rs, sc_link);
+ }
+ return retcode;
}
raidunlock(rs);
return (0);
-
}
static void
@@ -912,7 +940,7 @@ raidstrategy(struct buf *bp)
int wlabel;
struct raid_softc *rs;
- if ((rs = raidget(unit)) == NULL) {
+ if ((rs = raidget(unit, false)) == NULL) {
bp->b_error = ENXIO;
goto done;
}
@@ -982,7 +1010,7 @@ raidread(dev_t dev, struct uio *uio, int
int unit = raidunit(dev);
struct raid_softc *rs;
- if ((rs = raidget(unit)) == NULL)
+ if ((rs = raidget(unit, false)) == NULL)
return ENXIO;
if ((rs->sc_flags & RAIDF_INITED) == 0)
@@ -999,7 +1027,7 @@ raidwrite(dev_t dev, struct uio *uio, in
int unit = raidunit(dev);
struct raid_softc *rs;
- if ((rs = raidget(unit)) == NULL)
+ if ((rs = raidget(unit, false)) == NULL)
return ENXIO;
if ((rs->sc_flags & RAIDF_INITED) == 0)
@@ -1036,6 +1064,12 @@ raid_detach_unlocked(struct raid_softc *
disk_detach(&rs->sc_dkdev);
disk_destroy(&rs->sc_dkdev);
+ /* Free the softc */
+ mutex_enter(&raid_lock);
+ LIST_REMOVE(rs, sc_link);
+ mutex_exit(&raid_lock);
+ kmem_free(rs, sizeof(*rs));
+
aprint_normal_dev(rs->sc_dev, "detached\n");
return 0;
@@ -1070,7 +1104,7 @@ raidioctl(dev_t dev, u_long cmd, void *d
struct disklabel newlabel;
#endif
- if ((rs = raidget(unit)) == NULL)
+ if ((rs = raidget(unit, false)) == NULL)
return ENXIO;
raidPtr = &rs->sc_r;
@@ -1190,23 +1224,27 @@ raidioctl(dev_t dev, u_long cmd, void *d
RF_Free(k_cfg, sizeof(RF_Config_t));
db1_printf(("rf_ioctl: retcode=%d copyin.1\n",
retcode));
- return (retcode);
+ goto no_config;
}
goto config;
config:
+ rs->sc_flags &= ~RAIDF_SHUTDOWN;
+
/* allocate a buffer for the layout-specific data, and copy it
* in */
if (k_cfg->layoutSpecificSize) {
if (k_cfg->layoutSpecificSize > 10000) {
/* sanity check */
RF_Free(k_cfg, sizeof(RF_Config_t));
- return (EINVAL);
+ retcode = EINVAL;
+ goto no_config;
}
RF_Malloc(specific_buf, k_cfg->layoutSpecificSize,
(u_char *));
if (specific_buf == NULL) {
RF_Free(k_cfg, sizeof(RF_Config_t));
- return (ENOMEM);
+ retcode = ENOMEM;
+ goto no_config;
}
retcode = copyin(k_cfg->layoutSpecific, specific_buf,
k_cfg->layoutSpecificSize);
@@ -1216,7 +1254,7 @@ raidioctl(dev_t dev, u_long cmd, void *d
k_cfg->layoutSpecificSize);
db1_printf(("rf_ioctl: retcode=%d copyin.2\n",
retcode));
- return (retcode);
+ goto no_config;
}
} else
specific_buf = NULL;
@@ -1253,6 +1291,13 @@ raidioctl(dev_t dev, u_long cmd, void *d
}
RF_Free(k_cfg, sizeof(RF_Config_t));
+ no_config:
+ /*
+ * If configuration failed, set sc_flags so that we
+ * will detach the device when we close it.
+ */
+ if (retcode != 0)
+ rs->sc_flags |= RAIDF_SHUTDOWN;
return (retcode);
/* shutdown the system */
@@ -2391,7 +2436,7 @@ raidgetdisklabel(dev_t dev)
struct cpu_disklabel *clp;
RF_Raid_t *raidPtr;
- if ((rs = raidget(unit)) == NULL)
+ if ((rs = raidget(unit, false)) == NULL)
return;
lp = rs->sc_dkdev.dk_label;
@@ -3765,7 +3810,7 @@ rf_auto_config_set(RF_ConfigSet_t *cset)
/* 1. Create a config structure */
config = malloc(sizeof(*config), M_RAIDFRAME, M_NOWAIT|M_ZERO);
if (config == NULL) {
- printf("Out of mem!?!?\n");
+ printf("%s: Out of mem - config!?!?\n", __func__);
/* XXX do something more intelligent here. */
return NULL;
}
@@ -3777,12 +3822,22 @@ rf_auto_config_set(RF_ConfigSet_t *cset)
*/
raidID = cset->ac->clabel->last_unit;
- for (sc = raidget(raidID); sc->sc_r.valid != 0; sc = raidget(++raidID))
+ for (sc = raidget(raidID, false); sc && sc->sc_r.valid != 0;
+ sc = raidget(++raidID, false))
continue;
#ifdef DEBUG
printf("Configuring raid%d:\n",raidID);
#endif
+ if (sc == NULL)
+ sc = raidget(raidID, true);
+ if (sc == NULL) {
+ printf("%s: Out of mem - softc!?!?\n", __func__);
+ /* XXX do something more intelligent here. */
+ free(config, M_RAIDFRAME);
+ return NULL;
+ }
+
raidPtr = &sc->sc_r;
/* XXX all this stuff should be done SOMEWHERE ELSE! */
@@ -3900,7 +3955,7 @@ static int
raid_detach(device_t self, int flags)
{
int error;
- struct raid_softc *rs = raidget(device_unit(self));
+ struct raid_softc *rs = raidget(device_unit(self), false);
if (rs == NULL)
return ENXIO;
@@ -3979,3 +4034,124 @@ rf_sync_component_caches(RF_Raid_t *raid
}
return error;
}
+
+/*
+ * Module interface
+ */
+
+static int
+raid_modcmd(modcmd_t cmd, void *data)
+{
+ int error;
+#ifdef _MODULE
+ int bmajor, cmajor;
+#endif
+
+ error = 0;
+ switch (cmd) {
+ case MODULE_CMD_INIT:
+ mutex_init(&raid_lock, MUTEX_DEFAULT, IPL_NONE);
+ mutex_enter(&raid_lock);
+#if (RF_INCLUDE_PARITY_DECLUSTERING_DS > 0)
+ rf_init_mutex2(rf_sparet_wait_mutex, IPL_VM);
+ rf_init_cond2(rf_sparet_wait_cv, "sparetw");
+ rf_init_cond2(rf_sparet_resp_cv, "rfgst");
+
+ rf_sparet_wait_queue = rf_sparet_resp_queue = NULL;
+#endif
+
+#ifdef _MODULE
+ bmajor = cmajor = -1;
+ error = devsw_attach("raid", &raid_bdevsw, &bmajor,
+ &raid_cdevsw, &cmajor);
+ if (error != 0) {
+ aprint_error("%s: devsw_attach failed %d\n",
+ __func__, error);
+ mutex_exit(&raid_lock);
+ break;
+ }
+ error = config_cfdriver_attach(&raid_cd);
+ if (error != 0) {
+ aprint_error("%s: config_cfdriver_attach failed %d\n",
+ __func__, error);
+ devsw_detach(&raid_bdevsw, &raid_cdevsw);
+ mutex_exit(&raid_lock);
+ break;
+ }
+#endif
+ error = config_cfattach_attach(raid_cd.cd_name, &raid_ca);
+ if (error != 0) {
+ aprint_error("%s: config_cfattach_attach failed %d\n",
+ __func__, error);
+ config_cfdriver_detach(&raid_cd);
+ devsw_detach(&raid_bdevsw, &raid_cdevsw);
+ mutex_exit(&raid_lock);
+ break;
+ }
+
+ raidautoconfigdone = false;
+
+ mutex_exit(&raid_lock);
+
+ if (error == 0) {
+ if (rf_BootRaidframe(true) == 0)
+ aprint_verbose("Kernelized RAIDframe "
+ "activated\n");
+ else
+ panic("Serious error activating RAID!!");
+ }
+
+ /*
+ * Register a finalizer which will be used to auto-config RAID
+ * sets once all real hardware devices have been found.
+ */
+ error = config_finalize_register(NULL, rf_autoconfig);
+ if (error != 0) {
+ aprint_error("WARNING: unable to register RAIDframe "
+ "finalizer\n");
+ }
+ break;
+ case MODULE_CMD_FINI:
+ mutex_enter(&raid_lock);
+
+ /* Don't allow unload if raid device(s) exist. */
+ if (!LIST_EMPTY(&raids)) {
+ mutex_exit(&raid_lock);
+ return EBUSY;
+ }
+
+ error = config_cfattach_detach(raid_cd.cd_name, &raid_ca);
+ if (error != 0) {
+ mutex_exit(&raid_lock);
+ break;
+ }
+#ifdef _MODULE
+ error = config_cfdriver_detach(&raid_cd);
+ if (error != 0) {
+ config_cfattach_attach(raid_cd.cd_name, &raid_ca);
+ mutex_exit(&raid_lock);
+ break;
+ }
+ error = devsw_detach(&raid_bdevsw, &raid_cdevsw);
+ if (error != 0) {
+ config_cfdriver_attach(&raid_cd);
+ config_cfattach_attach(raid_cd.cd_name, &raid_ca);
+ mutex_exit(&raid_lock);
+ break;
+ }
+#endif
+ rf_BootRaidframe(false);
+#if (RF_INCLUDE_PARITY_DECLUSTERING_DS > 0)
+ rf_destroy_mutex2(rf_sparet_wait_mutex);
+ rf_destroy_cond2(rf_sparet_wait_cv);
+ rf_destroy_cond2(rf_sparet_resp_cv);
+#endif
+ mutex_exit(&raid_lock);
+ mutex_destroy(&raid_lock);
+ break;
+ default:
+ error = ENOTTY;
+ break;
+ }
+ return error;
+}
Index: sys/rump/dev/lib/libraidframe/raidframe_component.c
===================================================================
RCS file: /cvsroot/src/sys/rump/dev/lib/libraidframe/raidframe_component.c,v
retrieving revision 1.2
diff -u -p -r1.2 raidframe_component.c
--- sys/rump/dev/lib/libraidframe/raidframe_component.c 20 Aug 2015 11:51:12
-0000 1.2
+++ sys/rump/dev/lib/libraidframe/raidframe_component.c 20 Dec 2015 04:27:10
-0000
@@ -39,7 +39,11 @@ __KERNEL_RCSID(0, "$NetBSD: raidframe_co
#include "ioconf.h"
-CFDRIVER_DECL(raid, DV_DISK, NULL);
+/*
+ * No longer needed - raidframe kernel module includes this declaration
+ *
+ * CFDRIVER_DECL(raid, DV_DISK, NULL);
+ */
RUMP_COMPONENT(RUMP_COMPONENT_DEV)
{
