I am liking this diff quite a bit but it needs more testers. So if you are using softraid crypto please try this diff.
On Fri, Jun 17, 2011 at 07:53:05PM +0100, Owain Ainsworth wrote: > So here's the problem: ENOMEM on io in a hba driver is bad juju. > > In more detail: > > When preparing for each io softraid crypto does a > sr_crypto_getcryptop(). This operation does a few things: > > - allocate a uio and iovec from a pool. > - if this is a read then allocate a dma buffer with dma_alloc (up to > MAXPHYS, i.e. 64k) > - allocate a cryptop to be used with the crypto(9) framework. > - initialise the above for use in the io. > > So if you are getting very low on memory, one of these operations > (probably the dma alloc) will fail. when this happens softraid returns > XS_DRIVER_STUFFUP to scsi land. This returns an EIO back to callers. > > Now it is fairly common to use softdep in these situations, synchronous > filesystems are sloooow. softdep has certain assumptions about how > things will work. if a queued dependancy fails that invalidates these > assumptions. softraid panics in this case. I used to hit this repeatedly > (several times a day) before I bought more memory. > > in general, most disk controllers prealloc everything they need (that > isn't passed down to them) so that this won't be a problem. > > This diff does the same for softraid crypto. It'll eat just over 1meg of > kva, but it is hardly the only driver to need to do that. > > The only slightly scary thing here is the cryptodesc list manipulations > (the comments explain why they are necesary). > > I'm typing from a machine that runs this. todd@ is also testing it and > marco kindly did some horrible io torture on a machine running this too > and it runs quite nicely. > > A side benefit is that due to missing out allocations in every disk io > this is actually slightly faster in io too. > > More testing would be appreciated. Cheers, > > -0- > > diff --git dev/softraid_crypto.c dev/softraid_crypto.c > index 3259121..a60824e 100644 > --- dev/softraid_crypto.c > +++ dev/softraid_crypto.c > @@ -56,9 +56,26 @@ > #include <dev/softraidvar.h> > #include <dev/rndvar.h> > > -struct cryptop *sr_crypto_getcryptop(struct sr_workunit *, int); > +/* > + * the per-io data that we need to preallocate. We can't afford to allow io > + * to start failing when memory pressure kicks in. > + * We can store this in the WU because we assert that only one > + * ccb per WU will ever be active. > + */ > +struct sr_crypto_wu { > + TAILQ_ENTRY(sr_crypto_wu) cr_link; > + struct uio cr_uio; > + struct iovec cr_iov; > + struct cryptop *cr_crp; > + struct cryptodesc *cr_descs; > + struct sr_workunit *cr_wu; > + void *cr_dmabuf; > +}; > + > + > +struct sr_crypto_wu *sr_crypto_wu_get(struct sr_workunit *, int); > +void sr_crypto_wu_put(struct sr_crypto_wu *); > int sr_crypto_create_keys(struct sr_discipline *); > -void *sr_crypto_putcryptop(struct cryptop *); > int sr_crypto_get_kdf(struct bioc_createraid *, > struct sr_discipline *); > int sr_crypto_decrypt(u_char *, u_char *, u_char *, size_t, int); > @@ -78,7 +95,7 @@ int sr_crypto_meta_opt_load(struct sr_discipline *, > struct sr_meta_opt *); > int sr_crypto_write(struct cryptop *); > int sr_crypto_rw(struct sr_workunit *); > -int sr_crypto_rw2(struct sr_workunit *, struct cryptop *); > +int sr_crypto_rw2(struct sr_workunit *, struct sr_crypto_wu *); > void sr_crypto_intr(struct buf *); > int sr_crypto_read(struct cryptop *); > void sr_crypto_finish_io(struct sr_workunit *); > @@ -230,40 +247,35 @@ done: > return (rv); > } > > -struct cryptop * > -sr_crypto_getcryptop(struct sr_workunit *wu, int encrypt) > + > +struct sr_crypto_wu * > +sr_crypto_wu_get(struct sr_workunit *wu, int encrypt) > { > struct scsi_xfer *xs = wu->swu_xs; > struct sr_discipline *sd = wu->swu_dis; > - struct cryptop *crp = NULL; > + struct sr_crypto_wu *crwu; > struct cryptodesc *crd; > - struct uio *uio = NULL; > - int flags, i, n, s; > + int flags, i, n; > daddr64_t blk = 0; > u_int keyndx; > > - DNPRINTF(SR_D_DIS, "%s: sr_crypto_getcryptop wu: %p encrypt: %d\n", > + DNPRINTF(SR_D_DIS, "%s: sr_crypto_wu_get wu: %p encrypt: %d\n", > DEVNAME(sd->sd_sc), wu, encrypt); > > - s = splbio(); > - uio = pool_get(&sd->mds.mdd_crypto.sr_uiopl, PR_ZERO | PR_NOWAIT); > - if (uio == NULL) > - goto poolunwind; > - uio->uio_iov = pool_get(&sd->mds.mdd_crypto.sr_iovpl, > - PR_ZERO | PR_NOWAIT); > - if (uio->uio_iov == NULL) > - goto poolunwind; > - splx(s); > + mtx_enter(&sd->mds.mdd_crypto.scr_mutex); > + if ((crwu = TAILQ_FIRST(&sd->mds.mdd_crypto.scr_wus)) != NULL) > + TAILQ_REMOVE(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link); > + mtx_leave(&sd->mds.mdd_crypto.scr_mutex); > + if (crwu == NULL) > + panic("sr_crypto_wu_get: out of wus"); > > - uio->uio_iovcnt = 1; > - uio->uio_iov->iov_len = xs->datalen; > + crwu->cr_uio.uio_iovcnt = 1; > + crwu->cr_uio.uio_iov->iov_len = xs->datalen; > if (xs->flags & SCSI_DATA_OUT) { > - uio->uio_iov->iov_base = dma_alloc(xs->datalen, PR_NOWAIT); > - if (uio->uio_iov->iov_base == NULL) > - goto unwind; > - bcopy(xs->data, uio->uio_iov->iov_base, xs->datalen); > + crwu->cr_uio.uio_iov->iov_base = crwu->cr_dmabuf; > + bcopy(xs->data, crwu->cr_uio.uio_iov->iov_base, xs->datalen); > } else > - uio->uio_iov->iov_base = xs->data; > + crwu->cr_uio.uio_iov->iov_base = xs->data; > > if (xs->cmdlen == 10) > blk = _4btol(((struct scsi_rw_big *)xs->cmd)->addr); > @@ -273,25 +285,40 @@ sr_crypto_getcryptop(struct sr_workunit *wu, int > encrypt) > blk = _3btol(((struct scsi_rw *)xs->cmd)->addr); > > n = xs->datalen >> DEV_BSHIFT; > + > + /* > + * we preallocated enough crypto descs for up to MAXPHYS of io. > + * since ios may be less than that we need to tweak the linked list > + * of crypto desc structures to be just long enough for our needs. > + * Otherwise crypto will get upset with us. So put n descs on the crp > + * and keep the rest. > + */ > + crd = crwu->cr_descs; > + i = 0; > + while (++i < n) { > + crd = crd->crd_next; > + KASSERT(crd); > + } > + crwu->cr_crp->crp_desc = crwu->cr_descs; > + crwu->cr_descs = crd->crd_next; > + crd->crd_next = NULL; > + > flags = (encrypt ? CRD_F_ENCRYPT : 0) | > CRD_F_IV_PRESENT | CRD_F_IV_EXPLICIT; > > - crp = crypto_getreq(n); > - if (crp == NULL) > - goto unwind; > - > /* Select crypto session based on block number */ > keyndx = blk >> SR_CRYPTO_KEY_BLKSHIFT; > if (keyndx >= SR_CRYPTO_MAXKEYS) > goto unwind; > - crp->crp_sid = sd->mds.mdd_crypto.scr_sid[keyndx]; > - if (crp->crp_sid == (u_int64_t)-1) > + crwu->cr_crp->crp_sid = sd->mds.mdd_crypto.scr_sid[keyndx]; > + if (crwu->cr_crp->crp_sid == (u_int64_t)-1) > goto unwind; > > - crp->crp_ilen = xs->datalen; > - crp->crp_alloctype = M_DEVBUF; > - crp->crp_buf = uio; > - for (i = 0, crd = crp->crp_desc; crd; i++, blk++, crd = crd->crd_next) { > + crwu->cr_crp->crp_ilen = xs->datalen; > + crwu->cr_crp->crp_alloctype = M_DEVBUF; > + crwu->cr_crp->crp_buf = &crwu->cr_uio; > + for (i = 0, crd = crwu->cr_crp->crp_desc; crd; > + i++, blk++, crd = crd->crd_next) { > crd->crd_skip = i << DEV_BSHIFT; > crd->crd_len = DEV_BSIZE; > crd->crd_inject = 0; > @@ -311,48 +338,50 @@ sr_crypto_getcryptop(struct sr_workunit *wu, int > encrypt) > crd->crd_key = sd->mds.mdd_crypto.scr_key[0]; > bcopy(&blk, crd->crd_iv, sizeof(blk)); > } > + crwu->cr_wu = wu; > + crwu->cr_crp->crp_opaque = crwu; > > - return (crp); > -poolunwind: > - splx(s); > + return (crwu); > unwind: > - if (crp) > - crypto_freereq(crp); > - if (uio && uio->uio_iov) > - if ((wu->swu_xs->flags & SCSI_DATA_OUT) && > - uio->uio_iov->iov_base) > - dma_free(uio->uio_iov->iov_base, uio->uio_iov->iov_len); > - > - s = splbio(); > - if (uio && uio->uio_iov) > - pool_put(&sd->mds.mdd_crypto.sr_iovpl, uio->uio_iov); > - if (uio) > - pool_put(&sd->mds.mdd_crypto.sr_uiopl, uio); > - splx(s); > + /* steal the descrptions back from the cryptop */ > + crd = crwu->cr_crp->crp_desc; > + while (crd->crd_next != NULL) { > + crd = crd->crd_next; > + } > > + /* join the lists back again */ > + crd->crd_next = crwu->cr_descs; > + crwu->cr_descs = crwu->cr_crp->crp_desc; > + crwu->cr_crp->crp_desc = NULL; > return (NULL); > } > > -void * > -sr_crypto_putcryptop(struct cryptop *crp) > +void > +sr_crypto_wu_put(struct sr_crypto_wu *crwu) > { > - struct uio *uio = crp->crp_buf; > - struct sr_workunit *wu = crp->crp_opaque; > + struct cryptop *crp = crwu->cr_crp; > + struct sr_workunit *wu = crwu->cr_wu; > struct sr_discipline *sd = wu->swu_dis; > - int s; > + struct cryptodesc *crd; > > - DNPRINTF(SR_D_DIS, "%s: sr_crypto_putcryptop crp: %p\n", > - DEVNAME(wu->swu_dis->sd_sc), crp); > + DNPRINTF(SR_D_DIS, "%s: sr_crypto_wu_put crwu: %p\n", > + DEVNAME(wu->swu_dis->sd_sc), crwu); > > - if ((wu->swu_xs->flags & SCSI_DATA_OUT) && uio->uio_iov->iov_base) > - dma_free(uio->uio_iov->iov_base, uio->uio_iov->iov_len); > - s = splbio(); > - pool_put(&sd->mds.mdd_crypto.sr_iovpl, uio->uio_iov); > - pool_put(&sd->mds.mdd_crypto.sr_uiopl, uio); > - splx(s); > - crypto_freereq(crp); > + /* steal the descrptions back from the cryptop */ > + crd = crp->crp_desc; > + KASSERT(crd); > + while (crd->crd_next != NULL) { > + crd = crd->crd_next; > + } > + > + /* join the lists back again */ > + crd->crd_next = crwu->cr_descs; > + crwu->cr_descs = crp->crp_desc; > + crp->crp_desc = NULL; > > - return (wu); > + mtx_enter(&sd->mds.mdd_crypto.scr_mutex); > + TAILQ_INSERT_TAIL(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link); > + mtx_leave(&sd->mds.mdd_crypto.scr_mutex); > } > > int > @@ -928,6 +957,7 @@ int > sr_crypto_alloc_resources(struct sr_discipline *sd) > { > struct cryptoini cri; > + struct sr_crypto_wu *crwu; > u_int num_keys, i; > > if (!sd) > @@ -936,11 +966,6 @@ sr_crypto_alloc_resources(struct sr_discipline *sd) > DNPRINTF(SR_D_DIS, "%s: sr_crypto_alloc_resources\n", > DEVNAME(sd->sd_sc)); > > - pool_init(&sd->mds.mdd_crypto.sr_uiopl, sizeof(struct uio), 0, 0, 0, > - "sr_uiopl", NULL); > - pool_init(&sd->mds.mdd_crypto.sr_iovpl, sizeof(struct iovec), 0, 0, 0, > - "sr_iovpl", NULL); > - > for (i = 0; i < SR_CRYPTO_MAXKEYS; i++) > sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1; > > @@ -950,6 +975,33 @@ sr_crypto_alloc_resources(struct sr_discipline *sd) > return (ENOMEM); > if (sr_crypto_decrypt_key(sd)) > return (EPERM); > + /* > + * For each wu allocate the uio, iovec and crypto structures. > + * these have to be allocated now because during runtime we can't > + * fail an allocation without failing the io (which can cause real > + * problems). > + */ > + mtx_init(&sd->mds.mdd_crypto.scr_mutex, IPL_BIO); > + TAILQ_INIT(&sd->mds.mdd_crypto.scr_wus); > + for (i = 0; i < sd->sd_max_wu; i++) { > + crwu = malloc(sizeof(*crwu), M_DEVBUF, > + M_WAITOK | M_ZERO | M_CANFAIL); > + if (crwu == NULL) > + return (ENOMEM); > + /* put it on the list now so if we fail it'll be freed */ > + mtx_enter(&sd->mds.mdd_crypto.scr_mutex); > + TAILQ_INSERT_TAIL(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link); > + mtx_leave(&sd->mds.mdd_crypto.scr_mutex); > + > + crwu->cr_uio.uio_iov = &crwu->cr_iov; > + crwu->cr_dmabuf = dma_alloc(MAXPHYS, PR_WAITOK); > + crwu->cr_crp = crypto_getreq(MAXPHYS >> DEV_BSHIFT); > + if (crwu->cr_crp == NULL) > + return (ENOMEM); > + /* steal the list of cryptodescs */ > + crwu->cr_descs = crwu->cr_crp->crp_desc; > + crwu->cr_crp->crp_desc = NULL; > + } > > bzero(&cri, sizeof(cri)); > cri.cri_alg = CRYPTO_AES_XTS; > @@ -992,6 +1044,7 @@ int > sr_crypto_free_resources(struct sr_discipline *sd) > { > int rv = EINVAL; > + struct sr_crypto_wu *crwu; > u_int i; > > if (!sd) > @@ -1013,14 +1066,21 @@ sr_crypto_free_resources(struct sr_discipline *sd) > sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1; > } > > + while ((crwu = TAILQ_FIRST(&sd->mds.mdd_crypto.scr_wus)) != NULL) { > + TAILQ_REMOVE(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link); > + > + dma_free(crwu->cr_dmabuf, MAXPHYS); > + /* twiddle cryptoreq back */ > + if (crwu->cr_crp) { > + crwu->cr_crp->crp_desc = crwu->cr_descs; > + crypto_freereq(crwu->cr_crp); > + } > + free(crwu, M_DEVBUF); > + } > + > sr_wu_free(sd); > sr_ccb_free(sd); > > - if (sd->mds.mdd_crypto.sr_uiopl.pr_serial != 0) > - pool_destroy(&sd->mds.mdd_crypto.sr_uiopl); > - if (sd->mds.mdd_crypto.sr_iovpl.pr_serial != 0) > - pool_destroy(&sd->mds.mdd_crypto.sr_iovpl); > - > rv = 0; > return (rv); > } > @@ -1104,23 +1164,22 @@ sr_crypto_meta_opt_load(struct sr_discipline *sd, > struct sr_meta_opt *om) > int > sr_crypto_rw(struct sr_workunit *wu) > { > - struct cryptop *crp; > + struct sr_crypto_wu *crwu; > int s, rv = 0; > > DNPRINTF(SR_D_DIS, "%s: sr_crypto_rw wu: %p\n", > DEVNAME(wu->swu_dis->sd_sc), wu); > > if (wu->swu_xs->flags & SCSI_DATA_OUT) { > - crp = sr_crypto_getcryptop(wu, 1); > - if (crp == NULL) > + crwu = sr_crypto_wu_get(wu, 1); > + if (crwu == NULL) > return (1); > - crp->crp_callback = sr_crypto_write; > - crp->crp_opaque = wu; > + crwu->cr_crp->crp_callback = sr_crypto_write; > s = splvm(); > - if (crypto_invoke(crp)) > + if (crypto_invoke(crwu->cr_crp)) > rv = 1; > else > - rv = crp->crp_etype; > + rv = crwu->cr_crp->crp_etype; > splx(s); > } else > rv = sr_crypto_rw2(wu, NULL); > @@ -1131,26 +1190,26 @@ sr_crypto_rw(struct sr_workunit *wu) > int > sr_crypto_write(struct cryptop *crp) > { > + struct sr_crypto_wu *crwu = crp->crp_opaque; > + struct sr_workunit *wu = crwu->cr_wu; > int s; > - struct sr_workunit *wu = crp->crp_opaque; > > DNPRINTF(SR_D_INTR, "%s: sr_crypto_write: wu %x xs: %x\n", > DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs); > > if (crp->crp_etype) { > /* fail io */ > - ((struct sr_workunit *)(crp->crp_opaque))->swu_xs->error = > - XS_DRIVER_STUFFUP; > + wu->swu_xs->error = XS_DRIVER_STUFFUP; > s = splbio(); > - sr_crypto_finish_io(crp->crp_opaque); > + sr_crypto_finish_io(wu); > splx(s); > } > > - return (sr_crypto_rw2(wu, crp)); > + return (sr_crypto_rw2(wu, crwu)); > } > > int > -sr_crypto_rw2(struct sr_workunit *wu, struct cryptop *crp) > +sr_crypto_rw2(struct sr_workunit *wu, struct sr_crypto_wu *crwu) > { > struct sr_discipline *sd = wu->swu_dis; > struct scsi_xfer *xs = wu->swu_xs; > @@ -1185,10 +1244,10 @@ sr_crypto_rw2(struct sr_workunit *wu, struct cryptop > *crp) > ccb->ccb_buf.b_flags |= B_READ; > ccb->ccb_buf.b_data = xs->data; > } else { > - uio = crp->crp_buf; > + uio = crwu->cr_crp->crp_buf; > ccb->ccb_buf.b_flags |= B_WRITE; > ccb->ccb_buf.b_data = uio->uio_iov->iov_base; > - ccb->ccb_opaque = crp; > + ccb->ccb_opaque = crwu; > } > > ccb->ccb_buf.b_error = 0; > @@ -1224,8 +1283,8 @@ queued: > return (0); > bad: > /* wu is unwound by sr_wu_put */ > - if (crp) > - crp->crp_etype = EINVAL; > + if (crwu) > + crwu->cr_crp->crp_etype = EINVAL; > return (1); > } > > @@ -1237,7 +1296,7 @@ sr_crypto_intr(struct buf *bp) > struct sr_discipline *sd = wu->swu_dis; > struct scsi_xfer *xs = wu->swu_xs; > struct sr_softc *sc = sd->sd_sc; > - struct cryptop *crp; > + struct sr_crypto_wu *crwu; > int s, s2, pend; > > DNPRINTF(SR_D_INTR, "%s: sr_crypto_intr bp: %x xs: %x\n", > @@ -1296,16 +1355,16 @@ sr_crypto_intr(struct buf *bp) > DEVNAME(sc), wu); > > if ((xs->flags & SCSI_DATA_IN) && (xs->error == XS_NOERROR)) { > - crp = sr_crypto_getcryptop(wu, 0); > - if (crp == NULL) > - panic("sr_crypto_intr: no crypto op"); > - ccb->ccb_opaque = crp; > - crp->crp_callback = sr_crypto_read; > - crp->crp_opaque = wu; > + /* only fails on implementation error */ > + crwu = sr_crypto_wu_get(wu, 0); > + if (crwu == NULL) > + panic("sr_crypto_intr: no wu"); > + crwu->cr_crp->crp_callback = sr_crypto_read; > + ccb->ccb_opaque = crwu; > DNPRINTF(SR_D_INTR, "%s: sr_crypto_intr: crypto_invoke " > - "%p\n", DEVNAME(sc), crp); > + "%p\n", DEVNAME(sc), crwu->cr_crp); > s2 = splvm(); > - crypto_invoke(crp); > + crypto_invoke(crwu->cr_crp); > splx(s2); > goto done; > } > @@ -1339,7 +1398,7 @@ sr_crypto_finish_io(struct sr_workunit *wu) > TAILQ_FOREACH(ccb, &wu->swu_ccb, ccb_link) { > if (ccb->ccb_opaque == NULL) > continue; > - sr_crypto_putcryptop(ccb->ccb_opaque); > + sr_crypto_wu_put(ccb->ccb_opaque); > } > > sr_scsi_done(sd, xs); > @@ -1351,8 +1410,9 @@ sr_crypto_finish_io(struct sr_workunit *wu) > int > sr_crypto_read(struct cryptop *crp) > { > + struct sr_crypto_wu *crwu = crp->crp_opaque; > + struct sr_workunit *wu = crwu->cr_wu; > int s; > - struct sr_workunit *wu = crp->crp_opaque; > > DNPRINTF(SR_D_INTR, "%s: sr_crypto_read: wu %x xs: %x\n", > DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs); > diff --git dev/softraid_vnode.c dev/softraid_vnode.c > new file mode 100644 > index 0000000..2c362d5 > --- /dev/null > +++ dev/softraid_vnode.c > @@ -0,0 +1,1378 @@ > +/* $OpenBSD$ */ > +/* > + * Copyright (c) 2011 Owain G. Ainsworth <o...@openbsd.org> > + * Copyright (c) 2008 Hans-Joerg Hoexer <hshoe...@openbsd.org> > + * Copyright (c) 2008 Damien Miller <d...@mindrot.org> > + * Copyright (c) 2009 Joel Sing <js...@openbsd.org> > + * > + * Permission to use, copy, modify, and distribute this software for any > + * purpose with or without fee is hereby granted, provided that the above > + * copyright notice and this permission notice appear in all copies. > + * > + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES > + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF > + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR > + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES > + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN > + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF > + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. > + */ > + > +#include "bio.h" > + > +#include <sys/param.h> > +#include <sys/systm.h> > +#include <sys/buf.h> > +#include <sys/device.h> > +#include <sys/ioctl.h> > +#include <sys/proc.h> > +#include <sys/malloc.h> > +#include <sys/pool.h> > +#include <sys/kernel.h> > +#include <sys/disk.h> > +#include <sys/rwlock.h> > +#include <sys/queue.h> > +#include <sys/fcntl.h> > +#include <sys/disklabel.h> > +#include <sys/mount.h> > +#include <sys/sensors.h> > +#include <sys/stat.h> > +#include <sys/conf.h> > +#include <sys/uio.h> > +#include <sys/dkio.h> > + > +#include <crypto/cryptodev.h> > +#include <crypto/cryptosoft.h> > +#include <crypto/rijndael.h> > +#include <crypto/md5.h> > +#include <crypto/sha1.h> > +#include <crypto/sha2.h> > +#include <crypto/hmac.h> > + > +#include <scsi/scsi_all.h> > +#include <scsi/scsiconf.h> > +#include <scsi/scsi_disk.h> > + > +#include <dev/softraidvar.h> > +#include <dev/rndvar.h> > + > +struct cryptop *sr_crypto_getcryptop(struct sr_workunit *, int); > +int sr_crypto_create_keys(struct sr_discipline *); > +void *sr_crypto_putcryptop(struct cryptop *); > +int sr_crypto_get_kdf(struct bioc_createraid *, > + struct sr_discipline *); > +int sr_crypto_decrypt(u_char *, u_char *, u_char *, size_t, int); > +int sr_crypto_encrypt(u_char *, u_char *, u_char *, size_t, int); > +int sr_crypto_decrypt_key(struct sr_discipline *); > +int sr_crypto_change_maskkey(struct sr_discipline *, > + struct sr_crypto_kdfinfo *, struct sr_crypto_kdfinfo *); > +int sr_crypto_create(struct sr_discipline *, > + struct bioc_createraid *, int, int64_t); > +int sr_crypto_assemble(struct sr_discipline *, > + struct bioc_createraid *, int); > +int sr_crypto_alloc_resources(struct sr_discipline *); > +int sr_crypto_free_resources(struct sr_discipline *); > +int sr_crypto_ioctl(struct sr_discipline *, > + struct bioc_discipline *); > +int sr_crypto_meta_opt_load(struct sr_discipline *, > + struct sr_meta_opt *); > +int sr_crypto_write(struct cryptop *); > +int sr_crypto_rw(struct sr_workunit *); > +int sr_crypto_rw2(struct sr_workunit *, struct cryptop *); > +void sr_crypto_intr(struct buf *); > +int sr_crypto_read(struct cryptop *); > +void sr_crypto_finish_io(struct sr_workunit *); > +void sr_crypto_calculate_check_hmac_sha1(u_int8_t *, int, > + u_int8_t *, int, u_char *); > +void sr_crypto_hotplug(struct sr_discipline *, struct disk *, int); > + > +#ifdef SR_DEBUG0 > +void sr_crypto_dumpkeys(struct sr_discipline *); > +#endif > + > +/* Discipline initialisation. */ > +void > +sr_crypto_discipline_init(struct sr_discipline *sd) > +{ > + int i; > + > + /* Fill out discipline members. */ > + sd->sd_type = SR_MD_CRYPTO; > + sd->sd_capabilities = SR_CAP_SYSTEM_DISK; > + sd->sd_max_wu = SR_CRYPTO_NOWU; > + > + for (i = 0; i < SR_CRYPTO_MAXKEYS; i++) > + sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1; > + > + /* Setup discipline pointers. */ > + sd->sd_create = sr_crypto_create; > + sd->sd_assemble = sr_crypto_assemble; > + sd->sd_alloc_resources = sr_crypto_alloc_resources; > + sd->sd_free_resources = sr_crypto_free_resources; > + sd->sd_start_discipline = NULL; > + sd->sd_ioctl_handler = sr_crypto_ioctl; > + sd->sd_meta_opt_load = sr_crypto_meta_opt_load; > + sd->sd_scsi_inquiry = sr_raid_inquiry; > + sd->sd_scsi_read_cap = sr_raid_read_cap; > + sd->sd_scsi_tur = sr_raid_tur; > + sd->sd_scsi_req_sense = sr_raid_request_sense; > + sd->sd_scsi_start_stop = sr_raid_start_stop; > + sd->sd_scsi_sync = sr_raid_sync; > + sd->sd_scsi_rw = sr_crypto_rw; > + /* XXX reuse raid 1 functions for now FIXME */ > + sd->sd_set_chunk_state = sr_raid1_set_chunk_state; > + sd->sd_set_vol_state = sr_raid1_set_vol_state; > +} > + > +int > +sr_crypto_create(struct sr_discipline *sd, struct bioc_createraid *bc, > + int no_chunk, int64_t coerced_size) > +{ > + struct sr_meta_opt_item *omi; > + int rv = EINVAL; > + > + if (no_chunk != 1) > + goto done; > + > + /* Create crypto optional metadata. */ > + omi = malloc(sizeof(struct sr_meta_opt_item), M_DEVBUF, > + M_WAITOK | M_ZERO); > + omi->omi_om.somi.som_type = SR_OPT_CRYPTO; > + SLIST_INSERT_HEAD(&sd->sd_meta_opt, omi, omi_link); > + sd->mds.mdd_crypto.scr_meta = &omi->omi_om.somi.som_meta.smm_crypto; > + sd->sd_meta->ssdi.ssd_opt_no++; > + > + sd->mds.mdd_crypto.key_disk = NULL; > + > + if (bc->bc_key_disk != NODEV) { > + > + /* Create a key disk. */ > + if (sr_crypto_get_kdf(bc, sd)) > + goto done; > + sd->mds.mdd_crypto.key_disk = > + sr_crypto_create_key_disk(sd, bc->bc_key_disk); > + if (sd->mds.mdd_crypto.key_disk == NULL) > + goto done; > + sd->sd_capabilities |= SR_CAP_AUTO_ASSEMBLE; > + > + } else if (bc->bc_opaque_flags & BIOC_SOOUT) { > + > + /* No hint available yet. */ > + bc->bc_opaque_status = BIOC_SOINOUT_FAILED; > + rv = EAGAIN; > + goto done; > + > + } else if (sr_crypto_get_kdf(bc, sd)) > + goto done; > + > + /* Passphrase volumes cannot be automatically assembled. */ > + if (!(bc->bc_flags & BIOC_SCNOAUTOASSEMBLE) && bc->bc_key_disk == NODEV) > + goto done; > + > + strlcpy(sd->sd_name, "CRYPTO", sizeof(sd->sd_name)); > + sd->sd_meta->ssdi.ssd_size = coerced_size; > + > + sr_crypto_create_keys(sd); > + > + sd->sd_max_ccb_per_wu = no_chunk; > + > + rv = 0; > +done: > + return (rv); > +} > + > +int > +sr_crypto_assemble(struct sr_discipline *sd, struct bioc_createraid *bc, > + int no_chunk) > +{ > + int rv = EINVAL; > + > + sd->mds.mdd_crypto.key_disk = NULL; > + > + /* Crypto optional metadata must already exist... */ > + if (sd->mds.mdd_crypto.scr_meta == NULL) > + goto done; > + > + if (bc->bc_key_disk != NODEV) { > + /* Read the mask key from the key disk. */ > + sd->mds.mdd_crypto.key_disk = > + sr_crypto_read_key_disk(sd, bc->bc_key_disk); > + if (sd->mds.mdd_crypto.key_disk == NULL) > + goto done; > + } else if (bc->bc_opaque_flags & BIOC_SOOUT) { > + /* provide userland with kdf hint */ > + if (bc->bc_opaque == NULL) > + goto done; > + > + if (sizeof(sd->mds.mdd_crypto.scr_meta->scm_kdfhint) < > + bc->bc_opaque_size) > + goto done; > + > + if (copyout(sd->mds.mdd_crypto.scr_meta->scm_kdfhint, > + bc->bc_opaque, bc->bc_opaque_size)) > + goto done; > + > + /* we're done */ > + bc->bc_opaque_status = BIOC_SOINOUT_OK; > + rv = EAGAIN; > + goto done; > + } else if (bc->bc_opaque_flags & BIOC_SOIN) { > + /* get kdf with maskkey from userland */ > + if (sr_crypto_get_kdf(bc, sd)) > + goto done; > + > + } > + > + sd->sd_max_ccb_per_wu = sd->sd_meta->ssdi.ssd_chunk_no; > + > + rv = 0; > +done: > + return (rv); > +} > + > +struct cryptop * > +sr_crypto_getcryptop(struct sr_workunit *wu, int encrypt) > +{ > + struct scsi_xfer *xs = wu->swu_xs; > + struct sr_discipline *sd = wu->swu_dis; > + struct cryptop *crp = NULL; > + struct cryptodesc *crd; > + struct uio *uio = NULL; > + int flags, i, n, s; > + daddr64_t blk = 0; > + u_int keyndx; > + > + DNPRINTF(SR_D_DIS, "%s: sr_crypto_getcryptop wu: %p encrypt: %d\n", > + DEVNAME(sd->sd_sc), wu, encrypt); > + > + s = splbio(); > + uio = pool_get(&sd->mds.mdd_crypto.sr_uiopl, PR_ZERO | PR_NOWAIT); > + if (uio == NULL) > + goto poolunwind; > + uio->uio_iov = pool_get(&sd->mds.mdd_crypto.sr_iovpl, > + PR_ZERO | PR_NOWAIT); > + if (uio->uio_iov == NULL) > + goto poolunwind; > + splx(s); > + > + uio->uio_iovcnt = 1; > + uio->uio_iov->iov_len = xs->datalen; > + if (xs->flags & SCSI_DATA_OUT) { > + uio->uio_iov->iov_base = dma_alloc(xs->datalen, PR_NOWAIT); > + if (uio->uio_iov->iov_base == NULL) > + goto unwind; > + bcopy(xs->data, uio->uio_iov->iov_base, xs->datalen); > + } else > + uio->uio_iov->iov_base = xs->data; > + > + if (xs->cmdlen == 10) > + blk = _4btol(((struct scsi_rw_big *)xs->cmd)->addr); > + else if (xs->cmdlen == 16) > + blk = _8btol(((struct scsi_rw_16 *)xs->cmd)->addr); > + else if (xs->cmdlen == 6) > + blk = _3btol(((struct scsi_rw *)xs->cmd)->addr); > + > + n = xs->datalen >> DEV_BSHIFT; > + flags = (encrypt ? CRD_F_ENCRYPT : 0) | > + CRD_F_IV_PRESENT | CRD_F_IV_EXPLICIT; > + > + crp = crypto_getreq(n); > + if (crp == NULL) > + goto unwind; > + > + /* Select crypto session based on block number */ > + keyndx = blk >> SR_CRYPTO_KEY_BLKSHIFT; > + if (keyndx >= SR_CRYPTO_MAXKEYS) > + goto unwind; > + crp->crp_sid = sd->mds.mdd_crypto.scr_sid[keyndx]; > + if (crp->crp_sid == (u_int64_t)-1) > + goto unwind; > + > + crp->crp_ilen = xs->datalen; > + crp->crp_alloctype = M_DEVBUF; > + crp->crp_buf = uio; > + for (i = 0, crd = crp->crp_desc; crd; i++, blk++, crd = crd->crd_next) { > + crd->crd_skip = i << DEV_BSHIFT; > + crd->crd_len = DEV_BSIZE; > + crd->crd_inject = 0; > + crd->crd_flags = flags; > + crd->crd_alg = CRYPTO_AES_XTS; > + > + switch (sd->mds.mdd_crypto.scr_meta->scm_alg) { > + case SR_CRYPTOA_AES_XTS_128: > + crd->crd_klen = 256; > + break; > + case SR_CRYPTOA_AES_XTS_256: > + crd->crd_klen = 512; > + break; > + default: > + goto unwind; > + } > + crd->crd_key = sd->mds.mdd_crypto.scr_key[0]; > + bcopy(&blk, crd->crd_iv, sizeof(blk)); > + } > + > + return (crp); > +poolunwind: > + splx(s); > +unwind: > + if (crp) > + crypto_freereq(crp); > + if (uio && uio->uio_iov) > + if ((wu->swu_xs->flags & SCSI_DATA_OUT) && > + uio->uio_iov->iov_base) > + dma_free(uio->uio_iov->iov_base, uio->uio_iov->iov_len); > + > + s = splbio(); > + if (uio && uio->uio_iov) > + pool_put(&sd->mds.mdd_crypto.sr_iovpl, uio->uio_iov); > + if (uio) > + pool_put(&sd->mds.mdd_crypto.sr_uiopl, uio); > + splx(s); > + > + return (NULL); > +} > + > +void * > +sr_crypto_putcryptop(struct cryptop *crp) > +{ > + struct uio *uio = crp->crp_buf; > + struct sr_workunit *wu = crp->crp_opaque; > + struct sr_discipline *sd = wu->swu_dis; > + int s; > + > + DNPRINTF(SR_D_DIS, "%s: sr_crypto_putcryptop crp: %p\n", > + DEVNAME(wu->swu_dis->sd_sc), crp); > + > + if ((wu->swu_xs->flags & SCSI_DATA_OUT) && uio->uio_iov->iov_base) > + dma_free(uio->uio_iov->iov_base, uio->uio_iov->iov_len); > + s = splbio(); > + pool_put(&sd->mds.mdd_crypto.sr_iovpl, uio->uio_iov); > + pool_put(&sd->mds.mdd_crypto.sr_uiopl, uio); > + splx(s); > + crypto_freereq(crp); > + > + return (wu); > +} > + > +int > +sr_crypto_get_kdf(struct bioc_createraid *bc, struct sr_discipline *sd) > +{ > + int rv = EINVAL; > + struct sr_crypto_kdfinfo *kdfinfo; > + > + if (!(bc->bc_opaque_flags & BIOC_SOIN)) > + return (rv); > + if (bc->bc_opaque == NULL) > + return (rv); > + if (bc->bc_opaque_size < sizeof(*kdfinfo)) > + return (rv); > + > + kdfinfo = malloc(bc->bc_opaque_size, M_DEVBUF, M_WAITOK | M_ZERO); > + if (copyin(bc->bc_opaque, kdfinfo, bc->bc_opaque_size)) > + goto out; > + > + if (kdfinfo->len != bc->bc_opaque_size) > + goto out; > + > + /* copy KDF hint to disk meta data */ > + if (kdfinfo->flags & SR_CRYPTOKDF_HINT) { > + if (sizeof(sd->mds.mdd_crypto.scr_meta->scm_kdfhint) < > + kdfinfo->genkdf.len) > + goto out; > + bcopy(&kdfinfo->genkdf, > + sd->mds.mdd_crypto.scr_meta->scm_kdfhint, > + kdfinfo->genkdf.len); > + } > + > + /* copy mask key to run-time meta data */ > + if ((kdfinfo->flags & SR_CRYPTOKDF_KEY)) { > + if (sizeof(sd->mds.mdd_crypto.scr_maskkey) < > + sizeof(kdfinfo->maskkey)) > + goto out; > + bcopy(&kdfinfo->maskkey, sd->mds.mdd_crypto.scr_maskkey, > + sizeof(kdfinfo->maskkey)); > + } > + > + bc->bc_opaque_status = BIOC_SOINOUT_OK; > + rv = 0; > +out: > + explicit_bzero(kdfinfo, bc->bc_opaque_size); > + free(kdfinfo, M_DEVBUF); > + > + return (rv); > +} > + > +int > +sr_crypto_encrypt(u_char *p, u_char *c, u_char *key, size_t size, int alg) > +{ > + rijndael_ctx ctx; > + int i, rv = 1; > + > + switch (alg) { > + case SR_CRYPTOM_AES_ECB_256: > + if (rijndael_set_key_enc_only(&ctx, key, 256) != 0) > + goto out; > + for (i = 0; i < size; i += RIJNDAEL128_BLOCK_LEN) > + rijndael_encrypt(&ctx, &p[i], &c[i]); > + rv = 0; > + break; > + default: > + DNPRINTF(SR_D_DIS, "%s: unsupported encryption algorithm %u\n", > + "softraid", alg); > + rv = -1; > + goto out; > + } > + > +out: > + explicit_bzero(&ctx, sizeof(ctx)); > + return (rv); > +} > + > +int > +sr_crypto_decrypt(u_char *c, u_char *p, u_char *key, size_t size, int alg) > +{ > + rijndael_ctx ctx; > + int i, rv = 1; > + > + switch (alg) { > + case SR_CRYPTOM_AES_ECB_256: > + if (rijndael_set_key(&ctx, key, 256) != 0) > + goto out; > + for (i = 0; i < size; i += RIJNDAEL128_BLOCK_LEN) > + rijndael_decrypt(&ctx, &c[i], &p[i]); > + rv = 0; > + break; > + default: > + DNPRINTF(SR_D_DIS, "%s: unsupported encryption algorithm %u\n", > + "softraid", alg); > + rv = -1; > + goto out; > + } > + > +out: > + explicit_bzero(&ctx, sizeof(ctx)); > + return (rv); > +} > + > +void > +sr_crypto_calculate_check_hmac_sha1(u_int8_t *maskkey, int maskkey_size, > + u_int8_t *key, int key_size, u_char *check_digest) > +{ > + u_char check_key[SHA1_DIGEST_LENGTH]; > + HMAC_SHA1_CTX hmacctx; > + SHA1_CTX shactx; > + > + bzero(check_key, sizeof(check_key)); > + bzero(&hmacctx, sizeof(hmacctx)); > + bzero(&shactx, sizeof(shactx)); > + > + /* k = SHA1(mask_key) */ > + SHA1Init(&shactx); > + SHA1Update(&shactx, maskkey, maskkey_size); > + SHA1Final(check_key, &shactx); > + > + /* mac = HMAC_SHA1_k(unencrypted key) */ > + HMAC_SHA1_Init(&hmacctx, check_key, sizeof(check_key)); > + HMAC_SHA1_Update(&hmacctx, key, key_size); > + HMAC_SHA1_Final(check_digest, &hmacctx); > + > + explicit_bzero(check_key, sizeof(check_key)); > + explicit_bzero(&hmacctx, sizeof(hmacctx)); > + explicit_bzero(&shactx, sizeof(shactx)); > +} > + > +int > +sr_crypto_decrypt_key(struct sr_discipline *sd) > +{ > + u_char check_digest[SHA1_DIGEST_LENGTH]; > + int rv = 1; > + > + DNPRINTF(SR_D_DIS, "%s: sr_crypto_decrypt_key\n", DEVNAME(sd->sd_sc)); > + > + if (sd->mds.mdd_crypto.scr_meta->scm_check_alg != SR_CRYPTOC_HMAC_SHA1) > + goto out; > + > + if (sr_crypto_decrypt((u_char *)sd->mds.mdd_crypto.scr_meta->scm_key, > + (u_char *)sd->mds.mdd_crypto.scr_key, > + sd->mds.mdd_crypto.scr_maskkey, sizeof(sd->mds.mdd_crypto.scr_key), > + sd->mds.mdd_crypto.scr_meta->scm_mask_alg) == -1) > + goto out; > + > +#ifdef SR_DEBUG0 > + sr_crypto_dumpkeys(sd); > +#endif > + > + /* Check that the key decrypted properly. */ > + sr_crypto_calculate_check_hmac_sha1(sd->mds.mdd_crypto.scr_maskkey, > + sizeof(sd->mds.mdd_crypto.scr_maskkey), > + (u_int8_t *)sd->mds.mdd_crypto.scr_key, > + sizeof(sd->mds.mdd_crypto.scr_key), > + check_digest); > + if (memcmp(sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac, > + check_digest, sizeof(check_digest)) != 0) { > + explicit_bzero(sd->mds.mdd_crypto.scr_key, > + sizeof(sd->mds.mdd_crypto.scr_key)); > + goto out; > + } > + > + rv = 0; /* Success */ > +out: > + /* we don't need the mask key anymore */ > + explicit_bzero(&sd->mds.mdd_crypto.scr_maskkey, > + sizeof(sd->mds.mdd_crypto.scr_maskkey)); > + > + explicit_bzero(check_digest, sizeof(check_digest)); > + > + return rv; > +} > + > +int > +sr_crypto_create_keys(struct sr_discipline *sd) > +{ > + > + DNPRINTF(SR_D_DIS, "%s: sr_crypto_create_keys\n", > + DEVNAME(sd->sd_sc)); > + > + if (AES_MAXKEYBYTES < sizeof(sd->mds.mdd_crypto.scr_maskkey)) > + return (1); > + > + /* XXX allow user to specify */ > + sd->mds.mdd_crypto.scr_meta->scm_alg = SR_CRYPTOA_AES_XTS_256; > + > + /* generate crypto keys */ > + arc4random_buf(sd->mds.mdd_crypto.scr_key, > + sizeof(sd->mds.mdd_crypto.scr_key)); > + > + /* Mask the disk keys. */ > + sd->mds.mdd_crypto.scr_meta->scm_mask_alg = SR_CRYPTOM_AES_ECB_256; > + sr_crypto_encrypt((u_char *)sd->mds.mdd_crypto.scr_key, > + (u_char *)sd->mds.mdd_crypto.scr_meta->scm_key, > + sd->mds.mdd_crypto.scr_maskkey, sizeof(sd->mds.mdd_crypto.scr_key), > + sd->mds.mdd_crypto.scr_meta->scm_mask_alg); > + > + /* Prepare key decryption check code. */ > + sd->mds.mdd_crypto.scr_meta->scm_check_alg = SR_CRYPTOC_HMAC_SHA1; > + sr_crypto_calculate_check_hmac_sha1(sd->mds.mdd_crypto.scr_maskkey, > + sizeof(sd->mds.mdd_crypto.scr_maskkey), > + (u_int8_t *)sd->mds.mdd_crypto.scr_key, > + sizeof(sd->mds.mdd_crypto.scr_key), > + sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac); > + > + /* Erase the plaintext disk keys */ > + explicit_bzero(sd->mds.mdd_crypto.scr_key, > + sizeof(sd->mds.mdd_crypto.scr_key)); > + > +#ifdef SR_DEBUG0 > + sr_crypto_dumpkeys(sd); > +#endif > + > + sd->mds.mdd_crypto.scr_meta->scm_flags = SR_CRYPTOF_KEY | > + SR_CRYPTOF_KDFHINT; > + > + return (0); > +} > + > +int > +sr_crypto_change_maskkey(struct sr_discipline *sd, > + struct sr_crypto_kdfinfo *kdfinfo1, struct sr_crypto_kdfinfo *kdfinfo2) > +{ > + u_char check_digest[SHA1_DIGEST_LENGTH]; > + u_char *c, *p = NULL; > + size_t ksz; > + int rv = 1; > + > + DNPRINTF(SR_D_DIS, "%s: sr_crypto_change_maskkey\n", > + DEVNAME(sd->sd_sc)); > + > + if (sd->mds.mdd_crypto.scr_meta->scm_check_alg != SR_CRYPTOC_HMAC_SHA1) > + goto out; > + > + c = (u_char *)sd->mds.mdd_crypto.scr_meta->scm_key; > + ksz = sizeof(sd->mds.mdd_crypto.scr_key); > + p = malloc(ksz, M_DEVBUF, M_WAITOK | M_ZERO); > + if (p == NULL) > + goto out; > + > + if (sr_crypto_decrypt(c, p, kdfinfo1->maskkey, ksz, > + sd->mds.mdd_crypto.scr_meta->scm_mask_alg) == -1) > + goto out; > + > +#ifdef SR_DEBUG0 > + sr_crypto_dumpkeys(sd); > +#endif > + > + sr_crypto_calculate_check_hmac_sha1(kdfinfo1->maskkey, > + sizeof(kdfinfo1->maskkey), p, ksz, check_digest); > + if (memcmp(sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac, > + check_digest, sizeof(check_digest)) != 0) { > + rv = EPERM; > + goto out; > + } > + > + /* Mask the disk keys. */ > + c = (u_char *)sd->mds.mdd_crypto.scr_meta->scm_key; > + if (sr_crypto_encrypt(p, c, kdfinfo2->maskkey, ksz, > + sd->mds.mdd_crypto.scr_meta->scm_mask_alg) == -1) > + goto out; > + > + /* Prepare key decryption check code. */ > + sd->mds.mdd_crypto.scr_meta->scm_check_alg = SR_CRYPTOC_HMAC_SHA1; > + sr_crypto_calculate_check_hmac_sha1(kdfinfo2->maskkey, > + sizeof(kdfinfo2->maskkey), (u_int8_t *)sd->mds.mdd_crypto.scr_key, > + sizeof(sd->mds.mdd_crypto.scr_key), check_digest); > + > + /* Copy new encrypted key and HMAC to metadata. */ > + bcopy(check_digest, sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac, > + sizeof(sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac)); > + > + rv = 0; /* Success */ > + > +out: > + if (p) { > + explicit_bzero(p, ksz); > + free(p, M_DEVBUF); > + } > + > + explicit_bzero(check_digest, sizeof(check_digest)); > + explicit_bzero(&kdfinfo1->maskkey, sizeof(kdfinfo1->maskkey)); > + explicit_bzero(&kdfinfo2->maskkey, sizeof(kdfinfo2->maskkey)); > + > + return (rv); > +} > + > +struct sr_chunk * > +sr_crypto_create_key_disk(struct sr_discipline *sd, dev_t dev) > +{ > + struct sr_softc *sc = sd->sd_sc; > + struct sr_discipline *fakesd = NULL; > + struct sr_metadata *sm = NULL; > + struct sr_meta_chunk *km; > + struct sr_meta_opt_item *omi = NULL; > + struct sr_chunk *key_disk = NULL; > + struct disklabel label; > + struct vnode *vn; > + char devname[32]; > + int c, part, open = 0; > + > + /* > + * Create a metadata structure on the key disk and store > + * keying material in the optional metadata. > + */ > + > + sr_meta_getdevname(sc, dev, devname, sizeof(devname)); > + > + /* Make sure chunk is not already in use. */ > + c = sr_chunk_in_use(sc, dev); > + if (c != BIOC_SDINVALID && c != BIOC_SDOFFLINE) { > + printf("%s: %s is already in use\n", DEVNAME(sc), devname); > + goto done; > + } > + > + /* Open device. */ > + if (bdevvp(dev, &vn)) { > + printf("%s:, sr_create_key_disk: can't allocate vnode\n", > + DEVNAME(sc)); > + goto done; > + } > + if (VOP_OPEN(vn, FREAD | FWRITE, NOCRED, curproc)) { > + DNPRINTF(SR_D_META,"%s: sr_create_key_disk cannot open %s\n", > + DEVNAME(sc), devname); > + vput(vn); > + goto fail; > + } > + open = 1; /* close dev on error */ > + > + /* Get partition details. */ > + part = DISKPART(dev); > + if (VOP_IOCTL(vn, DIOCGDINFO, (caddr_t)&label, > + FREAD, NOCRED, curproc)) { > + DNPRINTF(SR_D_META, "%s: sr_create_key_disk ioctl failed\n", > + DEVNAME(sc)); > + VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc); > + vput(vn); > + goto fail; > + } > + if (label.d_partitions[part].p_fstype != FS_RAID) { > + printf("%s: %s partition not of type RAID (%d)\n", > + DEVNAME(sc), devname, > + label.d_partitions[part].p_fstype); > + goto fail; > + } > + > + /* > + * Create and populate chunk metadata. > + */ > + > + key_disk = malloc(sizeof(struct sr_chunk), M_DEVBUF, M_WAITOK | M_ZERO); > + km = &key_disk->src_meta; > + > + key_disk->src_dev_mm = dev; > + key_disk->src_vn = vn; > + strlcpy(key_disk->src_devname, devname, sizeof(km->scmi.scm_devname)); > + key_disk->src_size = 0; > + > + km->scmi.scm_volid = sd->sd_meta->ssdi.ssd_level; > + km->scmi.scm_chunk_id = 0; > + km->scmi.scm_size = 0; > + km->scmi.scm_coerced_size = 0; > + strlcpy(km->scmi.scm_devname, devname, sizeof(km->scmi.scm_devname)); > + bcopy(&sd->sd_meta->ssdi.ssd_uuid, &km->scmi.scm_uuid, > + sizeof(struct sr_uuid)); > + > + sr_checksum(sc, km, &km->scm_checksum, > + sizeof(struct sr_meta_chunk_invariant)); > + > + km->scm_status = BIOC_SDONLINE; > + > + /* > + * Create and populate our own discipline and metadata. > + */ > + > + sm = malloc(sizeof(struct sr_metadata), M_DEVBUF, M_WAITOK | M_ZERO); > + sm->ssdi.ssd_magic = SR_MAGIC; > + sm->ssdi.ssd_version = SR_META_VERSION; > + sm->ssd_ondisk = 0; > + sm->ssdi.ssd_vol_flags = 0; > + bcopy(&sd->sd_meta->ssdi.ssd_uuid, &sm->ssdi.ssd_uuid, > + sizeof(struct sr_uuid)); > + sm->ssdi.ssd_chunk_no = 1; > + sm->ssdi.ssd_volid = SR_KEYDISK_VOLID; > + sm->ssdi.ssd_level = SR_KEYDISK_LEVEL; > + sm->ssdi.ssd_size = 0; > + strlcpy(sm->ssdi.ssd_vendor, "OPENBSD", sizeof(sm->ssdi.ssd_vendor)); > + snprintf(sm->ssdi.ssd_product, sizeof(sm->ssdi.ssd_product), > + "SR %s", "KEYDISK"); > + snprintf(sm->ssdi.ssd_revision, sizeof(sm->ssdi.ssd_revision), > + "%03d", SR_META_VERSION); > + > + fakesd = malloc(sizeof(struct sr_discipline), M_DEVBUF, > + M_WAITOK | M_ZERO); > + fakesd->sd_sc = sd->sd_sc; > + fakesd->sd_meta = sm; > + fakesd->sd_meta_type = SR_META_F_NATIVE; > + fakesd->sd_vol_status = BIOC_SVONLINE; > + strlcpy(fakesd->sd_name, "KEYDISK", sizeof(fakesd->sd_name)); > + SLIST_INIT(&fakesd->sd_meta_opt); > + > + /* Add chunk to volume. */ > + fakesd->sd_vol.sv_chunks = malloc(sizeof(struct sr_chunk *), M_DEVBUF, > + M_WAITOK | M_ZERO); > + fakesd->sd_vol.sv_chunks[0] = key_disk; > + SLIST_INIT(&fakesd->sd_vol.sv_chunk_list); > + SLIST_INSERT_HEAD(&fakesd->sd_vol.sv_chunk_list, key_disk, src_link); > + > + /* Generate mask key. */ > + arc4random_buf(sd->mds.mdd_crypto.scr_maskkey, > + sizeof(sd->mds.mdd_crypto.scr_maskkey)); > + > + /* Copy mask key to optional metadata area. */ > + sm->ssdi.ssd_opt_no = 1; > + omi = malloc(sizeof(struct sr_meta_opt_item), M_DEVBUF, > + M_WAITOK | M_ZERO); > + omi->omi_om.somi.som_type = SR_OPT_KEYDISK; > + bcopy(sd->mds.mdd_crypto.scr_maskkey, > + omi->omi_om.somi.som_meta.smm_keydisk.skm_maskkey, > + sizeof(omi->omi_om.somi.som_meta.smm_keydisk.skm_maskkey)); > + SLIST_INSERT_HEAD(&fakesd->sd_meta_opt, omi, omi_link); > + > + /* Save metadata. */ > + if (sr_meta_save(fakesd, SR_META_DIRTY)) { > + printf("%s: could not save metadata to %s\n", > + DEVNAME(sc), devname); > + goto fail; > + } > + > + goto done; > + > +fail: > + if (key_disk) > + free(key_disk, M_DEVBUF); > + key_disk = NULL; > + > +done: > + if (omi) > + free(omi, M_DEVBUF); > + if (fakesd && fakesd->sd_vol.sv_chunks) > + free(fakesd->sd_vol.sv_chunks, M_DEVBUF); > + if (fakesd) > + free(fakesd, M_DEVBUF); > + if (sm) > + free(sm, M_DEVBUF); > + if (open) { > + VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc); > + vput(vn); > + } > + > + return key_disk; > +} > + > +struct sr_chunk * > +sr_crypto_read_key_disk(struct sr_discipline *sd, dev_t dev) > +{ > + struct sr_softc *sc = sd->sd_sc; > + struct sr_metadata *sm = NULL; > + struct sr_meta_opt *om; > + struct sr_chunk *key_disk = NULL; > + struct disklabel label; > + struct vnode *vn = NULL; > + char devname[32]; > + int c, part, open = 0; > + > + /* > + * Load a key disk and load keying material into memory. > + */ > + > + sr_meta_getdevname(sc, dev, devname, sizeof(devname)); > + > + /* Make sure chunk is not already in use. */ > + c = sr_chunk_in_use(sc, dev); > + if (c != BIOC_SDINVALID && c != BIOC_SDOFFLINE) { > + printf("%s: %s is already in use\n", DEVNAME(sc), devname); > + goto done; > + } > + > + /* Open device. */ > + if (bdevvp(dev, &vn)) { > + printf("%s:, sr_read_key_disk: can't allocate vnode\n", > + DEVNAME(sc)); > + goto done; > + } > + if (VOP_OPEN(vn, FREAD | FWRITE, NOCRED, curproc)) { > + DNPRINTF(SR_D_META,"%s: sr_read_key_disk cannot open %s\n", > + DEVNAME(sc), devname); > + vput(vn); > + goto done; > + } > + open = 1; /* close dev on error */ > + > + /* Get partition details. */ > + part = DISKPART(dev); > + if (VOP_IOCTL(vn, DIOCGDINFO, (caddr_t)&label, FREAD, > + NOCRED, curproc)) { > + DNPRINTF(SR_D_META, "%s: sr_read_key_disk ioctl failed\n", > + DEVNAME(sc)); > + VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc); > + vput(vn); > + goto done; > + } > + if (label.d_partitions[part].p_fstype != FS_RAID) { > + printf("%s: %s partition not of type RAID (%d)\n", > + DEVNAME(sc), devname, > + label.d_partitions[part].p_fstype); > + goto done; > + } > + > + /* > + * Read and validate key disk metadata. > + */ > + sm = malloc(SR_META_SIZE * 512, M_DEVBUF, M_WAITOK | M_ZERO); > + if (sr_meta_native_read(sd, dev, sm, NULL)) { > + printf("%s: native bootprobe could not read native " > + "metadata\n", DEVNAME(sc)); > + goto done; > + } > + > + if (sr_meta_validate(sd, dev, sm, NULL)) { > + DNPRINTF(SR_D_META, "%s: invalid metadata\n", > + DEVNAME(sc)); > + goto done; > + } > + > + /* Make sure this is a key disk. */ > + if (sm->ssdi.ssd_level != SR_KEYDISK_LEVEL) { > + printf("%s: %s is not a key disk\n", DEVNAME(sc), devname); > + goto done; > + } > + > + /* Construct key disk chunk. */ > + key_disk = malloc(sizeof(struct sr_chunk), M_DEVBUF, M_WAITOK | M_ZERO); > + key_disk->src_dev_mm = dev; > + key_disk->src_vn = vn; > + key_disk->src_size = 0; > + > + bcopy((struct sr_meta_chunk *)(sm + 1), &key_disk->src_meta, > + sizeof(key_disk->src_meta)); > + > + /* Read mask key from optional metadata. */ > + om = (struct sr_meta_opt *)((u_int8_t *)(sm + 1) + > + sizeof(struct sr_meta_chunk) * sm->ssdi.ssd_chunk_no); > + for (c = 0; c < sm->ssdi.ssd_opt_no; c++) { > + if (om->somi.som_type == SR_OPT_KEYDISK) { > + bcopy(&om->somi.som_meta.smm_keydisk.skm_maskkey, > + sd->mds.mdd_crypto.scr_maskkey, > + sizeof(sd->mds.mdd_crypto.scr_maskkey)); > + break; > + } else if (om->somi.som_type == SR_OPT_CRYPTO) { > + bcopy(&om->somi.som_meta.smm_crypto, > + sd->mds.mdd_crypto.scr_maskkey, > + sizeof(sd->mds.mdd_crypto.scr_maskkey)); > + break; > + } > + om++; > + } > + > + open = 0; > + > +done: > + if (sm) > + free(sm, M_DEVBUF); > + > + if (vn && open) { > + VOP_CLOSE(vn, FREAD, NOCRED, curproc); > + vput(vn); > + } > + > + return key_disk; > +} > + > +int > +sr_crypto_alloc_resources(struct sr_discipline *sd) > +{ > + struct cryptoini cri; > + u_int num_keys, i; > + > + if (!sd) > + return (EINVAL); > + > + DNPRINTF(SR_D_DIS, "%s: sr_crypto_alloc_resources\n", > + DEVNAME(sd->sd_sc)); > + > + pool_init(&sd->mds.mdd_crypto.sr_uiopl, sizeof(struct uio), 0, 0, 0, > + "sr_uiopl", NULL); > + pool_init(&sd->mds.mdd_crypto.sr_iovpl, sizeof(struct iovec), 0, 0, 0, > + "sr_iovpl", NULL); > + > + for (i = 0; i < SR_CRYPTO_MAXKEYS; i++) > + sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1; > + > + if (sr_wu_alloc(sd)) > + return (ENOMEM); > + if (sr_ccb_alloc(sd)) > + return (ENOMEM); > + if (sr_crypto_decrypt_key(sd)) > + return (EPERM); > + > + bzero(&cri, sizeof(cri)); > + cri.cri_alg = CRYPTO_AES_XTS; > + switch (sd->mds.mdd_crypto.scr_meta->scm_alg) { > + case SR_CRYPTOA_AES_XTS_128: > + cri.cri_klen = 256; > + break; > + case SR_CRYPTOA_AES_XTS_256: > + cri.cri_klen = 512; > + break; > + default: > + return (EINVAL); > + } > + > + /* Allocate a session for every 2^SR_CRYPTO_KEY_BLKSHIFT blocks */ > + num_keys = sd->sd_meta->ssdi.ssd_size >> SR_CRYPTO_KEY_BLKSHIFT; > + if (num_keys >= SR_CRYPTO_MAXKEYS) > + return (EFBIG); > + for (i = 0; i <= num_keys; i++) { > + cri.cri_key = sd->mds.mdd_crypto.scr_key[i]; > + if (crypto_newsession(&sd->mds.mdd_crypto.scr_sid[i], > + &cri, 0) != 0) { > + for (i = 0; > + sd->mds.mdd_crypto.scr_sid[i] != (u_int64_t)-1; > + i++) { > + crypto_freesession( > + sd->mds.mdd_crypto.scr_sid[i]); > + sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1; > + } > + return (EINVAL); > + } > + } > + > + sr_hotplug_register(sd, sr_crypto_hotplug); > + > + return (0); > +} > + > +int > +sr_crypto_free_resources(struct sr_discipline *sd) > +{ > + int rv = EINVAL; > + u_int i; > + > + if (!sd) > + return (rv); > + > + DNPRINTF(SR_D_DIS, "%s: sr_crypto_free_resources\n", > + DEVNAME(sd->sd_sc)); > + > + if (sd->mds.mdd_crypto.key_disk != NULL) { > + explicit_bzero(sd->mds.mdd_crypto.key_disk, sizeof > + sd->mds.mdd_crypto.key_disk); > + free(sd->mds.mdd_crypto.key_disk, M_DEVBUF); > + } > + > + sr_hotplug_unregister(sd, sr_crypto_hotplug); > + > + for (i = 0; sd->mds.mdd_crypto.scr_sid[i] != (u_int64_t)-1; i++) { > + crypto_freesession(sd->mds.mdd_crypto.scr_sid[i]); > + sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1; > + } > + > + sr_wu_free(sd); > + sr_ccb_free(sd); > + > + if (sd->mds.mdd_crypto.sr_uiopl.pr_serial != 0) > + pool_destroy(&sd->mds.mdd_crypto.sr_uiopl); > + if (sd->mds.mdd_crypto.sr_iovpl.pr_serial != 0) > + pool_destroy(&sd->mds.mdd_crypto.sr_iovpl); > + > + rv = 0; > + return (rv); > +} > + > +int > +sr_crypto_ioctl(struct sr_discipline *sd, struct bioc_discipline *bd) > +{ > + struct sr_crypto_kdfpair kdfpair; > + struct sr_crypto_kdfinfo kdfinfo1, kdfinfo2; > + int size, rv = 1; > + > + DNPRINTF(SR_D_IOCTL, "%s: sr_crypto_ioctl %u\n", > + DEVNAME(sd->sd_sc), bd->bd_cmd); > + > + switch (bd->bd_cmd) { > + case SR_IOCTL_GET_KDFHINT: > + > + /* Get KDF hint for userland. */ > + size = sizeof(sd->mds.mdd_crypto.scr_meta->scm_kdfhint); > + if (bd->bd_data == NULL || bd->bd_size > size) > + goto bad; > + if (copyout(sd->mds.mdd_crypto.scr_meta->scm_kdfhint, > + bd->bd_data, bd->bd_size)) > + goto bad; > + > + rv = 0; > + > + break; > + > + case SR_IOCTL_CHANGE_PASSPHRASE: > + > + /* Attempt to change passphrase. */ > + > + size = sizeof(kdfpair); > + if (bd->bd_data == NULL || bd->bd_size > size) > + goto bad; > + if (copyin(bd->bd_data, &kdfpair, size)) > + goto bad; > + > + size = sizeof(kdfinfo1); > + if (kdfpair.kdfinfo1 == NULL || kdfpair.kdfsize1 > size) > + goto bad; > + if (copyin(kdfpair.kdfinfo1, &kdfinfo1, size)) > + goto bad; > + > + size = sizeof(kdfinfo2); > + if (kdfpair.kdfinfo2 == NULL || kdfpair.kdfsize2 > size) > + goto bad; > + if (copyin(kdfpair.kdfinfo2, &kdfinfo2, size)) > + goto bad; > + > + if (sr_crypto_change_maskkey(sd, &kdfinfo1, &kdfinfo2)) > + goto bad; > + > + /* Save metadata to disk. */ > + rv = sr_meta_save(sd, SR_META_DIRTY); > + > + break; > + } > + > +bad: > + explicit_bzero(&kdfpair, sizeof(kdfpair)); > + explicit_bzero(&kdfinfo1, sizeof(kdfinfo1)); > + explicit_bzero(&kdfinfo2, sizeof(&kdfinfo2)); > + return (rv); > +} > + > +int > +sr_crypto_meta_opt_load(struct sr_discipline *sd, struct sr_meta_opt *om) > +{ > + int rv = EINVAL; > + > + if (om->somi.som_type == SR_OPT_CRYPTO) { > + sd->mds.mdd_crypto.scr_meta = &om->somi.som_meta.smm_crypto; > + rv = 0; > + } > + > + return (rv); > +} > + > +int > +sr_crypto_rw(struct sr_workunit *wu) > +{ > + struct cryptop *crp; > + int s, rv = 0; > + > + DNPRINTF(SR_D_DIS, "%s: sr_crypto_rw wu: %p\n", > + DEVNAME(wu->swu_dis->sd_sc), wu); > + > + if (wu->swu_xs->flags & SCSI_DATA_OUT) { > + /* vndencrypt if key */ > + crp = sr_crypto_getcryptop(wu, 1); > + if (crp == NULL) > + return (1); > + crp->crp_callback = sr_crypto_write; > + crp->crp_opaque = wu; > + s = splvm(); > + if (crypto_invoke(crp)) > + rv = 1; > + else > + rv = crp->crp_etype; > + splx(s); > + } else > + rv = sr_crypto_rw2(wu, NULL); > + > + return (rv); > +} > + > +int > +sr_crypto_write(struct cryptop *crp) > +{ > + int s; > + struct sr_workunit *wu = crp->crp_opaque; > + > + DNPRINTF(SR_D_INTR, "%s: sr_crypto_write: wu %x xs: %x\n", > + DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs); > + > + if (crp->crp_etype) { > + /* fail io */ > + ((struct sr_workunit *)(crp->crp_opaque))->swu_xs->error = > + XS_DRIVER_STUFFUP; > + s = splbio(); > + sr_crypto_finish_io(crp->crp_opaque); > + splx(s); > + } > + > + return (sr_crypto_rw2(wu, crp)); > +} > + > +int > +sr_crypto_rw2(struct sr_workunit *wu, struct cryptop *crp) > +{ > + struct sr_discipline *sd = wu->swu_dis; > + struct scsi_xfer *xs = wu->swu_xs; > + struct sr_ccb *ccb; > + struct uio *uio; > + int s; > + daddr64_t blk; > + > + if (sr_validate_io(wu, &blk, "sr_crypto_rw2")) > + goto bad; > + > + blk += sd->sd_meta->ssd_data_offset; > + > + wu->swu_io_count = 1; > + > + ccb = sr_ccb_get(sd); > + if (!ccb) { > + /* should never happen but handle more gracefully */ > + printf("%s: %s: too many ccbs queued\n", > + DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname); > + goto bad; > + } > + > + ccb->ccb_buf.b_flags = B_CALL | B_PHYS; > + ccb->ccb_buf.b_iodone = sr_crypto_intr; > + ccb->ccb_buf.b_blkno = blk; > + ccb->ccb_buf.b_bcount = xs->datalen; > + ccb->ccb_buf.b_bufsize = xs->datalen; > + ccb->ccb_buf.b_resid = xs->datalen; > + > + if (xs->flags & SCSI_DATA_IN) { > + ccb->ccb_buf.b_flags |= B_READ; > + ccb->ccb_buf.b_data = xs->data; > + } else { > + uio = crp->crp_buf; > + ccb->ccb_buf.b_flags |= B_WRITE; > + ccb->ccb_buf.b_data = uio->uio_iov->iov_base; > + ccb->ccb_opaque = crp; > + } > + > + ccb->ccb_buf.b_error = 0; > + ccb->ccb_buf.b_proc = curproc; > + ccb->ccb_wu = wu; > + ccb->ccb_target = 0; > + ccb->ccb_buf.b_dev = sd->sd_vol.sv_chunks[0]->src_dev_mm; > + ccb->ccb_buf.b_vp = sd->sd_vol.sv_chunks[0]->src_vn; > + if ((ccb->ccb_buf.b_flags & B_READ) == 0) > + ccb->ccb_buf.b_vp->v_numoutput++; > + > + LIST_INIT(&ccb->ccb_buf.b_dep); > + > + if (wu->swu_cb_active == 1) > + panic("%s: sr_crypto_rw2", DEVNAME(sd->sd_sc)); > + TAILQ_INSERT_TAIL(&wu->swu_ccb, ccb, ccb_link); > + > + DNPRINTF(SR_D_DIS, "%s: %s: sr_crypto_rw2: b_bcount: %d " > + "b_blkno: %x b_flags 0x%0x b_data %p\n", > + DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname, > + ccb->ccb_buf.b_bcount, ccb->ccb_buf.b_blkno, > + ccb->ccb_buf.b_flags, ccb->ccb_buf.b_data); > + > + s = splbio(); > + > + if (sr_check_io_collision(wu)) > + goto queued; > + > + sr_raid_startwu(wu); > + > +queued: > + splx(s); > + return (0); > +bad: > + /* wu is unwound by sr_wu_put */ > + if (crp) > + crp->crp_etype = EINVAL; > + return (1); > +} > + > +void > +sr_crypto_intr(struct buf *bp) > +{ > + struct sr_ccb *ccb = (struct sr_ccb *)bp; > + struct sr_workunit *wu = ccb->ccb_wu, *wup; > + struct sr_discipline *sd = wu->swu_dis; > + struct scsi_xfer *xs = wu->swu_xs; > + struct sr_softc *sc = sd->sd_sc; > + struct cryptop *crp; > + int s, s2, pend; > + > + DNPRINTF(SR_D_INTR, "%s: sr_crypto_intr bp: %x xs: %x\n", > + DEVNAME(sc), bp, wu->swu_xs); > + > + DNPRINTF(SR_D_INTR, "%s: sr_crypto_intr: b_bcount: %d b_resid: %d" > + " b_flags: 0x%0x\n", DEVNAME(sc), ccb->ccb_buf.b_bcount, > + ccb->ccb_buf.b_resid, ccb->ccb_buf.b_flags); > + > + s = splbio(); > + > + if (ccb->ccb_buf.b_flags & B_ERROR) { > + printf("%s: i/o error on block %lld\n", DEVNAME(sc), > + ccb->ccb_buf.b_blkno); > + wu->swu_ios_failed++; > + ccb->ccb_state = SR_CCB_FAILED; > + if (ccb->ccb_target != -1) > + sd->sd_set_chunk_state(sd, ccb->ccb_target, > + BIOC_SDOFFLINE); > + else > + panic("%s: invalid target on wu: %p", DEVNAME(sc), wu); > + } else { > + ccb->ccb_state = SR_CCB_OK; > + wu->swu_ios_succeeded++; > + } > + wu->swu_ios_complete++; > + > + DNPRINTF(SR_D_INTR, "%s: sr_crypto_intr: comp: %d count: %d\n", > + DEVNAME(sc), wu->swu_ios_complete, wu->swu_io_count); > + > + if (wu->swu_ios_complete == wu->swu_io_count) { > + if (wu->swu_ios_failed == wu->swu_ios_complete) > + xs->error = XS_DRIVER_STUFFUP; > + else > + xs->error = XS_NOERROR; > + > + pend = 0; > + TAILQ_FOREACH(wup, &sd->sd_wu_pendq, swu_link) { > + if (wu == wup) { > + TAILQ_REMOVE(&sd->sd_wu_pendq, wu, swu_link); > + pend = 1; > + > + if (wu->swu_collider) { > + wu->swu_collider->swu_state = > + SR_WU_INPROGRESS; > + TAILQ_REMOVE(&sd->sd_wu_defq, > + wu->swu_collider, swu_link); > + sr_raid_startwu(wu->swu_collider); > + } > + break; > + } > + } > + > + if (!pend) > + printf("%s: wu: %p not on pending queue\n", > + DEVNAME(sc), wu); > + > + if ((xs->flags & SCSI_DATA_IN) && (xs->error == XS_NOERROR)) { > + crp = sr_crypto_getcryptop(wu, 0); > + if (crp == NULL) > + panic("sr_crypto_intr: no crypto op"); > + ccb->ccb_opaque = crp; > + crp->crp_callback = sr_crypto_read; > + crp->crp_opaque = wu; > + DNPRINTF(SR_D_INTR, "%s: sr_crypto_intr: crypto_invoke " > + "%p\n", DEVNAME(sc), crp); > + s2 = splvm(); > + crypto_invoke(crp); > + splx(s2); > + goto done; > + } > + > + sr_crypto_finish_io(wu); > + } > + > +done: > + splx(s); > +} > + > +void > +sr_crypto_finish_io(struct sr_workunit *wu) > +{ > + struct sr_discipline *sd = wu->swu_dis; > + struct scsi_xfer *xs = wu->swu_xs; > + struct sr_ccb *ccb; > +#ifdef SR_DEBUG > + struct sr_softc *sc = sd->sd_sc; > +#endif /* SR_DEBUG */ > + > + splassert(IPL_BIO); > + > + DNPRINTF(SR_D_INTR, "%s: sr_crypto_finish_io: wu %x xs: %x\n", > + DEVNAME(sc), wu, xs); > + > + xs->resid = 0; > + > + if (wu->swu_cb_active == 1) > + panic("%s: sr_crypto_finish_io", DEVNAME(sd->sd_sc)); > + TAILQ_FOREACH(ccb, &wu->swu_ccb, ccb_link) { > + if (ccb->ccb_opaque == NULL) > + continue; > + sr_crypto_putcryptop(ccb->ccb_opaque); > + } > + > + sr_scsi_done(sd, xs); > + > + if (sd->sd_sync && sd->sd_wu_pending == 0) > + wakeup(sd); > +} > + > +int > +sr_crypto_read(struct cryptop *crp) > +{ > + int s; > + struct sr_workunit *wu = crp->crp_opaque; > + > + DNPRINTF(SR_D_INTR, "%s: sr_crypto_read: wu %x xs: %x\n", > + DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs); > + > + if (crp->crp_etype) > + wu->swu_xs->error = XS_DRIVER_STUFFUP; > + > + s = splbio(); > + sr_crypto_finish_io(wu); > + splx(s); > + > + return (0); > +} > + > +void > +sr_crypto_hotplug(struct sr_discipline *sd, struct disk *diskp, int action) > +{ > + DNPRINTF(SR_D_MISC, "%s: sr_crypto_hotplug: %s %d\n", > + DEVNAME(sd->sd_sc), diskp->dk_name, action); > +} > diff --git dev/softraidvar.h dev/softraidvar.h > index 6c21b5e..a925199 100644 > --- dev/softraidvar.h > +++ dev/softraidvar.h > @@ -383,12 +383,11 @@ struct sr_raid6 { > /* CRYPTO */ > #define SR_CRYPTO_NOWU 16 > struct sr_crypto { > + struct mutex scr_mutex; > + TAILQ_HEAD(,sr_crypto_wu) scr_wus; > struct sr_meta_crypto *scr_meta; > struct sr_chunk *key_disk; > > - struct pool sr_uiopl; > - struct pool sr_iovpl; > - > /* XXX only keep scr_sid over time */ > u_int8_t scr_key[SR_CRYPTO_MAXKEYS][SR_CRYPTO_KEYBYTES]; > u_int8_t scr_maskkey[SR_CRYPTO_MAXKEYBYTES]; > -- > 1.7.5 > > > -- > Do not try to solve all life's problems at once -- learn to dread each > day as it comes. > -- Donald Kaul