Module Name: src
Committed By: riastradh
Date: Sat Jun 13 18:58:26 UTC 2020
Modified Files:
src/sys/arch/arm/sunxi: sun8i_crypto.c
Log Message:
Draft opencrypto support for Allwinner Crypto Engine.
XXX Can't handle nonzero crd_skip yet.
To generate a diff of this commit:
cvs rdiff -u -r1.16 -r1.17 src/sys/arch/arm/sunxi/sun8i_crypto.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/arm/sunxi/sun8i_crypto.c
diff -u src/sys/arch/arm/sunxi/sun8i_crypto.c:1.16 src/sys/arch/arm/sunxi/sun8i_crypto.c:1.17
--- src/sys/arch/arm/sunxi/sun8i_crypto.c:1.16 Sat Jun 13 18:57:54 2020
+++ src/sys/arch/arm/sunxi/sun8i_crypto.c Sat Jun 13 18:58:26 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: sun8i_crypto.c,v 1.16 2020/06/13 18:57:54 riastradh Exp $ */
+/* $NetBSD: sun8i_crypto.c,v 1.17 2020/06/13 18:58:26 riastradh Exp $ */
/*-
* Copyright (c) 2019 The NetBSD Foundation, Inc.
@@ -43,7 +43,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.16 2020/06/13 18:57:54 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.17 2020/06/13 18:58:26 riastradh Exp $");
#include <sys/types.h>
#include <sys/param.h>
@@ -51,16 +51,21 @@ __KERNEL_RCSID(1, "$NetBSD: sun8i_crypto
#include <sys/bus.h>
#include <sys/callout.h>
#include <sys/conf.h>
+#include <sys/cprng.h>
#include <sys/device.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
+#include <sys/mbuf.h>
#include <sys/mutex.h>
#include <sys/rndsource.h>
+#include <sys/sdt.h>
#include <sys/sysctl.h>
#include <sys/workqueue.h>
#include <dev/fdt/fdtvar.h>
+#include <opencrypto/cryptodev.h>
+
#include <arm/sunxi/sun8i_crypto.h>
#define SUN8I_CRYPTO_TIMEOUT hz
@@ -110,15 +115,22 @@ struct sun8i_crypto_softc {
const struct sysctlnode *cy_root_node;
const struct sysctlnode *cy_trng_node;
} sc_sysctl;
+ struct sun8i_crypto_opencrypto {
+ uint32_t co_driverid;
+ } sc_opencrypto;
};
struct sun8i_crypto_task {
struct sun8i_crypto_buf ct_descbuf;
struct sun8i_crypto_taskdesc *ct_desc;
+ struct sun8i_crypto_buf ct_ivbuf;
+ void *ct_iv;
+ struct sun8i_crypto_buf ct_ctrbuf;
+ void *ct_ctr;
bus_dmamap_t ct_descmap;
bus_dmamap_t ct_keymap;
- bus_dmamap_t ct_ivmap;
- bus_dmamap_t ct_ctrmap;
+ bus_dmamap_t ct_ivmap; /* IV input */
+ bus_dmamap_t ct_ctrmap; /* updated IV output */
bus_dmamap_t ct_srcmap;
bus_dmamap_t ct_dstmap;
uint32_t ct_nbytes;
@@ -159,8 +171,8 @@ static void sun8i_crypto_task_put(struct
static int sun8i_crypto_task_load(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, uint32_t,
uint32_t, uint32_t, uint32_t);
-static int sun8i_crypto_task_scatter(struct sun8i_crypto_adrlen *,
- bus_dmamap_t, uint32_t);
+static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *,
+ struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t);
static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, uint32_t);
@@ -196,19 +208,106 @@ static int sun8i_crypto_sysctl_rng(SYSCT
static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, void *, int);
+static void sun8i_crypto_register(struct sun8i_crypto_softc *);
+static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t);
+static int sun8i_crypto_newsession(void *, uint32_t *,
+ struct cryptoini *);
+static int sun8i_crypto_freesession(void *, uint64_t);
+static u_int sun8i_crypto_ivlen(const struct cryptodesc *);
+static int sun8i_crypto_process(void *, struct cryptop *, int);
+static void sun8i_crypto_callback(struct sun8i_crypto_softc *,
+ struct sun8i_crypto_task *, void *, int);
+
+/*
+ * Probes
+ */
+
+SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read,
+ "bus_size_t"/*reg*/,
+ "uint32_t"/*value*/);
+SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write,
+ "bus_size_t"/*reg*/,
+ "uint32_t"/*write*/);
+
+SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success,
+ "struct sun8i_crypto_task *"/*task*/);
+SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure,
+ "int"/*error*/);
+SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor,
+ "struct sun8i_crypto_task *"/*task*/);
+SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get,
+ "struct sun8i_crypto_task *"/*task*/);
+SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put,
+ "struct sun8i_crypto_task *"/*task*/);
+
+SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load,
+ "struct sun8i_crypto_task *"/*task*/,
+ "uint32_t"/*tdqc*/,
+ "uint32_t"/*tdqs*/,
+ "uint32_t"/*tdqa*/,
+ "struct sun8i_crypto_taskdesc *"/*desc*/,
+ "int"/*error*/);
+SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned,
+ "struct sun8i_crypto_task *"/*task*/,
+ "bus_addr_t"/*ds_addr*/,
+ "bus_size_t"/*ds_len*/);
+SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done,
+ "struct sun8i_crypto_task *"/*task*/,
+ "int"/*error*/);
+
+SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure,
+ "struct sun8i_crypto_softc *"/*sc*/,
+ "struct sun8i_crypto_task *"/*task*/,
+ "int"/*error*/);
+SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success,
+ "struct sun8i_crypto_softc *"/*sc*/,
+ "struct sun8i_crypto_task *"/*task*/,
+ "unsigned"/*chan*/);
+SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr,
+ "struct sun8i_crypto_softc *"/*sc*/,
+ "uint32_t"/*isr*/,
+ "uint32_t"/*esr*/);
+SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done,
+ "struct sun8i_crypto_softc *"/*sc*/,
+ "unsigned"/*chan*/,
+ "int"/*error*/);
+
+SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry,
+ "struct sun8i_crypto_softc *"/*sc*/,
+ "struct cryptop *"/*crp*/,
+ "int"/*hint*/);
+SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy,
+ "struct sun8i_crypto_softc *"/*sc*/,
+ "struct cryptop *"/*crp*/,
+ "int"/*hint*/);
+SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued,
+ "struct sun8i_crypto_softc *"/*sc*/,
+ "struct cryptop *"/*crp*/,
+ "int"/*hint*/,
+ "struct sun8i_crypto_task *"/*task*/);
+SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done,
+ "struct sun8i_crypto_softc *"/*sc*/,
+ "struct cryptop *"/*crp*/,
+ "int"/*error*/);
+
/*
* Register access
*/
static uint32_t
-sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_addr_t reg)
+sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg)
{
- return bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg);
+ uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg);
+
+ SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v);
+ return v;
}
static void
-sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_addr_t reg, uint32_t v)
+sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v)
{
+
+ SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v);
bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v);
}
@@ -259,6 +358,13 @@ sun8i_crypto_attach(device_t parent, dev
return;
}
+ /*
+ * Prime the pool with enough tasks that each channel can be
+ * busy with a task as we prepare another task for when it's
+ * done.
+ */
+ pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN);
+
/* Get and map device registers. */
if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) {
aprint_error(": couldn't get registers\n");
@@ -327,6 +433,9 @@ sun8i_crypto_attach(device_t parent, dev
/* Perform self-tests. */
config_interrupts(self, sun8i_crypto_selftest);
+
+ /* Register opencrypto handlers. */
+ sun8i_crypto_register(sc);
}
static int
@@ -344,55 +453,71 @@ sun8i_crypto_task_ctor(void *cookie, voi
goto fail0;
task->ct_desc = task->ct_descbuf.cb_kva;
+ /* Create DMA buffers for the IV and CTR. */
+ error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES,
+ &task->ct_ivbuf, dmaflags);
+ if (error)
+ goto fail1;
+ task->ct_iv = task->ct_ivbuf.cb_kva;
+ error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES,
+ &task->ct_ctrbuf, dmaflags);
+ if (error)
+ goto fail2;
+ task->ct_ctr = task->ct_ctrbuf.cb_kva;
+
/* Create a DMA map for the task descriptor and preload it. */
error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1,
sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap);
if (error)
- goto fail1;
+ goto fail3;
error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc,
sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK);
if (error)
- goto fail2;
+ goto fail4;
/* Create DMA maps for the key, IV, and CTR. */
error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1,
SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap);
if (error)
- goto fail3;
+ goto fail5;
error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1,
SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap);
if (error)
- goto fail4;
+ goto fail6;
error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1,
SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap);
if (error)
- goto fail5;
+ goto fail7;
/* Create DMA maps for the src and dst scatter/gather vectors. */
error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE,
SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags,
&task->ct_srcmap);
if (error)
- goto fail6;
+ goto fail8;
error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE,
SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags,
&task->ct_dstmap);
if (error)
- goto fail7;
+ goto fail9;
/* Success! */
+ SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task);
return 0;
-fail8: __unused
+fail10: __unused
bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap);
-fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap);
-fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap);
-fail5: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap);
-fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
-fail3: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
-fail2: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
+fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap);
+fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap);
+fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap);
+fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
+fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
+fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
+fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf);
+fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf);
fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf);
-fail0: return error;
+fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error);
+ return error;
}
static void
@@ -401,6 +526,8 @@ sun8i_crypto_task_dtor(void *cookie, voi
struct sun8i_crypto_softc *sc = cookie;
struct sun8i_crypto_task *task = vtask;
+ SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task);
+
/* XXX Zero the bounce buffers if there are any. */
bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap);
@@ -410,6 +537,8 @@ sun8i_crypto_task_dtor(void *cookie, voi
bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
+ sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf);
+ sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf);
sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf);
}
@@ -433,12 +562,14 @@ sun8i_crypto_task_get(struct sun8i_crypt
/* Allocate a task, or fail if we can't. */
task = pool_cache_get(sc->sc_taskpool, pflags);
if (task == NULL)
- return NULL;
+ goto out;
/* Set up flags and the callback. */
task->ct_flags = 0;
task->ct_callback = callback;
task->ct_cookie = cookie;
+
+out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task);
return task;
}
@@ -467,6 +598,8 @@ sun8i_crypto_task_put(struct sun8i_crypt
struct sun8i_crypto_task *task)
{
+ SDT_PROBE1(sdt, sun8i_crypto, task, put, task);
+
task->ct_cookie = task->ct_callback;
task->ct_callback = &sun8i_crypto_task_invalid;
pool_cache_put(sc->sc_taskpool, task);
@@ -480,7 +613,6 @@ sun8i_crypto_task_put(struct sun8i_crypt
* sun8i_crypto_chan_done. May fail if input is inadequately
* aligned.
*
- * XXX Teach this to fail gracefully if any alignment is wrong.
* XXX Teach this to support task chains.
*/
static int
@@ -496,6 +628,9 @@ sun8i_crypto_task_load(struct sun8i_cryp
memset(desc, 0, sizeof(*desc));
+ /* Always enable interrupt for the task. */
+ tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN;
+
desc->td_tdqc = htole32(tdqc);
desc->td_tdqs = htole32(tdqs);
desc->td_tdqa = htole32(tdqa);
@@ -519,7 +654,7 @@ sun8i_crypto_task_load(struct sun8i_cryp
KASSERT(ctrmap->dm_nsegs == 1);
desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr);
bus_dmamap_sync(sc->sc_dmat, ctrmap, 0,
- ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE);
+ ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD);
}
if (task->ct_flags & TASK_BYTES)
@@ -530,7 +665,7 @@ sun8i_crypto_task_load(struct sun8i_cryp
if (task->ct_flags & TASK_SRC) {
bus_dmamap_t srcmap = task->ct_srcmap;
KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize);
- error = sun8i_crypto_task_scatter(desc->td_src, srcmap,
+ error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap,
nbytes);
if (error)
return error;
@@ -538,36 +673,52 @@ sun8i_crypto_task_load(struct sun8i_cryp
BUS_DMASYNC_PREWRITE);
}
- error = sun8i_crypto_task_scatter(desc->td_dst, task->ct_dstmap,
+ error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap,
nbytes);
if (error)
- return error;
+ goto out;
bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes,
BUS_DMASYNC_PREREAD);
task->ct_nbytes = nbytes;
/* Success! */
- return 0;
+ error = 0;
+
+out: SDT_PROBE6(sdt, sun8i_crypto, task, load,
+ task, tdqc, tdqs, tdqa, desc, error);
+ return error;
}
/*
- * sun8i_crypto_task_scatter(adrlen, map, nbytes)
+ * sun8i_crypto_task_scatter(task, adrlen, map, nbytes)
*
* Set up a task's scatter/gather vector -- src or dst -- with the
* given DMA map for a transfer of nbytes. May fail if input is
* inadequately aligned.
*/
static int
-sun8i_crypto_task_scatter(struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map,
+sun8i_crypto_task_scatter(struct sun8i_crypto_task *task,
+ struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map,
uint32_t nbytes __diagused)
{
uint32_t total __diagused = 0;
unsigned i;
+ /*
+ * Verify that the alignment is correct and initialize the
+ * scatter/gather vector.
+ */
KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS);
for (i = 0; i < map->dm_nsegs; i++) {
- KASSERT((map->dm_segs[i].ds_addr % 4) == 0);
+ if ((map->dm_segs[i].ds_addr % 4) |
+ (map->dm_segs[i].ds_len % 4)) {
+ SDT_PROBE3(sdt, sun8i_crypto, task, misaligned,
+ task,
+ map->dm_segs[i].ds_addr,
+ map->dm_segs[i].ds_len);
+ return EINVAL;
+ }
KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX);
KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total);
adrlen[i].adr = htole32(map->dm_segs[i].ds_addr);
@@ -575,10 +726,10 @@ sun8i_crypto_task_scatter(struct sun8i_c
total += map->dm_segs[i].ds_len;
}
- /* Verify the remainder are zero. */
+ /* Set the remainder to zero. */
for (; i < SUN8I_CRYPTO_MAXSEGS; i++) {
- KASSERT(adrlen[i].adr == 0);
- KASSERT(adrlen[i].len == 0);
+ adrlen[i].adr = 0;
+ adrlen[i].len = 0;
}
/* Verify the total size matches the transfer length. */
@@ -607,7 +758,6 @@ sun8i_crypto_task_load_trng(struct sun8i
KASSERT((task->ct_flags & TASK_SRC) == 0);
/* Set up the task descriptor queue control words. */
- tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN;
tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG,
SUN8I_CRYPTO_TDQC_METHOD);
@@ -629,9 +779,9 @@ sun8i_crypto_task_load_aesecb(struct sun
KASSERT(task->ct_flags & TASK_SRC);
/* Set up the task descriptor queue control word. */
- tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN;
tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES,
SUN8I_CRYPTO_TDQC_METHOD);
+ tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR);
#ifdef DIAGNOSTIC
switch (keysize) {
@@ -749,6 +899,12 @@ sun8i_crypto_submit(struct sun8i_crypto_
/* XXX Consider polling if cold to get entropy earlier. */
out: /* Done! */
+ if (error)
+ SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure,
+ sc, task, error);
+ else
+ SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success,
+ sc, task, i);
mutex_exit(&sc->sc_lock);
return error;
}
@@ -815,6 +971,8 @@ sun8i_crypto_intr(void *cookie)
sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr);
sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr);
+ SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr);
+
/* Start the worker if necessary. */
sun8i_crypto_schedule_worker(sc);
@@ -942,6 +1100,8 @@ sun8i_crypto_chan_done(struct sun8i_cryp
KASSERT(mutex_owned(&sc->sc_lock));
+ SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error);
+
/* Claim the task if there is one; bail if not. */
if ((task = sc->sc_chan[i].cc_task) == NULL) {
device_printf(sc->sc_dev, "channel %u: no task but error=%d\n",
@@ -975,7 +1135,7 @@ sun8i_crypto_chan_done(struct sun8i_cryp
BUS_DMASYNC_POSTWRITE);
if (task->ct_flags & TASK_CTR)
bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0,
- task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);
+ task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD);
if (task->ct_flags & TASK_IV)
bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0,
task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);
@@ -986,6 +1146,7 @@ sun8i_crypto_chan_done(struct sun8i_cryp
/* Temporarily release the lock to invoke the callback. */
mutex_exit(&sc->sc_lock);
+ SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error);
(*task->ct_callback)(sc, task, task->ct_cookie, error);
mutex_enter(&sc->sc_lock);
}
@@ -1003,8 +1164,8 @@ sun8i_crypto_allocbuf(struct sun8i_crypt
int error;
/* Allocate a DMA-safe buffer. */
- error = bus_dmamem_alloc(sc->sc_dmat, size, 0, 0, buf->cb_seg,
- __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags);
+ error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0,
+ buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags);
if (error)
goto fail0;
@@ -1053,13 +1214,17 @@ sun8i_crypto_rng_attach(struct sun8i_cry
/* Preallocate a buffer to reuse. */
error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf,
BUS_DMA_WAITOK);
- if (error)
+ if (error) {
+ aprint_error_dev(self, "failed to allocate RNG buffer: %d\n",
+ error);
goto fail0;
+ }
/* Create a task to reuse. */
task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done,
rng, PR_WAITOK);
if (rng->cr_task == NULL) {
+ aprint_error_dev(self, "failed to allocate RNG task\n");
error = ENOMEM;
goto fail1;
}
@@ -1067,8 +1232,11 @@ sun8i_crypto_rng_attach(struct sun8i_cry
/* Preload the destination map. */
error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT);
- if (error)
+ if (error) {
+ aprint_error_dev(self, "failed to load RNG buffer: %d\n",
+ error);
goto fail2;
+ }
/*
* Attach the rndsource. This is _not_ marked as RND_TYPE_RNG
@@ -1088,7 +1256,7 @@ fail3: __unused
bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
fail2: sun8i_crypto_task_put(sc, task);
fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf);
-fail0: aprint_error_dev(self, "failed to set up RNG, error=%d\n", error);
+fail0: return;
}
/*
@@ -1250,8 +1418,10 @@ sun8i_crypto_selftest(device_t self)
goto fail6;
/* Set up the task descriptor. */
- sun8i_crypto_task_load_aesecb(sc, task, nbytes,
+ error = sun8i_crypto_task_load_aesecb(sc, task, nbytes,
SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC);
+ if (error)
+ goto fail7;
/* Submit! */
error = sun8i_crypto_submit(sc, task);
@@ -1532,3 +1702,717 @@ sun8i_crypto_sysctl_rng_done(struct sun8
mutex_destroy(&req->cu_lock);
kmem_free(req, sizeof(*req));
}
+
+/*
+ * sun8i_crypto_register(sc)
+ *
+ * Register opencrypto algorithms supported by the crypto engine.
+ */
+static void
+sun8i_crypto_register(struct sun8i_crypto_softc *sc)
+{
+ struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto;
+
+ co->co_driverid = crypto_get_driverid(0);
+ if (co->co_driverid == (uint32_t)-1) {
+ aprint_error_dev(sc->sc_dev,
+ "failed to register crypto driver\n");
+ return;
+ }
+
+ sun8i_crypto_register1(sc, CRYPTO_AES_CBC);
+ sun8i_crypto_register1(sc, CRYPTO_AES_CTR);
+#ifdef CRYPTO_AES_ECB
+ sun8i_crypto_register1(sc, CRYPTO_AES_ECB);
+#endif
+#ifdef CRYPTO_AES_XTS
+ sun8i_crypto_register1(sc, CRYPTO_AES_XTS);
+#endif
+#ifdef CRYPTO_DES_CBC
+ sun8i_crypto_register1(sc, CRYPTO_DES_CBC);
+#endif
+#ifdef CRYPTO_DES_ECB
+ sun8i_crypto_register1(sc, CRYPTO_DES_ECB);
+#endif
+ sun8i_crypto_register1(sc, CRYPTO_3DES_CBC);
+#ifdef CRYPTO_3DES_ECB
+ sun8i_crypto_register1(sc, CRYPTO_3DES_ECB);
+#endif
+
+ sun8i_crypto_register1(sc, CRYPTO_MD5);
+ sun8i_crypto_register1(sc, CRYPTO_SHA1);
+#ifdef CRYPTO_SHA224
+ sun8i_crypto_register1(sc, CRYPTO_SHA224);
+#endif
+#ifdef CRYPTO_SHA256
+ sun8i_crypto_register1(sc, CRYPTO_SHA256);
+#endif
+
+ sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC);
+ sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC);
+
+ //sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */
+}
+
+/*
+ * sun8i_crypto_register1(sc, alg)
+ *
+ * Register support for one algorithm alg using
+ * sun8i_crypto_newsession/freesession/process.
+ */
+static void
+sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg)
+{
+
+ crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0,
+ sun8i_crypto_newsession,
+ sun8i_crypto_freesession,
+ sun8i_crypto_process,
+ sc);
+}
+
+/*
+ * sun8i_crypto_newsession(cookie, sidp, cri)
+ *
+ * Called by opencrypto to allocate a new session. We don't keep
+ * track of sessions, since there are no persistent keys in the
+ * hardware that we take advantage of, so this only validates the
+ * crypto operations and returns a zero session id.
+ */
+static int
+sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri)
+{
+
+ /* No composition of operations is supported here. */
+ if (cri->cri_next)
+ return EINVAL;
+
+ /*
+ * No variation of rounds is supported here. (XXX Unused and
+ * unimplemented in opencrypto(9) altogether?
+ */
+ if (cri->cri_rnd)
+ return EINVAL;
+
+ /*
+ * Validate per-algorithm key length.
+ *
+ * XXX Does opencrypto(9) do this internally?
+ */
+ switch (cri->cri_alg) {
+ case CRYPTO_MD5:
+ case CRYPTO_SHA1:
+#ifdef CRYPTO_SHA224
+ case CRYPTO_SHA224:
+#endif
+#ifdef CRYPTO_SHA256
+ case CRYPTO_SHA256:
+#endif
+ if (cri->cri_klen)
+ return EINVAL;
+ break;
+ case CRYPTO_AES_CBC:
+#ifdef CRYPTO_AES_ECB
+ case CRYPTO_AES_ECB:
+#endif
+ switch (cri->cri_klen) {
+ case 128:
+ case 192:
+ case 256:
+ break;
+ default:
+ return EINVAL;
+ }
+ break;
+ case CRYPTO_AES_CTR:
+ /*
+ * opencrypto `AES-CTR' takes four bytes of the input
+ * block as the last four bytes of the key, for reasons
+ * that are not entirely clear.
+ */
+ switch (cri->cri_klen) {
+ case 128 + 32:
+ case 192 + 32:
+ case 256 + 32:
+ break;
+ default:
+ return EINVAL;
+ }
+ break;
+#ifdef CRYPTO_AES_XTS
+ case CRYPTO_AES_XTS:
+ switch (cri->cri_klen) {
+ case 256:
+ case 384:
+ case 512:
+ break;
+ default:
+ return EINVAL;
+ }
+ break;
+#endif
+ case CRYPTO_DES_CBC:
+#ifdef CRYPTO_DES_ECB
+ case CRYPTO_DES_ECB:
+#endif
+ switch (cri->cri_klen) {
+ case 64:
+ break;
+ default:
+ return EINVAL;
+ }
+ break;
+ case CRYPTO_3DES_CBC:
+#ifdef CRYPTO_3DES_ECB
+ case CRYPTO_3DES_ECB:
+#endif
+ switch (cri->cri_klen) {
+ case 192:
+ break;
+ default:
+ return EINVAL;
+ }
+ break;
+ case CRYPTO_SHA1_HMAC:
+ /*
+ * XXX Unclear what the length limit is, but since HMAC
+ * behaves qualitatively different for a key of at
+ * least the full block size -- and is generally best
+ * to use with half the block size -- let's limit it to
+ * one block.
+ */
+ if (cri->cri_klen % 8)
+ return EINVAL;
+ if (cri->cri_klen > 512)
+ return EINVAL;
+ break;
+ case CRYPTO_SHA2_256_HMAC:
+ if (cri->cri_klen % 8)
+ return EINVAL;
+ if (cri->cri_klen > 512)
+ return EINVAL;
+ break;
+ default:
+ panic("unsupported algorithm %d", cri->cri_alg);
+ }
+
+ KASSERT(cri->cri_klen % 8 == 0);
+
+ /* Success! */
+ *sidp = 1;
+ return 0;
+}
+
+/*
+ * sun8i_crypto_freesession(cookie, dsid)
+ *
+ * Called by opencrypto to free a session. We don't keep track of
+ * sessions, since there are no persistent keys in the hardware
+ * that we take advantage of, so this is a no-op.
+ *
+ * Note: dsid is actually a 64-bit quantity containing both the
+ * driver id in the high half and the session id in the low half.
+ */
+static int
+sun8i_crypto_freesession(void *cookie, uint64_t dsid)
+{
+
+ KASSERT((dsid & 0xffffffff) == 1);
+
+ /* Success! */
+ return 0;
+}
+
+/*
+ * sun8i_crypto_ivlen(crd)
+ *
+ * Return the crypto engine's notion of `IV length', in bytes, for
+ * an opencrypto operation.
+ */
+static u_int
+sun8i_crypto_ivlen(const struct cryptodesc *crd)
+{
+
+ switch (crd->crd_alg) {
+ case CRYPTO_AES_CBC:
+ return 16;
+#ifdef CRYPTO_AES_XTS
+ case CRYPTO_AES_XTS:
+ return 16;
+#endif
+ case CRYPTO_AES_CTR: /* XXX opencrypto quirk */
+ return 8;
+#ifdef CRYPTO_DES_CBC
+ case CRYPTO_DES_CBC:
+ return 8;
+#endif
+ case CRYPTO_3DES_CBC:
+ return 8;
+ case CRYPTO_MD5:
+ return 16;
+#ifdef CRYPTO_SHA224
+ case CRYPTO_SHA224:
+ return 32;
+#endif
+#ifdef CRYPTO_SHA256
+ case CRYPTO_SHA256:
+ return 32;
+#endif
+ case CRYPTO_SHA1_HMAC:
+ return 20;
+ case CRYPTO_SHA2_256_HMAC:
+ return 32;
+ default:
+ return 0;
+ }
+}
+
+/*
+ * sun8i_crypto_process(cookie, crp, hint)
+ *
+ * Main opencrypto processing dispatch.
+ */
+static int
+sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint)
+{
+ struct sun8i_crypto_softc *sc = cookie;
+ struct sun8i_crypto_task *task;
+ struct cryptodesc *crd = crp->crp_desc;
+ unsigned klen, ivlen;
+ uint32_t tdqc = 0, tdqs = 0;
+ uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0;
+ const uint32_t tdqa = 0;
+ int error;
+
+ SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint);
+
+ /* Reject compositions -- we do not handle them. */
+ if (crd->crd_next != NULL) {
+ error = EOPNOTSUPP;
+ goto fail0;
+ }
+
+ /* Reject transfers with nonsense skip. */
+ if (crd->crd_skip < 0) {
+ error = EINVAL;
+ goto fail0;
+ }
+
+ /*
+ * Actually just reject any nonzero skip, because it requires
+ * DMA segment bookkeeping that we don't do yet.
+ */
+ if (crd->crd_skip) {
+ error = EOPNOTSUPP;
+ goto fail0;
+ }
+
+ /* Reject large transfers. */
+ if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) {
+ error = EFBIG;
+ goto fail0;
+ }
+
+ /* Reject nonsense, unaligned, or mismatched lengths. */
+ if (crd->crd_len < 0 ||
+ crd->crd_len % 4 ||
+ crd->crd_len != crp->crp_ilen) {
+ error = EINVAL;
+ goto fail0;
+ }
+
+ /* Reject mismatched buffer lengths. */
+ /* XXX Handle crd_skip. */
+ if (crp->crp_flags & CRYPTO_F_IMBUF) {
+ struct mbuf *m = crp->crp_buf;
+ uint32_t nbytes = 0;
+ while (m != NULL) {
+ KASSERT(m->m_len >= 0);
+ if (m->m_len > crd->crd_len ||
+ nbytes > crd->crd_len - m->m_len) {
+ error = EINVAL;
+ goto fail0;
+ }
+ nbytes += m->m_len;
+ m = m->m_next;
+ }
+ if (nbytes != crd->crd_len) {
+ error = EINVAL;
+ goto fail0;
+ }
+ } else if (crp->crp_flags & CRYPTO_F_IOV) {
+ struct uio *uio = crp->crp_buf;
+ if (uio->uio_resid != crd->crd_len) {
+ error = EINVAL;
+ goto fail0;
+ }
+ }
+
+ /* Get a task, or fail with ERESTART if we can't. */
+ task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp,
+ PR_NOWAIT);
+ if (task == NULL) {
+ /*
+ * Don't invoke crypto_done -- we are asking the
+ * opencrypto(9) machinery to queue the request and get
+ * back to us.
+ */
+ SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint);
+ return ERESTART;
+ }
+
+ /* Load key in, if relevant. */
+ klen = crd->crd_klen;
+ if (klen) {
+ if (crd->crd_alg == CRYPTO_AES_CTR)
+ /* AES-CTR is special -- see IV processing below. */
+ klen -= 32;
+ error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap,
+ crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT);
+ if (error)
+ goto fail1;
+ task->ct_flags |= TASK_KEY;
+ }
+
+ /* Handle the IV, if relevant. */
+ ivlen = sun8i_crypto_ivlen(crd);
+ if (ivlen) {
+ void *iv;
+
+ /*
+ * If there's an explicit IV, use it; otherwise
+ * randomly generate one.
+ */
+ if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
+ iv = crd->crd_iv;
+ } else {
+ cprng_fast(task->ct_iv, ivlen);
+ iv = task->ct_iv;
+ }
+
+ /*
+ * If the IV is not already present in the user's
+ * buffer, copy it over.
+ */
+ if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
+ if (crp->crp_flags & CRYPTO_F_IMBUF) {
+ m_copyback(crp->crp_buf, crd->crd_inject,
+ ivlen, iv);
+ } else if (crp->crp_flags & CRYPTO_F_IOV) {
+ cuio_copyback(crp->crp_buf, crd->crd_inject,
+ ivlen, iv);
+ } else {
+ panic("invalid buffer type %x",
+ crp->crp_flags);
+ }
+ }
+
+ /*
+ * opencrypto's idea of `AES-CTR' is special.
+ *
+ * - The low 4 bytes of the input block are drawn from
+ * an extra 4 bytes at the end of the key.
+ *
+ * - The next 8 bytes of the input block are drawn from
+ * the opencrypto iv.
+ *
+ * - The high 4 bytes are the big-endian block counter,
+ * which starts at 1 because why not.
+ */
+ if (crd->crd_alg == CRYPTO_AES_CTR) {
+ uint8_t block[16];
+ uint32_t blkno = 1;
+
+ /* Format the initial input block. */
+ memcpy(block, crd->crd_key + klen/8, 4);
+ memcpy(block + 4, iv, 8);
+ be32enc(block + 12, blkno);
+
+ /* Copy it into the DMA buffer. */
+ memcpy(task->ct_iv, block, 16);
+ iv = task->ct_iv;
+ ivlen = 16;
+ }
+
+ /* Load the IV. */
+ error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen,
+ NULL, BUS_DMA_NOWAIT);
+ if (error)
+ goto fail1;
+ task->ct_flags |= TASK_IV;
+ }
+
+ /* Load the src and dst. */
+ if (crp->crp_flags & CRYPTO_F_IMBUF) {
+ struct mbuf *m = crp->crp_buf;
+
+ /* XXX Handle crd_skip. */
+ KASSERT(crd->crd_skip == 0);
+ error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m,
+ BUS_DMA_NOWAIT);
+ if (error)
+ goto fail1;
+ task->ct_flags |= TASK_SRC;
+
+ /* XXX Handle crd_skip. */
+ KASSERT(crd->crd_skip == 0);
+ error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m,
+ BUS_DMA_NOWAIT);
+ if (error)
+ goto fail1;
+ } else if (crp->crp_flags & CRYPTO_F_IOV) {
+ struct uio *uio = crp->crp_buf;
+
+ /* XXX Handle crd_skip. */
+ KASSERT(crd->crd_skip == 0);
+ error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio,
+ BUS_DMA_NOWAIT);
+ if (error)
+ goto fail1;
+ task->ct_flags |= TASK_SRC;
+
+ /* XXX Handle crd_skip. */
+ KASSERT(crd->crd_skip == 0);
+ error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio,
+ BUS_DMA_NOWAIT);
+ if (error)
+ goto fail1;
+ } else {
+ panic("invalid buffer type %x", crp->crp_flags);
+ }
+
+ /* Set the encryption direction. */
+ if (crd->crd_flags & CRD_F_ENCRYPT)
+ dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC;
+ else
+ dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC;
+ tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR);
+
+ /* Set the method. */
+ switch (crd->crd_alg) {
+ case CRYPTO_AES_CBC:
+ case CRYPTO_AES_CTR:
+#ifdef CRYPTO_AES_ECB
+ case CRYPTO_AES_ECB:
+#endif
+ method = SUN8I_CRYPTO_TDQC_METHOD_AES;
+ break;
+#ifdef CRYPTO_AES_XTS
+ case CRYPTO_AES_XTS:
+ method = SUN8I_CRYPTO_TDQC_METHOD_AES;
+ break;
+#endif
+ case CRYPTO_DES_CBC:
+#ifdef CRYPTO_DES_ECB
+ case CRYPTO_DES_ECB:
+#endif
+ method = SUN8I_CRYPTO_TDQC_METHOD_DES;
+ break;
+ case CRYPTO_3DES_CBC:
+#ifdef CRYPTO_3DES_ECB
+ case CRYPTO_3DES_ECB:
+#endif
+ method = SUN8I_CRYPTO_TDQC_METHOD_3DES;
+ break;
+ case CRYPTO_MD5:
+ method = SUN8I_CRYPTO_TDQC_METHOD_MD5;
+ break;
+ case CRYPTO_SHA1:
+ method = SUN8I_CRYPTO_TDQC_METHOD_SHA1;
+ break;
+#ifdef CRYPTO_SHA224
+ case CRYPTO_SHA224:
+ method = SUN8I_CRYPTO_TDQC_METHOD_SHA224;
+ break;
+#endif
+#ifdef CRYPTO_SHA256
+ case CRYPTO_SHA256:
+ method = SUN8I_CRYPTO_TDQC_METHOD_SHA256;
+ break;
+#endif
+ case CRYPTO_SHA1_HMAC:
+ method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1;
+ break;
+ case CRYPTO_SHA2_256_HMAC:
+ method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256;
+ break;
+ default:
+ panic("unknown algorithm %d", crd->crd_alg);
+ }
+ tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD);
+
+ /* Set the key selector. No idea how to use the internal keys. */
+ tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx,
+ SUN8I_CRYPTO_TDQS_SKEY_SELECT);
+
+ /* XXX Deal with AES_CTS_Last_Block_Flag. */
+
+ /* Set the mode. */
+ switch (crd->crd_alg) {
+#ifdef CRYPTO_AES_ECB
+ case CRYPTO_AES_ECB:
+ mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
+ break;
+#endif
+#ifdef CRYPTO_DES_ECB
+ case CRYPTO_DES_ECB:
+ mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
+ break;
+#endif
+#ifdef CRYPTO_3DES_ECB
+ case CRYPTO_3DES_ECB:
+ mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
+ break;
+#endif
+ case CRYPTO_AES_CBC:
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
+ mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC;
+ break;
+ case CRYPTO_AES_CTR:
+ mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR;
+ break;
+#ifdef CRYPTO_AES_XTS
+ case CRYPTO_AES_XTS:
+ mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS;
+ break;
+#endif
+ default:
+ panic("unknown algorithm %d", crd->crd_alg);
+ }
+ tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE);
+
+ /* Set the CTR width. */
+ switch (crd->crd_alg) {
+ case CRYPTO_AES_CTR:
+ ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32;
+ break;
+ }
+ tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH);
+
+ /* Set the AES key size. */
+ switch (crd->crd_alg) {
+ case CRYPTO_AES_CBC:
+#ifdef CRYPTO_AES_ECB
+ case CRYPTO_AES_ECB:
+#endif
+ switch (crd->crd_klen) {
+ case 128:
+ aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
+ break;
+ case 192:
+ aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
+ break;
+ case 256:
+ aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
+ break;
+ default:
+ panic("invalid AES key size in bits: %u",
+ crd->crd_klen);
+ }
+ break;
+ case CRYPTO_AES_CTR:
+ switch (crd->crd_klen) {
+ case 128 + 32:
+ aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
+ break;
+ case 192 + 32:
+ aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
+ break;
+ case 256 + 32:
+ aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
+ break;
+ default:
+ panic("invalid `AES-CTR' ` ``key'' size' in bits: %u",
+ crd->crd_klen);
+ }
+ break;
+#ifdef CRYPTO_AES_XTS
+ case CRYPTO_AES_XTS:
+ switch (crd->crd_klen) {
+ case 256:
+ aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
+ break;
+ case 384:
+ aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
+ break;
+ case 512:
+ aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
+ break;
+ default:
+ panic("invalid AES-XTS key size in bits: %u",
+ crd->crd_klen);
+ }
+ break;
+#endif
+ }
+ tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE);
+
+ /* Set up the task descriptor. */
+ error = sun8i_crypto_task_load(sc, task, crd->crd_len,
+ tdqc, tdqs, tdqa);
+ if (error)
+ goto fail2;
+
+ /* Submit! */
+ error = sun8i_crypto_submit(sc, task);
+ if (error)
+ goto fail2;
+
+ /* Success! */
+ SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task);
+ return 0;
+
+fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
+fail1: if (task->ct_flags & TASK_SRC)
+ bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
+ if (task->ct_flags & TASK_CTR)
+ bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap);
+ if (task->ct_flags & TASK_IV)
+ bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap);
+ if (task->ct_flags & TASK_KEY)
+ bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
+ sun8i_crypto_task_put(sc, task);
+fail0: KASSERT(error);
+ KASSERT(error != ERESTART);
+ crp->crp_etype = error;
+ SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error);
+ crypto_done(crp);
+ return 0;
+}
+
+/*
+ * sun8i_crypto_callback(sc, task, cookie, error)
+ *
+ * Completion callback for a task submitted via opencrypto.
+ * Release the task and pass the error on to opencrypto with
+ * crypto_done.
+ */
+static void
+sun8i_crypto_callback(struct sun8i_crypto_softc *sc,
+ struct sun8i_crypto_task *task, void *cookie, int error)
+{
+ struct cryptop *crp = cookie;
+ struct cryptodesc *crd = crp->crp_desc;
+
+ KASSERT(error != ERESTART);
+ KASSERT(crd != NULL);
+ KASSERT(crd->crd_next == NULL);
+
+ /* Return the number of bytes processed. */
+ crp->crp_olen = error ? 0 : crp->crp_ilen;
+
+ bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
+ bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
+ if (task->ct_flags & TASK_CTR)
+ bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap);
+ if (task->ct_flags & TASK_IV)
+ bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap);
+ if (task->ct_flags & TASK_KEY)
+ bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
+ sun8i_crypto_task_put(sc, task);
+ KASSERT(error != ERESTART);
+ crp->crp_etype = error;
+ SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error);
+ crypto_done(crp);
+}
