Module Name: src
Committed By: riastradh
Date: Sat May 29 08:46:38 UTC 2021
Modified Files:
src/sys/dev/ic: nvme.c nvmevar.h
src/sys/dev/pci: nvme_pci.c
Log Message:
nvme(4): Add suspend/resume, derived from OpenBSD.
To generate a diff of this commit:
cvs rdiff -u -r1.55 -r1.56 src/sys/dev/ic/nvme.c
cvs rdiff -u -r1.21 -r1.22 src/sys/dev/ic/nvmevar.h
cvs rdiff -u -r1.29 -r1.30 src/sys/dev/pci/nvme_pci.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/ic/nvme.c
diff -u src/sys/dev/ic/nvme.c:1.55 src/sys/dev/ic/nvme.c:1.56
--- src/sys/dev/ic/nvme.c:1.55 Sat Apr 24 23:36:55 2021
+++ src/sys/dev/ic/nvme.c Sat May 29 08:46:38 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: nvme.c,v 1.55 2021/04/24 23:36:55 thorpej Exp $ */
+/* $NetBSD: nvme.c,v 1.56 2021/05/29 08:46:38 riastradh Exp $ */
/* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */
/*
@@ -18,7 +18,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.55 2021/04/24 23:36:55 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.56 2021/05/29 08:46:38 riastradh Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -82,6 +82,7 @@ static void nvme_empty_done(struct nvme_
static struct nvme_queue *
nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int);
static int nvme_q_create(struct nvme_softc *, struct nvme_queue *);
+static void nvme_q_reset(struct nvme_softc *, struct nvme_queue *);
static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *);
static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *,
struct nvme_ccb *, void (*)(struct nvme_queue *,
@@ -339,7 +340,6 @@ nvme_attach(struct nvme_softc *sc)
{
uint64_t cap;
uint32_t reg;
- u_int dstrd;
u_int mps = PAGE_SHIFT;
u_int ncq, nsq;
uint16_t adminq_entries = nvme_adminq_size;
@@ -360,7 +360,7 @@ nvme_attach(struct nvme_softc *sc)
NVME_VS_MNR(reg), NVME_VS_TER(reg));
cap = nvme_read8(sc, NVME_CAP);
- dstrd = NVME_CAP_DSTRD(cap);
+ sc->sc_dstrd = NVME_CAP_DSTRD(cap);
if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) {
aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u "
"is greater than CPU page size %u\n",
@@ -383,7 +383,8 @@ nvme_attach(struct nvme_softc *sc)
return 1;
}
- sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd);
+ sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries,
+ sc->sc_dstrd);
if (sc->sc_admin_q == NULL) {
aprint_error_dev(sc->sc_dev,
"unable to allocate admin queue\n");
@@ -428,7 +429,8 @@ nvme_attach(struct nvme_softc *sc)
sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP);
for (i = 0; i < sc->sc_nq; i++) {
- sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd);
+ sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries,
+ sc->sc_dstrd);
if (sc->sc_q[i] == NULL) {
aprint_error_dev(sc->sc_dev,
"unable to allocate io queue\n");
@@ -564,7 +566,69 @@ nvme_detach(struct nvme_softc *sc, int f
return 0;
}
-static int
+int
+nvme_suspend(struct nvme_softc *sc)
+{
+
+ return nvme_shutdown(sc);
+}
+
+int
+nvme_resume(struct nvme_softc *sc)
+{
+ int ioq_entries = nvme_ioq_size;
+ uint64_t cap;
+ int i, error;
+
+ error = nvme_disable(sc);
+ if (error) {
+ device_printf(sc->sc_dev, "unable to disable controller\n");
+ return error;
+ }
+
+ nvme_q_reset(sc, sc->sc_admin_q);
+
+ error = nvme_enable(sc, ffs(sc->sc_mps) - 1);
+ if (error) {
+ device_printf(sc->sc_dev, "unable to enable controller\n");
+ return error;
+ }
+
+ for (i = 0; i < sc->sc_nq; i++) {
+ cap = nvme_read8(sc, NVME_CAP);
+ if (ioq_entries > NVME_CAP_MQES(cap))
+ ioq_entries = NVME_CAP_MQES(cap);
+ sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries,
+ sc->sc_dstrd);
+ if (sc->sc_q[i] == NULL) {
+ error = ENOMEM;
+ device_printf(sc->sc_dev, "unable to allocate io q %d"
+ "\n", i);
+ goto disable;
+ }
+ if (nvme_q_create(sc, sc->sc_q[i]) != 0) {
+ error = EIO;
+ device_printf(sc->sc_dev, "unable to create io q %d"
+ "\n", i);
+ nvme_q_free(sc, sc->sc_q[i]);
+ goto free_q;
+ }
+ }
+
+ nvme_write4(sc, NVME_INTMC, 1);
+
+ return 0;
+
+free_q:
+ while (i --> 0)
+ nvme_q_free(sc, sc->sc_q[i]);
+disable:
+ (void)nvme_disable(sc);
+
+ return error;
+}
+
+int
nvme_shutdown(struct nvme_softc *sc)
{
uint32_t cc, csts;
@@ -1832,6 +1896,24 @@ free:
}
static void
+nvme_q_reset(struct nvme_softc *sc, struct nvme_queue *q)
+{
+
+ memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem));
+ memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem));
+
+ q->q_sqtdbl = NVME_SQTDBL(q->q_id, sc->sc_dstrd);
+ q->q_cqhdbl = NVME_CQHDBL(q->q_id, sc->sc_dstrd);
+
+ q->q_sq_tail = 0;
+ q->q_cq_head = 0;
+ q->q_cq_phase = NVME_CQE_PHASE;
+
+ nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE);
+ nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
+}
+
+static void
nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q)
{
nvme_ccbs_free(q);
Index: src/sys/dev/ic/nvmevar.h
diff -u src/sys/dev/ic/nvmevar.h:1.21 src/sys/dev/ic/nvmevar.h:1.22
--- src/sys/dev/ic/nvmevar.h:1.21 Tue Jul 28 09:36:05 2020
+++ src/sys/dev/ic/nvmevar.h Sat May 29 08:46:38 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmevar.h,v 1.21 2020/07/28 09:36:05 jdolecek Exp $ */
+/* $NetBSD: nvmevar.h,v 1.22 2021/05/29 08:46:38 riastradh Exp $ */
/* $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */
/*
@@ -122,6 +122,7 @@ struct nvme_softc {
size_t sc_mps; /* memory page size */
size_t sc_mdts; /* max data trasfer size */
u_int sc_max_sgl; /* max S/G segments */
+ u_int sc_dstrd;
struct nvm_identify_controller
sc_identify;
@@ -163,6 +164,8 @@ int nvme_attach(struct nvme_softc *);
int nvme_detach(struct nvme_softc *, int flags);
int nvme_rescan(device_t, const char *, const int *);
void nvme_childdet(device_t, device_t);
+int nvme_suspend(struct nvme_softc *);
+int nvme_resume(struct nvme_softc *);
int nvme_intr(void *);
void nvme_softintr_intx(void *);
int nvme_intr_msi(void *);
Index: src/sys/dev/pci/nvme_pci.c
diff -u src/sys/dev/pci/nvme_pci.c:1.29 src/sys/dev/pci/nvme_pci.c:1.30
--- src/sys/dev/pci/nvme_pci.c:1.29 Wed Jul 29 07:14:45 2020
+++ src/sys/dev/pci/nvme_pci.c Sat May 29 08:46:38 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: nvme_pci.c,v 1.29 2020/07/29 07:14:45 jdolecek Exp $ */
+/* $NetBSD: nvme_pci.c,v 1.30 2021/05/29 08:46:38 riastradh Exp $ */
/* $OpenBSD: nvme_pci.c,v 1.3 2016/04/14 11:18:32 dlg Exp $ */
/*
@@ -43,7 +43,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvme_pci.c,v 1.29 2020/07/29 07:14:45 jdolecek Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvme_pci.c,v 1.30 2021/05/29 08:46:38 riastradh Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -82,6 +82,8 @@ static int nvme_pci_match(device_t, cfda
static void nvme_pci_attach(device_t, device_t, void *);
static int nvme_pci_detach(device_t, int);
static int nvme_pci_rescan(device_t, const char *, const int *);
+static bool nvme_pci_suspend(device_t, const pmf_qual_t *);
+static bool nvme_pci_resume(device_t, const pmf_qual_t *);
CFATTACH_DECL3_NEW(nvme_pci, sizeof(struct nvme_pci_softc),
nvme_pci_match, nvme_pci_attach, nvme_pci_detach, NULL, nvme_pci_rescan,
@@ -232,7 +234,7 @@ nvme_pci_attach(device_t parent, device_
goto softintr_free;
}
- if (!pmf_device_register(self, NULL, NULL))
+ if (!pmf_device_register(self, nvme_pci_suspend, nvme_pci_resume))
aprint_error_dev(self, "couldn't establish power handler\n");
SET(sc->sc_flags, NVME_F_ATTACHED);
@@ -256,6 +258,34 @@ nvme_pci_rescan(device_t self, const cha
return nvme_rescan(self, attr, flags);
}
+static bool
+nvme_pci_suspend(device_t self, const pmf_qual_t *qual)
+{
+ struct nvme_pci_softc *psc = device_private(self);
+ struct nvme_softc *sc = &psc->psc_nvme;
+ int error;
+
+ error = nvme_suspend(sc);
+ if (error)
+ return false;
+
+ return true;
+}
+
+static bool
+nvme_pci_resume(device_t self, const pmf_qual_t *qual)
+{
+ struct nvme_pci_softc *psc = device_private(self);
+ struct nvme_softc *sc = &psc->psc_nvme;
+ int error;
+
+ error = nvme_resume(sc);
+ if (error)
+ return false;
+
+ return true;
+}
+
static int
nvme_pci_detach(device_t self, int flags)
{
