Module Name: src
Committed By: jdolecek
Date: Tue Oct 10 17:19:39 UTC 2017
Modified Files:
src/sys/dev/ata: ata.c ataconf.h atavar.h files.ata
Added Files:
src/sys/dev/ata: ata_subr.c
Log Message:
split off functions used by 'wd* at umass?' into separate file, unfortunately
the previous approach with NATABUS doesn't work for kernels which include
MODULAR, but not atabus - such as macppc and evbarm
To generate a diff of this commit:
cvs rdiff -u -r1.135 -r1.136 src/sys/dev/ata/ata.c
cvs rdiff -u -r0 -r1.1 src/sys/dev/ata/ata_subr.c
cvs rdiff -u -r1.4 -r1.5 src/sys/dev/ata/ataconf.h
cvs rdiff -u -r1.93 -r1.94 src/sys/dev/ata/atavar.h
cvs rdiff -u -r1.26 -r1.27 src/sys/dev/ata/files.ata
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/ata/ata.c
diff -u src/sys/dev/ata/ata.c:1.135 src/sys/dev/ata/ata.c:1.136
--- src/sys/dev/ata/ata.c:1.135 Sun Oct 8 19:00:29 2017
+++ src/sys/dev/ata/ata.c Tue Oct 10 17:19:38 2017
@@ -1,4 +1,4 @@
-/* $NetBSD: ata.c,v 1.135 2017/10/08 19:00:29 jdolecek Exp $ */
+/* $NetBSD: ata.c,v 1.136 2017/10/10 17:19:38 jdolecek Exp $ */
/*
* Copyright (c) 1998, 2001 Manuel Bouyer. All rights reserved.
@@ -25,7 +25,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.135 2017/10/08 19:00:29 jdolecek Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.136 2017/10/10 17:19:38 jdolecek Exp $");
#include "opt_ata.h"
@@ -83,7 +83,6 @@ int atadebug_mask = ATADEBUG_MASK;
#define ATADEBUG_PRINT(args, level)
#endif
-#if NATABUS
static ONCE_DECL(ata_init_ctrl);
/*
@@ -134,7 +133,6 @@ static void ata_channel_idle(struct ata_
static void ata_channel_thaw_locked(struct ata_channel *);
static void ata_activate_xfer_locked(struct ata_channel *, struct ata_xfer *);
static void ata_channel_freeze_locked(struct ata_channel *);
-static struct ata_xfer *ata_queue_get_active_xfer_locked(struct ata_channel *);
static void ata_thread_wake_locked(struct ata_channel *);
/*
@@ -186,177 +184,7 @@ ataprint(void *aux, const char *pnp)
return (UNCONF);
}
-#endif /* NATABUS */
-static void
-ata_queue_reset(struct ata_queue *chq)
-{
- /* make sure that we can use polled commands */
- TAILQ_INIT(&chq->queue_xfer);
- TAILQ_INIT(&chq->active_xfers);
- chq->queue_freeze = 0;
- chq->queue_active = 0;
- chq->active_xfers_used = 0;
- chq->queue_xfers_avail = __BIT(chq->queue_openings) - 1;
-}
-
-struct ata_xfer *
-ata_queue_hwslot_to_xfer(struct ata_channel *chp, int hwslot)
-{
- struct ata_queue *chq = chp->ch_queue;
- struct ata_xfer *xfer = NULL;
-
- ata_channel_lock(chp);
-
- KASSERTMSG(hwslot < chq->queue_openings, "hwslot %d > openings %d",
- hwslot, chq->queue_openings);
- KASSERTMSG((chq->active_xfers_used & __BIT(hwslot)) != 0,
- "hwslot %d not active", hwslot);
-
- /* Usually the first entry will be the one */
- TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) {
- if (xfer->c_slot == hwslot)
- break;
- }
-
- ata_channel_unlock(chp);
-
- KASSERTMSG((xfer != NULL),
- "%s: xfer with slot %d not found (active %x)", __func__,
- hwslot, chq->active_xfers_used);
-
- return xfer;
-}
-
-static struct ata_xfer *
-ata_queue_get_active_xfer_locked(struct ata_channel *chp)
-{
- struct ata_xfer *xfer;
-
- KASSERT(mutex_owned(&chp->ch_lock));
- xfer = TAILQ_FIRST(&chp->ch_queue->active_xfers);
-
- if (xfer && ISSET(xfer->c_flags, C_NCQ)) {
- /* Spurious call, never return NCQ xfer from this interface */
- xfer = NULL;
- }
-
- return xfer;
-}
-
-/*
- * This interface is supposed only to be used when there is exactly
- * one outstanding command, when there is no information about the slot,
- * which triggered the command. ata_queue_hwslot_to_xfer() interface
- * is preferred in all NCQ cases.
- */
-struct ata_xfer *
-ata_queue_get_active_xfer(struct ata_channel *chp)
-{
- struct ata_xfer *xfer = NULL;
-
- ata_channel_lock(chp);
- xfer = ata_queue_get_active_xfer_locked(chp);
- ata_channel_unlock(chp);
-
- return xfer;
-}
-
-struct ata_xfer *
-ata_queue_drive_active_xfer(struct ata_channel *chp, int drive)
-{
- struct ata_xfer *xfer = NULL;
-
- ata_channel_lock(chp);
-
- TAILQ_FOREACH(xfer, &chp->ch_queue->active_xfers, c_activechain) {
- if (xfer->c_drive == drive)
- break;
- }
- KASSERT(xfer != NULL);
-
- ata_channel_unlock(chp);
-
- return xfer;
-}
-
-static void
-ata_xfer_init(struct ata_xfer *xfer, uint8_t slot)
-{
- memset(xfer, 0, sizeof(*xfer));
-
- xfer->c_slot = slot;
-
- cv_init(&xfer->c_active, "ataact");
- cv_init(&xfer->c_finish, "atafin");
- callout_init(&xfer->c_timo_callout, 0); /* XXX MPSAFE */
- callout_init(&xfer->c_retry_callout, 0); /* XXX MPSAFE */
-}
-
-static void
-ata_xfer_destroy(struct ata_xfer *xfer)
-{
- callout_halt(&xfer->c_timo_callout, NULL); /* XXX MPSAFE */
- callout_destroy(&xfer->c_timo_callout);
- callout_halt(&xfer->c_retry_callout, NULL); /* XXX MPSAFE */
- callout_destroy(&xfer->c_retry_callout);
- cv_destroy(&xfer->c_active);
- cv_destroy(&xfer->c_finish);
-}
-
-struct ata_queue *
-ata_queue_alloc(uint8_t openings)
-{
- if (openings == 0)
- openings = 1;
-
- if (openings > ATA_MAX_OPENINGS)
- openings = ATA_MAX_OPENINGS;
-
- struct ata_queue *chq = malloc(offsetof(struct ata_queue, queue_xfers[openings]),
- M_DEVBUF, M_WAITOK | M_ZERO);
-
- chq->queue_openings = openings;
- ata_queue_reset(chq);
-
- cv_init(&chq->queue_busy, "ataqbusy");
- cv_init(&chq->queue_drain, "atdrn");
- cv_init(&chq->queue_idle, "qidl");
-
- for (uint8_t i = 0; i < openings; i++)
- ata_xfer_init(&chq->queue_xfers[i], i);
-
- return chq;
-}
-
-void
-ata_queue_free(struct ata_queue *chq)
-{
- for (uint8_t i = 0; i < chq->queue_openings; i++)
- ata_xfer_destroy(&chq->queue_xfers[i]);
-
- cv_destroy(&chq->queue_busy);
- cv_destroy(&chq->queue_drain);
- cv_destroy(&chq->queue_idle);
-
- free(chq, M_DEVBUF);
-}
-
-void
-ata_channel_init(struct ata_channel *chp)
-{
- mutex_init(&chp->ch_lock, MUTEX_DEFAULT, IPL_BIO);
- cv_init(&chp->ch_thr_idle, "atath");
-}
-
-void
-ata_channel_destroy(struct ata_channel *chp)
-{
- mutex_destroy(&chp->ch_lock);
- cv_destroy(&chp->ch_thr_idle);
-}
-
-#if NATABUS
/*
* ata_channel_attach:
*
@@ -1431,129 +1259,7 @@ ata_xfer_start(struct ata_xfer *xfer)
return rv;
}
-#endif /* NATABUS */
-
-/*
- * Does it's own locking, does not require splbio().
- * flags - whether to block waiting for free xfer
- * openings - limit of openings supported by device, <= 0 means tag not
- * relevant, and any available xfer can be returned
- */
-struct ata_xfer *
-ata_get_xfer_ext(struct ata_channel *chp, int flags, uint8_t openings)
-{
- struct ata_queue *chq = chp->ch_queue;
- struct ata_xfer *xfer = NULL;
- uint32_t avail, slot, mask;
- int error;
-
- ATADEBUG_PRINT(("%s: channel %d flags %x openings %d\n",
- __func__, chp->ch_channel, flags, openings),
- DEBUG_XFERS);
-
- ata_channel_lock(chp);
-
- /*
- * When openings is just 1, can't reserve anything for
- * recovery. KASSERT() here is to catch code which naively
- * relies on C_RECOVERY to work under this condition.
- */
- KASSERT((flags & C_RECOVERY) == 0 || chq->queue_openings > 1);
-
- if (flags & C_RECOVERY) {
- mask = UINT32_MAX;
- } else {
- if (openings <= 0 || openings > chq->queue_openings)
- openings = chq->queue_openings;
-
- if (openings > 1) {
- mask = __BIT(openings - 1) - 1;
- } else {
- mask = UINT32_MAX;
- }
- }
-
-retry:
- avail = ffs32(chq->queue_xfers_avail & mask);
- if (avail == 0) {
- /*
- * Catch code which tries to get another recovery xfer while
- * already holding one (wrong recursion).
- */
- KASSERTMSG((flags & C_RECOVERY) == 0,
- "recovery xfer busy openings %d mask %x avail %x",
- openings, mask, chq->queue_xfers_avail);
-
- if (flags & C_WAIT) {
- chq->queue_flags |= QF_NEED_XFER;
- error = cv_wait_sig(&chq->queue_busy, &chp->ch_lock);
- if (error == 0)
- goto retry;
- }
-
- goto out;
- }
-
- slot = avail - 1;
- xfer = &chq->queue_xfers[slot];
- chq->queue_xfers_avail &= ~__BIT(slot);
-
- KASSERT((chq->active_xfers_used & __BIT(slot)) == 0);
-
- /* zero everything after the callout member */
- memset(&xfer->c_startzero, 0,
- sizeof(struct ata_xfer) - offsetof(struct ata_xfer, c_startzero));
-
-out:
- ata_channel_unlock(chp);
- return xfer;
-}
-
-/*
- * ata_deactivate_xfer() must be always called prior to ata_free_xfer()
- */
-void
-ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
-{
- struct ata_queue *chq = chp->ch_queue;
-
- ata_channel_lock(chp);
-
- if (xfer->c_flags & (C_WAITACT|C_WAITTIMO)) {
- /* Someone is waiting for this xfer, so we can't free now */
- xfer->c_flags |= C_FREE;
- cv_signal(&xfer->c_active);
- goto out;
- }
-
-#if NATA_PIOBM /* XXX wdc dependent code */
- if (xfer->c_flags & C_PIOBM) {
- struct wdc_softc *wdc = CHAN_TO_WDC(chp);
-
- /* finish the busmastering PIO */
- (*wdc->piobm_done)(wdc->dma_arg,
- chp->ch_channel, xfer->c_drive);
- chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT);
- }
-#endif
-
- if (chp->ch_atac->atac_free_hw)
- chp->ch_atac->atac_free_hw(chp);
-
- KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0);
- KASSERT((chq->queue_xfers_avail & __BIT(xfer->c_slot)) == 0);
- chq->queue_xfers_avail |= __BIT(xfer->c_slot);
-
-out:
- if (chq->queue_flags & QF_NEED_XFER) {
- chq->queue_flags &= ~QF_NEED_XFER;
- cv_broadcast(&chq->queue_busy);
- }
-
- ata_channel_unlock(chp);
-}
-#if NATABUS
static void
ata_activate_xfer_locked(struct ata_channel *chp, struct ata_xfer *xfer)
{
@@ -2552,74 +2258,7 @@ atacmd_toncq(struct ata_xfer *xfer, uint
if (xfer->c_bio.flags & ATA_FUA)
*device |= WDSD_FUA;
}
-#endif /* NATABUS */
-
-/*
- * Must be called without any locks, i.e. with both drive and channel locks
- * released.
- */
-void
-ata_channel_start(struct ata_channel *chp, int drive)
-{
- int i, s;
- struct ata_drive_datas *drvp;
-
- s = splbio();
-
- KASSERT(chp->ch_ndrives > 0);
-
-#define ATA_DRIVE_START(chp, drive) \
- do { \
- KASSERT(drive < chp->ch_ndrives); \
- drvp = &chp->ch_drive[drive]; \
- \
- if (drvp->drive_type != ATA_DRIVET_ATA && \
- drvp->drive_type != ATA_DRIVET_ATAPI && \
- drvp->drive_type != ATA_DRIVET_OLD) \
- continue; \
- \
- if (drvp->drv_start != NULL) \
- (*drvp->drv_start)(drvp->drv_softc); \
- } while (0)
-
- /*
- * Process drives in round robin fashion starting with next one after
- * the one which finished transfer. Thus no single drive would
- * completely starve other drives on same channel.
- * This loop processes all but the current drive, so won't do anything
- * if there is only one drive in channel.
- */
- for (i = (drive + 1) % chp->ch_ndrives; i != drive;
- i = (i + 1) % chp->ch_ndrives) {
- ATA_DRIVE_START(chp, i);
- }
-
- /* Now try to kick off xfers on the current drive */
- ATA_DRIVE_START(chp, drive);
-
- splx(s);
-#undef ATA_DRIVE_START
-}
-
-void
-ata_channel_lock(struct ata_channel *chp)
-{
- mutex_enter(&chp->ch_lock);
-}
-
-void
-ata_channel_unlock(struct ata_channel *chp)
-{
- mutex_exit(&chp->ch_lock);
-}
-
-void
-ata_channel_lock_owned(struct ata_channel *chp)
-{
- KASSERT(mutex_owned(&chp->ch_lock));
-}
-#if NATABUS
void
ata_wait_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
{
@@ -2635,4 +2274,3 @@ ata_wake_xfer(struct ata_channel *chp, s
cv_signal(&xfer->c_finish);
}
-#endif /* NATABUS */
Index: src/sys/dev/ata/ataconf.h
diff -u src/sys/dev/ata/ataconf.h:1.4 src/sys/dev/ata/ataconf.h:1.5
--- src/sys/dev/ata/ataconf.h:1.4 Sun Oct 8 19:00:29 2017
+++ src/sys/dev/ata/ataconf.h Tue Oct 10 17:19:38 2017
@@ -1,4 +1,4 @@
-/* $NetBSD: ataconf.h,v 1.4 2017/10/08 19:00:29 jdolecek Exp $ */
+/* $NetBSD: ataconf.h,v 1.5 2017/10/10 17:19:38 jdolecek Exp $ */
/*
* Written in 2006 by ITOH Yasufumi.
@@ -18,7 +18,6 @@
# define NATA_DMA 1
# define NATA_UDMA 1
# define NATA_PIOBM 1
-# define NATABUS 1
#else
Index: src/sys/dev/ata/atavar.h
diff -u src/sys/dev/ata/atavar.h:1.93 src/sys/dev/ata/atavar.h:1.94
--- src/sys/dev/ata/atavar.h:1.93 Sat Oct 7 16:05:32 2017
+++ src/sys/dev/ata/atavar.h Tue Oct 10 17:19:38 2017
@@ -1,4 +1,4 @@
-/* $NetBSD: atavar.h,v 1.93 2017/10/07 16:05:32 jdolecek Exp $ */
+/* $NetBSD: atavar.h,v 1.94 2017/10/10 17:19:38 jdolecek Exp $ */
/*
* Copyright (c) 1998, 2001 Manuel Bouyer.
@@ -556,11 +556,14 @@ void ata_dmaerr(struct ata_drive_datas *
struct ata_queue *
ata_queue_alloc(uint8_t openings);
void ata_queue_free(struct ata_queue *);
+void ata_queue_reset(struct ata_queue *);
struct ata_xfer *
ata_queue_hwslot_to_xfer(struct ata_channel *, int);
struct ata_xfer *
ata_queue_get_active_xfer(struct ata_channel *);
struct ata_xfer *
+ ata_queue_get_active_xfer_locked(struct ata_channel *);
+struct ata_xfer *
ata_queue_drive_active_xfer(struct ata_channel *, int);
void ata_delay(struct ata_channel *, int, const char *, int);
Index: src/sys/dev/ata/files.ata
diff -u src/sys/dev/ata/files.ata:1.26 src/sys/dev/ata/files.ata:1.27
--- src/sys/dev/ata/files.ata:1.26 Sun Oct 8 19:00:29 2017
+++ src/sys/dev/ata/files.ata Tue Oct 10 17:19:38 2017
@@ -1,4 +1,4 @@
-# $NetBSD: files.ata,v 1.26 2017/10/08 19:00:29 jdolecek Exp $
+# $NetBSD: files.ata,v 1.27 2017/10/10 17:19:38 jdolecek Exp $
#
# Config file and device description for machine-independent devices
# which attach to ATA busses. Included by ports that need it. Ports
@@ -14,7 +14,8 @@ file dev/ata/ata_wdc.c wd & atabus & wd
defflag opt_wd.h WD_SOFTBADSECT
defflag opt_wd.h WD_CHAOS_MONKEY
-file dev/ata/ata.c (ata_hl | atapi)
+file dev/ata/ata.c (ata_hl | atapi) & atabus
+file dev/ata/ata_subr.c (ata_hl | atapi)
# ATA RAID configuration support
defpseudodev ataraid {[vendtype = -1], [unit = -1]}
Added files:
Index: src/sys/dev/ata/ata_subr.c
diff -u /dev/null src/sys/dev/ata/ata_subr.c:1.1
--- /dev/null Tue Oct 10 17:19:39 2017
+++ src/sys/dev/ata/ata_subr.c Tue Oct 10 17:19:38 2017
@@ -0,0 +1,420 @@
+/* $NetBSD: ata_subr.c,v 1.1 2017/10/10 17:19:38 jdolecek Exp $ */
+
+/*
+ * Copyright (c) 1998, 2001 Manuel Bouyer. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: ata_subr.c,v 1.1 2017/10/10 17:19:38 jdolecek Exp $");
+
+#include "opt_ata.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/device.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <sys/proc.h>
+#include <sys/kthread.h>
+#include <sys/errno.h>
+#include <sys/ataio.h>
+#include <sys/kmem.h>
+#include <sys/intr.h>
+#include <sys/bus.h>
+#include <sys/once.h>
+#include <sys/bitops.h>
+
+#define ATABUS_PRIVATE
+
+#include <dev/ata/ataconf.h>
+#include <dev/ata/atareg.h>
+#include <dev/ata/atavar.h>
+#include <dev/ic/wdcvar.h> /* for PIOBM */
+
+#define DEBUG_FUNCS 0x08
+#define DEBUG_PROBE 0x10
+#define DEBUG_DETACH 0x20
+#define DEBUG_XFERS 0x40
+#ifdef ATADEBUG
+extern int atadebug_mask;
+#define ATADEBUG_PRINT(args, level) \
+ if (atadebug_mask & (level)) \
+ printf args
+#else
+#define ATADEBUG_PRINT(args, level)
+#endif
+
+void
+ata_queue_reset(struct ata_queue *chq)
+{
+ /* make sure that we can use polled commands */
+ TAILQ_INIT(&chq->queue_xfer);
+ TAILQ_INIT(&chq->active_xfers);
+ chq->queue_freeze = 0;
+ chq->queue_active = 0;
+ chq->active_xfers_used = 0;
+ chq->queue_xfers_avail = __BIT(chq->queue_openings) - 1;
+}
+
+struct ata_xfer *
+ata_queue_hwslot_to_xfer(struct ata_channel *chp, int hwslot)
+{
+ struct ata_queue *chq = chp->ch_queue;
+ struct ata_xfer *xfer = NULL;
+
+ ata_channel_lock(chp);
+
+ KASSERTMSG(hwslot < chq->queue_openings, "hwslot %d > openings %d",
+ hwslot, chq->queue_openings);
+ KASSERTMSG((chq->active_xfers_used & __BIT(hwslot)) != 0,
+ "hwslot %d not active", hwslot);
+
+ /* Usually the first entry will be the one */
+ TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) {
+ if (xfer->c_slot == hwslot)
+ break;
+ }
+
+ ata_channel_unlock(chp);
+
+ KASSERTMSG((xfer != NULL),
+ "%s: xfer with slot %d not found (active %x)", __func__,
+ hwslot, chq->active_xfers_used);
+
+ return xfer;
+}
+
+struct ata_xfer *
+ata_queue_get_active_xfer_locked(struct ata_channel *chp)
+{
+ struct ata_xfer *xfer;
+
+ KASSERT(mutex_owned(&chp->ch_lock));
+ xfer = TAILQ_FIRST(&chp->ch_queue->active_xfers);
+
+ if (xfer && ISSET(xfer->c_flags, C_NCQ)) {
+ /* Spurious call, never return NCQ xfer from this interface */
+ xfer = NULL;
+ }
+
+ return xfer;
+}
+
+/*
+ * This interface is supposed only to be used when there is exactly
+ * one outstanding command, when there is no information about the slot,
+ * which triggered the command. ata_queue_hwslot_to_xfer() interface
+ * is preferred in all NCQ cases.
+ */
+struct ata_xfer *
+ata_queue_get_active_xfer(struct ata_channel *chp)
+{
+ struct ata_xfer *xfer = NULL;
+
+ ata_channel_lock(chp);
+ xfer = ata_queue_get_active_xfer_locked(chp);
+ ata_channel_unlock(chp);
+
+ return xfer;
+}
+
+struct ata_xfer *
+ata_queue_drive_active_xfer(struct ata_channel *chp, int drive)
+{
+ struct ata_xfer *xfer = NULL;
+
+ ata_channel_lock(chp);
+
+ TAILQ_FOREACH(xfer, &chp->ch_queue->active_xfers, c_activechain) {
+ if (xfer->c_drive == drive)
+ break;
+ }
+ KASSERT(xfer != NULL);
+
+ ata_channel_unlock(chp);
+
+ return xfer;
+}
+
+static void
+ata_xfer_init(struct ata_xfer *xfer, uint8_t slot)
+{
+ memset(xfer, 0, sizeof(*xfer));
+
+ xfer->c_slot = slot;
+
+ cv_init(&xfer->c_active, "ataact");
+ cv_init(&xfer->c_finish, "atafin");
+ callout_init(&xfer->c_timo_callout, 0); /* XXX MPSAFE */
+ callout_init(&xfer->c_retry_callout, 0); /* XXX MPSAFE */
+}
+
+static void
+ata_xfer_destroy(struct ata_xfer *xfer)
+{
+ callout_halt(&xfer->c_timo_callout, NULL); /* XXX MPSAFE */
+ callout_destroy(&xfer->c_timo_callout);
+ callout_halt(&xfer->c_retry_callout, NULL); /* XXX MPSAFE */
+ callout_destroy(&xfer->c_retry_callout);
+ cv_destroy(&xfer->c_active);
+ cv_destroy(&xfer->c_finish);
+}
+
+struct ata_queue *
+ata_queue_alloc(uint8_t openings)
+{
+ if (openings == 0)
+ openings = 1;
+
+ if (openings > ATA_MAX_OPENINGS)
+ openings = ATA_MAX_OPENINGS;
+
+ struct ata_queue *chq = malloc(offsetof(struct ata_queue, queue_xfers[openings]),
+ M_DEVBUF, M_WAITOK | M_ZERO);
+
+ chq->queue_openings = openings;
+ ata_queue_reset(chq);
+
+ cv_init(&chq->queue_busy, "ataqbusy");
+ cv_init(&chq->queue_drain, "atdrn");
+ cv_init(&chq->queue_idle, "qidl");
+
+ for (uint8_t i = 0; i < openings; i++)
+ ata_xfer_init(&chq->queue_xfers[i], i);
+
+ return chq;
+}
+
+void
+ata_queue_free(struct ata_queue *chq)
+{
+ for (uint8_t i = 0; i < chq->queue_openings; i++)
+ ata_xfer_destroy(&chq->queue_xfers[i]);
+
+ cv_destroy(&chq->queue_busy);
+ cv_destroy(&chq->queue_drain);
+ cv_destroy(&chq->queue_idle);
+
+ free(chq, M_DEVBUF);
+}
+
+void
+ata_channel_init(struct ata_channel *chp)
+{
+ mutex_init(&chp->ch_lock, MUTEX_DEFAULT, IPL_BIO);
+ cv_init(&chp->ch_thr_idle, "atath");
+}
+
+void
+ata_channel_destroy(struct ata_channel *chp)
+{
+ mutex_destroy(&chp->ch_lock);
+ cv_destroy(&chp->ch_thr_idle);
+}
+
+/*
+ * Does it's own locking, does not require splbio().
+ * flags - whether to block waiting for free xfer
+ * openings - limit of openings supported by device, <= 0 means tag not
+ * relevant, and any available xfer can be returned
+ */
+struct ata_xfer *
+ata_get_xfer_ext(struct ata_channel *chp, int flags, uint8_t openings)
+{
+ struct ata_queue *chq = chp->ch_queue;
+ struct ata_xfer *xfer = NULL;
+ uint32_t avail, slot, mask;
+ int error;
+
+ ATADEBUG_PRINT(("%s: channel %d flags %x openings %d\n",
+ __func__, chp->ch_channel, flags, openings),
+ DEBUG_XFERS);
+
+ ata_channel_lock(chp);
+
+ /*
+ * When openings is just 1, can't reserve anything for
+ * recovery. KASSERT() here is to catch code which naively
+ * relies on C_RECOVERY to work under this condition.
+ */
+ KASSERT((flags & C_RECOVERY) == 0 || chq->queue_openings > 1);
+
+ if (flags & C_RECOVERY) {
+ mask = UINT32_MAX;
+ } else {
+ if (openings <= 0 || openings > chq->queue_openings)
+ openings = chq->queue_openings;
+
+ if (openings > 1) {
+ mask = __BIT(openings - 1) - 1;
+ } else {
+ mask = UINT32_MAX;
+ }
+ }
+
+retry:
+ avail = ffs32(chq->queue_xfers_avail & mask);
+ if (avail == 0) {
+ /*
+ * Catch code which tries to get another recovery xfer while
+ * already holding one (wrong recursion).
+ */
+ KASSERTMSG((flags & C_RECOVERY) == 0,
+ "recovery xfer busy openings %d mask %x avail %x",
+ openings, mask, chq->queue_xfers_avail);
+
+ if (flags & C_WAIT) {
+ chq->queue_flags |= QF_NEED_XFER;
+ error = cv_wait_sig(&chq->queue_busy, &chp->ch_lock);
+ if (error == 0)
+ goto retry;
+ }
+
+ goto out;
+ }
+
+ slot = avail - 1;
+ xfer = &chq->queue_xfers[slot];
+ chq->queue_xfers_avail &= ~__BIT(slot);
+
+ KASSERT((chq->active_xfers_used & __BIT(slot)) == 0);
+
+ /* zero everything after the callout member */
+ memset(&xfer->c_startzero, 0,
+ sizeof(struct ata_xfer) - offsetof(struct ata_xfer, c_startzero));
+
+out:
+ ata_channel_unlock(chp);
+ return xfer;
+}
+
+/*
+ * ata_deactivate_xfer() must be always called prior to ata_free_xfer()
+ */
+void
+ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
+{
+ struct ata_queue *chq = chp->ch_queue;
+
+ ata_channel_lock(chp);
+
+ if (xfer->c_flags & (C_WAITACT|C_WAITTIMO)) {
+ /* Someone is waiting for this xfer, so we can't free now */
+ xfer->c_flags |= C_FREE;
+ cv_signal(&xfer->c_active);
+ goto out;
+ }
+
+#if NATA_PIOBM /* XXX wdc dependent code */
+ if (xfer->c_flags & C_PIOBM) {
+ struct wdc_softc *wdc = CHAN_TO_WDC(chp);
+
+ /* finish the busmastering PIO */
+ (*wdc->piobm_done)(wdc->dma_arg,
+ chp->ch_channel, xfer->c_drive);
+ chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT);
+ }
+#endif
+
+ if (chp->ch_atac->atac_free_hw)
+ chp->ch_atac->atac_free_hw(chp);
+
+ KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0);
+ KASSERT((chq->queue_xfers_avail & __BIT(xfer->c_slot)) == 0);
+ chq->queue_xfers_avail |= __BIT(xfer->c_slot);
+
+out:
+ if (chq->queue_flags & QF_NEED_XFER) {
+ chq->queue_flags &= ~QF_NEED_XFER;
+ cv_broadcast(&chq->queue_busy);
+ }
+
+ ata_channel_unlock(chp);
+}
+
+/*
+ * Must be called without any locks, i.e. with both drive and channel locks
+ * released.
+ */
+void
+ata_channel_start(struct ata_channel *chp, int drive)
+{
+ int i, s;
+ struct ata_drive_datas *drvp;
+
+ s = splbio();
+
+ KASSERT(chp->ch_ndrives > 0);
+
+#define ATA_DRIVE_START(chp, drive) \
+ do { \
+ KASSERT(drive < chp->ch_ndrives); \
+ drvp = &chp->ch_drive[drive]; \
+ \
+ if (drvp->drive_type != ATA_DRIVET_ATA && \
+ drvp->drive_type != ATA_DRIVET_ATAPI && \
+ drvp->drive_type != ATA_DRIVET_OLD) \
+ continue; \
+ \
+ if (drvp->drv_start != NULL) \
+ (*drvp->drv_start)(drvp->drv_softc); \
+ } while (0)
+
+ /*
+ * Process drives in round robin fashion starting with next one after
+ * the one which finished transfer. Thus no single drive would
+ * completely starve other drives on same channel.
+ * This loop processes all but the current drive, so won't do anything
+ * if there is only one drive in channel.
+ */
+ for (i = (drive + 1) % chp->ch_ndrives; i != drive;
+ i = (i + 1) % chp->ch_ndrives) {
+ ATA_DRIVE_START(chp, i);
+ }
+
+ /* Now try to kick off xfers on the current drive */
+ ATA_DRIVE_START(chp, drive);
+
+ splx(s);
+#undef ATA_DRIVE_START
+}
+
+void
+ata_channel_lock(struct ata_channel *chp)
+{
+ mutex_enter(&chp->ch_lock);
+}
+
+void
+ata_channel_unlock(struct ata_channel *chp)
+{
+ mutex_exit(&chp->ch_lock);
+}
+
+void
+ata_channel_lock_owned(struct ata_channel *chp)
+{
+ KASSERT(mutex_owned(&chp->ch_lock));
+}
