Author: np
Date: Sat Dec 26 00:26:02 2015
New Revision: 292736
URL: https://svnweb.freebsd.org/changeset/base/292736
Log:
cxgbe(4): Updates to the base NIC driver and t4_tom to support the iSCSI
offload driver. These changes come from projects/cxl_iscsi.
Modified:
head/sys/dev/cxgbe/adapter.h
head/sys/dev/cxgbe/offload.h
head/sys/dev/cxgbe/t4_main.c
head/sys/dev/cxgbe/tom/t4_cpl_io.c
head/sys/dev/cxgbe/tom/t4_ddp.c
head/sys/dev/cxgbe/tom/t4_tom.c
head/sys/dev/cxgbe/tom/t4_tom.h
Modified: head/sys/dev/cxgbe/adapter.h
==============================================================================
--- head/sys/dev/cxgbe/adapter.h Fri Dec 25 21:39:45 2015
(r292735)
+++ head/sys/dev/cxgbe/adapter.h Sat Dec 26 00:26:02 2015
(r292736)
@@ -766,7 +766,7 @@ struct adapter {
void *tom_softc; /* (struct tom_data *) */
struct tom_tunables tt;
void *iwarp_softc; /* (struct c4iw_dev *) */
- void *iscsi_softc;
+ void *iscsi_ulp_softc; /* (struct cxgbei_data *) */
#endif
struct l2t_data *l2t; /* L2 table */
struct tid_info tids;
Modified: head/sys/dev/cxgbe/offload.h
==============================================================================
--- head/sys/dev/cxgbe/offload.h Fri Dec 25 21:39:45 2015
(r292735)
+++ head/sys/dev/cxgbe/offload.h Sat Dec 26 00:26:02 2015
(r292736)
@@ -156,7 +156,7 @@ int t4_register_uld(struct uld_info *);
int t4_unregister_uld(struct uld_info *);
int t4_activate_uld(struct adapter *, int);
int t4_deactivate_uld(struct adapter *, int);
-void t4_iscsi_init(struct ifnet *, unsigned int, const unsigned int *);
+void t4_iscsi_init(struct adapter *, u_int, const u_int *);
int uld_active(struct adapter *, int);
#endif
#endif
Modified: head/sys/dev/cxgbe/t4_main.c
==============================================================================
--- head/sys/dev/cxgbe/t4_main.c Fri Dec 25 21:39:45 2015
(r292735)
+++ head/sys/dev/cxgbe/t4_main.c Sat Dec 26 00:26:02 2015
(r292736)
@@ -8801,11 +8801,8 @@ t4_ioctl(struct cdev *dev, unsigned long
#ifdef TCP_OFFLOAD
void
-t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
- const unsigned int *pgsz_order)
+t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order)
{
- struct vi_info *vi = ifp->if_softc;
- struct adapter *sc = vi->pi->adapter;
t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
Modified: head/sys/dev/cxgbe/tom/t4_cpl_io.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_cpl_io.c Fri Dec 25 21:39:45 2015
(r292735)
+++ head/sys/dev/cxgbe/tom/t4_cpl_io.c Sat Dec 26 00:26:02 2015
(r292736)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2012 Chelsio Communications, Inc.
+ * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
* All rights reserved.
* Written by: Navdeep Parhar <n...@freebsd.org>
*
@@ -71,33 +71,6 @@ VNET_DECLARE(int, tcp_autorcvbuf_inc);
VNET_DECLARE(int, tcp_autorcvbuf_max);
#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max)
-/*
- * For ULP connections HW may add headers, e.g., for digests, that aren't part
- * of the messages sent by the host but that are part of the TCP payload and
- * therefore consume TCP sequence space. Tx connection parameters that
- * operate in TCP sequence space are affected by the HW additions and need to
- * compensate for them to accurately track TCP sequence numbers. This array
- * contains the compensating extra lengths for ULP packets. It is indexed by
- * a packet's ULP submode.
- */
-const unsigned int t4_ulp_extra_len[] = {0, 4, 4, 8};
-
-/*
- * Return the length of any HW additions that will be made to a Tx packet.
- * Such additions can happen for some types of ULP packets.
- */
-static inline unsigned int
-ulp_extra_len(struct mbuf *m, int *ulp_mode)
-{
- struct m_tag *mtag;
-
- if ((mtag = m_tag_find(m, CXGBE_ISCSI_MBUF_TAG, NULL)) == NULL)
- return (0);
- *ulp_mode = *((int *)(mtag + 1));
-
- return (t4_ulp_extra_len[*ulp_mode & 3]);
-}
-
void
send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
{
@@ -384,13 +357,10 @@ t4_rcvd(struct toedev *tod, struct tcpcb
KASSERT(toep->sb_cc >= sbused(sb),
("%s: sb %p has more data (%d) than last time (%d).",
__func__, sb, sbused(sb), toep->sb_cc));
- if (toep->ulp_mode == ULP_MODE_ISCSI) {
- toep->rx_credits += toep->sb_cc;
- toep->sb_cc = 0;
- } else {
- toep->rx_credits += toep->sb_cc - sbused(sb);
- toep->sb_cc = sbused(sb);
- }
+
+ toep->rx_credits += toep->sb_cc - sbused(sb);
+ toep->sb_cc = sbused(sb);
+
if (toep->rx_credits > 0 &&
(tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 ||
(toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) ||
@@ -490,25 +460,16 @@ max_dsgl_nsegs(int tx_credits)
static inline void
write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
- unsigned int plen, uint8_t credits, int shove, int ulp_mode, int txalign)
+ unsigned int plen, uint8_t credits, int shove, int ulp_submode, int
txalign)
{
struct fw_ofld_tx_data_wr *txwr = dst;
- unsigned int wr_ulp_mode;
txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
V_FW_WR_IMMDLEN(immdlen));
txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
V_FW_WR_LEN16(credits));
-
- /* for iscsi, the mode & submode setting is per-packet */
- if (toep->ulp_mode == ULP_MODE_ISCSI)
- wr_ulp_mode = V_TX_ULP_MODE(ulp_mode >> 4) |
- V_TX_ULP_SUBMODE(ulp_mode & 3);
- else
- wr_ulp_mode = V_TX_ULP_MODE(toep->ulp_mode);
-
- txwr->lsodisable_to_flags = htobe32(wr_ulp_mode | V_TX_URG(0) | /*XXX*/
- V_TX_SHOVE(shove));
+ txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) |
+ V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove));
txwr->plen = htobe32(plen);
if (txalign > 0) {
@@ -616,6 +577,9 @@ t4_push_frames(struct adapter *sc, struc
toep->ulp_mode == ULP_MODE_RDMA,
("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
+ if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
+ return;
+
/*
* This function doesn't resume by itself. Someone else must clear the
* flag and call this function.
@@ -802,56 +766,70 @@ t4_push_frames(struct adapter *sc, struc
close_conn(sc, toep);
}
-/* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */
+static inline void
+rqdrop_locked(struct mbufq *q, int plen)
+{
+ struct mbuf *m;
+
+ while (plen > 0) {
+ m = mbufq_dequeue(q);
+
+ /* Too many credits. */
+ MPASS(m != NULL);
+ M_ASSERTPKTHDR(m);
+
+ /* Partial credits. */
+ MPASS(plen >= m->m_pkthdr.len);
+
+ plen -= m->m_pkthdr.len;
+ m_freem(m);
+ }
+}
+
void
-t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
+t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop)
{
- struct mbuf *sndptr, *m = NULL;
+ struct mbuf *sndptr, *m;
struct fw_ofld_tx_data_wr *txwr;
struct wrqe *wr;
- unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
+ u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
+ u_int adjusted_plen, ulp_submode;
struct inpcb *inp = toep->inp;
- struct tcpcb *tp;
- struct socket *so;
- struct sockbuf *sb;
- int tx_credits, ulp_len = 0, ulp_mode = 0, qlen = 0;
- int shove, compl;
- struct ofld_tx_sdesc *txsd;
+ struct tcpcb *tp = intotcpcb(inp);
+ int tx_credits, shove;
+ struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
+ struct mbufq *pduq = &toep->ulp_pduq;
+ static const u_int ulp_extra_len[] = {0, 4, 4, 8};
INP_WLOCK_ASSERT(inp);
- if (toep->flags & TPF_ABORT_SHUTDOWN)
- return;
-
- tp = intotcpcb(inp);
- so = inp->inp_socket;
- sb = &so->so_snd;
- txsd = &toep->txsd[toep->txsd_pidx];
-
KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
+ KASSERT(toep->ulp_mode == ULP_MODE_ISCSI,
+ ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
+
+ if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
+ return;
/*
* This function doesn't resume by itself. Someone else must clear the
* flag and call this function.
*/
- if (__predict_false(toep->flags & TPF_TX_SUSPENDED))
+ if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
+ KASSERT(drop == 0,
+ ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
return;
+ }
- sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen);
- if (!qlen)
- return;
+ if (drop)
+ rqdrop_locked(&toep->ulp_pdu_reclaimq, drop);
+
+ while ((sndptr = mbufq_first(pduq)) != NULL) {
+ M_ASSERTPKTHDR(sndptr);
- do {
tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
max_imm = max_imm_payload(tx_credits);
max_nsegs = max_dsgl_nsegs(tx_credits);
- if (drop) {
- t4_cpl_iscsi_callback(toep->td, toep, &drop,
- CPL_FW4_ACK);
- drop = 0;
- }
-
plen = 0;
nsegs = 0;
max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
@@ -861,7 +839,10 @@ t4_ulp_push_frames(struct adapter *sc, s
nsegs += n;
plen += m->m_len;
- /* This mbuf sent us _over_ the nsegs limit, return */
+ /*
+ * This mbuf would send us _over_ the nsegs limit.
+ * Suspend tx because the PDU can't be sent out.
+ */
if (plen > max_imm && nsegs > max_nsegs) {
toep->flags |= TPF_TX_SUSPENDED;
return;
@@ -869,30 +850,35 @@ t4_ulp_push_frames(struct adapter *sc, s
if (max_nsegs_1mbuf < n)
max_nsegs_1mbuf = n;
-
- /* This mbuf put us right at the max_nsegs limit */
- if (plen > max_imm && nsegs == max_nsegs) {
- toep->flags |= TPF_TX_SUSPENDED;
- return;
- }
- }
-
- shove = m == NULL && !(tp->t_flags & TF_MORETOCOME);
- /* nothing to send */
- if (plen == 0) {
- KASSERT(m == NULL,
- ("%s: nothing to send, but m != NULL", __func__));
- break;
}
if (__predict_false(toep->flags & TPF_FIN_SENT))
panic("%s: excess tx.", __func__);
- ulp_len = plen + ulp_extra_len(sndptr, &ulp_mode);
+ /*
+ * We have a PDU to send. All of it goes out in one WR so 'm'
+ * is NULL. A PDU's length is always a multiple of 4.
+ */
+ MPASS(m == NULL);
+ MPASS((plen & 3) == 0);
+ MPASS(sndptr->m_pkthdr.len == plen);
+
+ shove = !(tp->t_flags & TF_MORETOCOME);
+ ulp_submode = mbuf_ulp_submode(sndptr);
+ MPASS(ulp_submode < nitems(ulp_extra_len));
+
+ /*
+ * plen doesn't include header and data digests, which are
+ * generated and inserted in the right places by the TOE, but
+ * they do occupy TCP sequence space and need to be accounted
+ * for.
+ */
+ adjusted_plen = plen + ulp_extra_len[ulp_submode];
if (plen <= max_imm) {
/* Immediate data tx */
- wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16),
+
+ wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16),
toep->ofld_txq);
if (wr == NULL) {
/* XXX: how will we recover from this? */
@@ -901,16 +887,17 @@ t4_ulp_push_frames(struct adapter *sc, s
}
txwr = wrtod(wr);
credits = howmany(wr->wr_len, 16);
- write_tx_wr(txwr, toep, plen, ulp_len, credits, shove,
- ulp_mode, 0);
+ write_tx_wr(txwr, toep, plen, adjusted_plen, credits,
+ shove, ulp_submode, sc->tt.tx_align);
m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
+ nsegs = 0;
} else {
int wr_len;
/* DSGL tx */
wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
- wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq);
+ wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
if (wr == NULL) {
/* XXX: how will we recover from this? */
toep->flags |= TPF_TX_SUSPENDED;
@@ -918,8 +905,8 @@ t4_ulp_push_frames(struct adapter *sc, s
}
txwr = wrtod(wr);
credits = howmany(wr_len, 16);
- write_tx_wr(txwr, toep, 0, ulp_len, credits, shove,
- ulp_mode, 0);
+ write_tx_wr(txwr, toep, 0, adjusted_plen, credits,
+ shove, ulp_submode, sc->tt.tx_align);
write_tx_sgl(txwr + 1, sndptr, m, nsegs,
max_nsegs_1mbuf);
if (wr_len & 0xf) {
@@ -932,28 +919,26 @@ t4_ulp_push_frames(struct adapter *sc, s
KASSERT(toep->tx_credits >= credits,
("%s: not enough credits", __func__));
+ m = mbufq_dequeue(pduq);
+ MPASS(m == sndptr);
+ mbufq_enqueue(&toep->ulp_pdu_reclaimq, m);
+
toep->tx_credits -= credits;
toep->tx_nocompl += credits;
toep->plen_nocompl += plen;
if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
- toep->tx_nocompl >= toep->tx_total / 4)
- compl = 1;
-
- if (compl) {
+ toep->tx_nocompl >= toep->tx_total / 4) {
txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL);
toep->tx_nocompl = 0;
toep->plen_nocompl = 0;
}
- tp->snd_nxt += ulp_len;
- tp->snd_max += ulp_len;
- /* goto next mbuf */
- sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen);
+ tp->snd_nxt += adjusted_plen;
+ tp->snd_max += adjusted_plen;
toep->flags |= TPF_TX_DATA_SENT;
- if (toep->tx_credits < MIN_OFLD_TX_CREDITS) {
+ if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
toep->flags |= TPF_TX_SUSPENDED;
- }
KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
txsd->plen = plen;
@@ -966,10 +951,10 @@ t4_ulp_push_frames(struct adapter *sc, s
toep->txsd_avail--;
t4_l2t_send(sc, wr, toep->l2te);
- } while (m != NULL);
+ }
- /* Send a FIN if requested, but only if there's no more data to send */
- if (m == NULL && toep->flags & TPF_SEND_FIN)
+ /* Send a FIN if requested, but only if there are no more PDUs to send
*/
+ if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN)
close_conn(sc, toep);
}
@@ -987,7 +972,10 @@ t4_tod_output(struct toedev *tod, struct
("%s: inp %p dropped.", __func__, inp));
KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
- t4_push_frames(sc, toep, 0);
+ if (toep->ulp_mode == ULP_MODE_ISCSI)
+ t4_push_pdus(sc, toep, 0);
+ else
+ t4_push_frames(sc, toep, 0);
return (0);
}
@@ -1009,7 +997,7 @@ t4_send_fin(struct toedev *tod, struct t
toep->flags |= TPF_SEND_FIN;
if (tp->t_state >= TCPS_ESTABLISHED) {
if (toep->ulp_mode == ULP_MODE_ISCSI)
- t4_ulp_push_frames(sc, toep, 0);
+ t4_push_pdus(sc, toep, 0);
else
t4_push_frames(sc, toep, 0);
}
@@ -1250,91 +1238,6 @@ abort_status_to_errno(struct tcpcb *tp,
}
}
-int
-cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *);
-/*
- * tom_cpl_iscsi_callback -
- * iscsi and tom would share the following cpl messages, so when any of these
- * message is received, after tom is done with processing it, the messages
- * needs to be forwarded to iscsi for further processing:
- * - CPL_SET_TCB_RPL
- * - CPL_RX_DATA_DDP
- */
-void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *,
- unsigned int);
-
-struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *);
-/*
- * Check if the handler function is set for a given CPL
- * return 0 if the function is NULL or cpl_not_handled, 1 otherwise.
- */
-int
-t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode)
-{
-
- MPASS(opcode < nitems(sc->cpl_handler));
-
- return (sc->cpl_handler[opcode] &&
- sc->cpl_handler[opcode] != cpl_not_handled);
-}
-
-/*
- * set the tom_cpl_iscsi_callback function, this function should be used
- * whenever both toe and iscsi need to process the same cpl msg.
- */
-void
-t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket
*,
- void *, unsigned int))
-{
-
- tom_cpl_iscsi_callback = fp;
-}
-
-void
-t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
- unsigned int, int *qlen))
-{
-
- tom_queue_iscsi_callback = fp;
-}
-
-int
-t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m,
- unsigned int opcode)
-{
- struct socket *so;
-
- if (opcode == CPL_FW4_ACK)
- so = toep->inp->inp_socket;
- else {
- INP_WLOCK(toep->inp);
- so = toep->inp->inp_socket;
- INP_WUNLOCK(toep->inp);
- }
-
- if (tom_cpl_iscsi_callback && so) {
- if (toep->ulp_mode == ULP_MODE_ISCSI) {
- tom_cpl_iscsi_callback(td, so, m, opcode);
- return (0);
- }
- }
-
- return (1);
-}
-
-struct mbuf *
-t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep,
- unsigned int cmd, int *qlen)
-{
-
- if (tom_queue_iscsi_callback && so) {
- if (toep->ulp_mode == ULP_MODE_ISCSI)
- return (tom_queue_iscsi_callback(so, cmd, qlen));
- }
-
- return (NULL);
-}
-
/*
* TCP RST from the peer, timeout, or some other such critical error.
*/
@@ -1733,21 +1636,34 @@ do_fw4_ack(struct sge_iq *iq, const stru
toep->tx_credits >= toep->tx_total / 4) {
toep->flags &= ~TPF_TX_SUSPENDED;
if (toep->ulp_mode == ULP_MODE_ISCSI)
- t4_ulp_push_frames(sc, toep, plen);
+ t4_push_pdus(sc, toep, plen);
else
t4_push_frames(sc, toep, plen);
} else if (plen > 0) {
struct sockbuf *sb = &so->so_snd;
+ int sbu;
- if (toep->ulp_mode == ULP_MODE_ISCSI)
- t4_cpl_iscsi_callback(toep->td, toep, &plen,
- CPL_FW4_ACK);
- else {
- SOCKBUF_LOCK(sb);
+ SOCKBUF_LOCK(sb);
+ sbu = sbused(sb);
+ if (toep->ulp_mode == ULP_MODE_ISCSI) {
+
+ if (__predict_false(sbu > 0)) {
+ /*
+ * The data trasmitted before the tid's ULP mode
+ * changed to ISCSI is still in so_snd.
+ * Incoming credits should account for so_snd
+ * first.
+ */
+ sbdrop_locked(sb, min(sbu, plen));
+ plen -= min(sbu, plen);
+ }
+ sowwakeup_locked(so); /* unlocks so_snd */
+ rqdrop_locked(&toep->ulp_pdu_reclaimq, plen);
+ } else {
sbdrop_locked(sb, plen);
- sowwakeup_locked(so);
- SOCKBUF_UNLOCK_ASSERT(sb);
+ sowwakeup_locked(so); /* unlocks so_snd */
}
+ SOCKBUF_UNLOCK_ASSERT(sb);
}
INP_WUNLOCK(inp);
@@ -1771,14 +1687,21 @@ do_set_tcb_rpl(struct sge_iq *iq, const
if (is_ftid(sc, tid))
return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */
- else {
- struct toepcb *toep = lookup_tid(sc, tid);
- t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL);
- return (0);
- }
+ /*
+ * TOM and/or other ULPs don't request replies for CPL_SET_TCB or
+ * CPL_SET_TCB_FIELD requests. This can easily change and when it does
+ * the dispatch code will go here.
+ */
+#ifdef INVARIANTS
+ panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__,
+ tid, iq);
+#else
+ log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n",
+ __func__, tid, iq);
+#endif
- CXGBE_UNIMPLEMENTED(__func__);
+ return (0);
}
void
Modified: head/sys/dev/cxgbe/tom/t4_ddp.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_ddp.c Fri Dec 25 21:39:45 2015
(r292735)
+++ head/sys/dev/cxgbe/tom/t4_ddp.c Sat Dec 26 00:26:02 2015
(r292736)
@@ -513,7 +513,6 @@ do_rx_data_ddp(struct sge_iq *iq, const
unsigned int tid = GET_TID(cpl);
uint32_t vld;
struct toepcb *toep = lookup_tid(sc, tid);
- struct tom_data *td = toep->td;
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
@@ -525,16 +524,11 @@ do_rx_data_ddp(struct sge_iq *iq, const
panic("%s: DDP error 0x%x (tid %d, toep %p)",
__func__, vld, tid, toep);
}
+
if (toep->ulp_mode == ULP_MODE_ISCSI) {
- m = m_get(M_NOWAIT, MT_DATA);
- if (m == NULL)
- CXGBE_UNIMPLEMENTED("mbuf alloc failure");
- memcpy(mtod(m, unsigned char *), cpl,
- sizeof(struct cpl_rx_data_ddp));
- if (!t4_cpl_iscsi_callback(td, toep, m, CPL_RX_DATA_DDP))
- return (0);
- m_freem(m);
- }
+ sc->cpl_handler[CPL_RX_ISCSI_DDP](iq, rss, m);
+ return (0);
+ }
handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len));
Modified: head/sys/dev/cxgbe/tom/t4_tom.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_tom.c Fri Dec 25 21:39:45 2015
(r292735)
+++ head/sys/dev/cxgbe/tom/t4_tom.c Sat Dec 26 00:26:02 2015
(r292736)
@@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
+#include <sys/limits.h>
#include <sys/module.h>
#include <sys/protosw.h>
#include <sys/domain.h>
@@ -158,6 +159,8 @@ alloc_toepcb(struct vi_info *vi, int txq
toep->ofld_txq = &sc->sge.ofld_txq[txqid];
toep->ofld_rxq = &sc->sge.ofld_rxq[rxqid];
toep->ctrlq = &sc->sge.ctrlq[pi->port_id];
+ mbufq_init(&toep->ulp_pduq, INT_MAX);
+ mbufq_init(&toep->ulp_pdu_reclaimq, INT_MAX);
toep->txsd_total = txsd_total;
toep->txsd_avail = txsd_total;
toep->txsd_pidx = 0;
@@ -273,6 +276,14 @@ release_offload_resources(struct toepcb
CTR5(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p, ce %p)",
__func__, toep, tid, toep->l2te, toep->ce);
+ /*
+ * These queues should have been emptied at approximately the same time
+ * that a normal connection's socket's so_snd would have been purged or
+ * drained. Do _not_ clean up here.
+ */
+ MPASS(mbufq_len(&toep->ulp_pduq) == 0);
+ MPASS(mbufq_len(&toep->ulp_pdu_reclaimq) == 0);
+
if (toep->ulp_mode == ULP_MODE_TCPDDP)
release_ddp_resources(toep);
@@ -380,6 +391,7 @@ final_cpl_received(struct toepcb *toep)
toep->inp = NULL;
toep->flags &= ~TPF_CPL_PENDING;
+ mbufq_drain(&toep->ulp_pdu_reclaimq);
if (!(toep->flags & TPF_ATTACHED))
release_offload_resources(toep);
Modified: head/sys/dev/cxgbe/tom/t4_tom.h
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_tom.h Fri Dec 25 21:39:45 2015
(r292735)
+++ head/sys/dev/cxgbe/tom/t4_tom.h Sat Dec 26 00:26:02 2015
(r292736)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2012 Chelsio Communications, Inc.
+ * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
* All rights reserved.
* Written by: Navdeep Parhar <n...@freebsd.org>
*
@@ -115,6 +115,10 @@ struct toepcb {
int rx_credits; /* rx credits (in bytes) to be returned to hw */
u_int ulp_mode; /* ULP mode */
+ void *ulpcb;
+ void *ulpcb2;
+ struct mbufq ulp_pduq; /* PDUs waiting to be sent out. */
+ struct mbufq ulp_pdu_reclaimq;
u_int ddp_flags;
struct ddp_buffer *db[2];
@@ -220,6 +224,22 @@ td_adapter(struct tom_data *td)
return (td->tod.tod_softc);
}
+static inline void
+set_mbuf_ulp_submode(struct mbuf *m, uint8_t ulp_submode)
+{
+
+ M_ASSERTPKTHDR(m);
+ m->m_pkthdr.PH_per.eight[0] = ulp_submode;
+}
+
+static inline uint8_t
+mbuf_ulp_submode(struct mbuf *m)
+{
+
+ M_ASSERTPKTHDR(m);
+ return (m->m_pkthdr.PH_per.eight[0]);
+}
+
/* t4_tom.c */
struct toepcb *alloc_toepcb(struct vi_info *, int, int, int);
void free_toepcb(struct toepcb *);
@@ -275,6 +295,7 @@ int t4_send_rst(struct toedev *, struct
void t4_set_tcb_field(struct adapter *, struct toepcb *, int, uint16_t,
uint64_t, uint64_t);
void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop);
+void t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop);
/* t4_ddp.c */
void t4_init_ddp(struct adapter *, struct tom_data *);
@@ -287,19 +308,4 @@ void handle_ddp_close(struct toepcb *, s
uint32_t);
void insert_ddp_data(struct toepcb *, uint32_t);
-/* ULP related */
-#define CXGBE_ISCSI_MBUF_TAG 50
-int t4tom_cpl_handler_registered(struct adapter *, unsigned int);
-void t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *,
- struct socket *, void *, unsigned int));
-void t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
- unsigned int, int *));
-void t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int);
-int t4_cpl_iscsi_callback(struct tom_data *, struct toepcb *, void *,
uint32_t);
-struct mbuf *t4_queue_iscsi_callback(struct socket *, struct toepcb *,
uint32_t,
- int *);
-extern void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *,
- void *, unsigned int);
-extern struct mbuf *(*tom_queue_iscsi_callback)(struct socket*, unsigned int,
- int *);
#endif
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
