[PATCH char-misc-next v4 10/13] misc: mic: SCIF messaging and node enumeration APIs
SCIF messaging APIs which allow sending messages between the SCIF
endpoints via a byte stream based ring buffer which has been
optimized to avoid reads across PCIe. The SCIF messaging APIs
are typically used for short < 1024 byte messages for best
performance while the RDMA APIs which will be submitted in a future
patch series is recommended for larger transfers. The node
enumeration API enables a user to query for the number of nodes
online in the SCIF network and their node ids.
Reviewed-by: Ashutosh Dixit
Signed-off-by: Nikhil Rao
Signed-off-by: Sudeep Dutt
---
drivers/misc/mic/scif/scif_epd.h| 4 +
drivers/misc/mic/scif/scif_api.c| 402
drivers/misc/mic/scif/scif_epd.c| 32 +++
drivers/misc/mic/scif/scif_fd.c | 93 +
drivers/misc/mic/scif/scif_nm.c | 17 ++
drivers/misc/mic/scif/scif_nodeqp.c | 36 +++-
6 files changed, 583 insertions(+), 1 deletion(-)
diff --git a/drivers/misc/mic/scif/scif_epd.h b/drivers/misc/mic/scif/scif_epd.h
index 65c628e..331322a 100644
--- a/drivers/misc/mic/scif/scif_epd.h
+++ b/drivers/misc/mic/scif/scif_epd.h
@@ -144,6 +144,8 @@ int scif_rsrv_port(u16 port);
void scif_get_port(u16 port);
int scif_get_new_port(void);
void scif_put_port(u16 port);
+int scif_user_send(scif_epd_t epd, void __user *msg, int len, int flags);
+int scif_user_recv(scif_epd_t epd, void __user *msg, int len, int flags);
void scif_cnctreq(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_cnctgnt(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_cnctgnt_ack(struct scif_dev *scifdev, struct scifmsg *msg);
@@ -151,6 +153,8 @@ void scif_cnctgnt_nack(struct scif_dev *scifdev, struct
scifmsg *msg);
void scif_cnctrej(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_discnct(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_discnt_ack(struct scif_dev *scifdev, struct scifmsg *msg);
+void scif_clientsend(struct scif_dev *scifdev, struct scifmsg *msg);
+void scif_clientrcvd(struct scif_dev *scifdev, struct scifmsg *msg);
int __scif_connect(scif_epd_t epd, struct scif_port_id *dst, bool non_block);
int __scif_flush(scif_epd_t epd);
#endif /* SCIF_EPD_H */
diff --git a/drivers/misc/mic/scif/scif_api.c b/drivers/misc/mic/scif/scif_api.c
index da64cee..f39d313 100644
--- a/drivers/misc/mic/scif/scif_api.c
+++ b/drivers/misc/mic/scif/scif_api.c
@@ -872,3 +872,405 @@ scif_accept_error_epalloc:
return err;
}
EXPORT_SYMBOL_GPL(scif_accept);
+
+/*
+ * scif_msg_param_check:
+ * @epd: The end point returned from scif_open()
+ * @len: Length to receive
+ * @flags: blocking or non blocking
+ *
+ * Validate parameters for messaging APIs scif_send(..)/scif_recv(..).
+ */
+static inline int scif_msg_param_check(scif_epd_t epd, int len, int flags)
+{
+ int ret = -EINVAL;
+
+ if (len < 0)
+ goto err_ret;
+ if (flags && (!(flags & SCIF_RECV_BLOCK)))
+ goto err_ret;
+ ret = 0;
+err_ret:
+ return ret;
+}
+
+static int _scif_send(scif_epd_t epd, void *msg, int len, int flags)
+{
+ struct scif_endpt *ep = (struct scif_endpt *)epd;
+ struct scifmsg notif_msg;
+ int curr_xfer_len = 0, sent_len = 0, write_count;
+ int ret = 0;
+ struct scif_qp *qp = ep->qp_info.qp;
+
+ if (flags & SCIF_SEND_BLOCK)
+ might_sleep();
+
+ spin_lock(>lock);
+ while (sent_len != len && SCIFEP_CONNECTED == ep->state) {
+ write_count = scif_rb_space(>outbound_q);
+ if (write_count) {
+ /* Best effort to send as much data as possible */
+ curr_xfer_len = min(len - sent_len, write_count);
+ ret = scif_rb_write(>outbound_q, msg,
+ curr_xfer_len);
+ if (ret < 0)
+ break;
+ /* Success. Update write pointer */
+ scif_rb_commit(>outbound_q);
+ /*
+* Send a notification to the peer about the
+* produced data message.
+*/
+ notif_msg.src = ep->port;
+ notif_msg.uop = SCIF_CLIENT_SENT;
+ notif_msg.payload[0] = ep->remote_ep;
+ ret = _scif_nodeqp_send(ep->remote_dev, _msg);
+ if (ret)
+ break;
+ sent_len += curr_xfer_len;
+ msg = msg + curr_xfer_len;
+ continue;
+ }
+ curr_xfer_len = min(len - sent_len, SCIF_ENDPT_QP_SIZE - 1);
+ /* Not enough RB space. return for the Non Blocking case */
+ if (!(flags & SCIF_SEND_BLOCK))
+ break;
+
+ spin_unlock(>lock);
+ /* Wait for a SCIF_CLIENT_RCVD message in
[PATCH char-misc-next v4 10/13] misc: mic: SCIF messaging and node enumeration APIs
SCIF messaging APIs which allow sending messages between the SCIF
endpoints via a byte stream based ring buffer which has been
optimized to avoid reads across PCIe. The SCIF messaging APIs
are typically used for short 1024 byte messages for best
performance while the RDMA APIs which will be submitted in a future
patch series is recommended for larger transfers. The node
enumeration API enables a user to query for the number of nodes
online in the SCIF network and their node ids.
Reviewed-by: Ashutosh Dixit ashutosh.di...@intel.com
Signed-off-by: Nikhil Rao nikhil@intel.com
Signed-off-by: Sudeep Dutt sudeep.d...@intel.com
---
drivers/misc/mic/scif/scif_epd.h| 4 +
drivers/misc/mic/scif/scif_api.c| 402
drivers/misc/mic/scif/scif_epd.c| 32 +++
drivers/misc/mic/scif/scif_fd.c | 93 +
drivers/misc/mic/scif/scif_nm.c | 17 ++
drivers/misc/mic/scif/scif_nodeqp.c | 36 +++-
6 files changed, 583 insertions(+), 1 deletion(-)
diff --git a/drivers/misc/mic/scif/scif_epd.h b/drivers/misc/mic/scif/scif_epd.h
index 65c628e..331322a 100644
--- a/drivers/misc/mic/scif/scif_epd.h
+++ b/drivers/misc/mic/scif/scif_epd.h
@@ -144,6 +144,8 @@ int scif_rsrv_port(u16 port);
void scif_get_port(u16 port);
int scif_get_new_port(void);
void scif_put_port(u16 port);
+int scif_user_send(scif_epd_t epd, void __user *msg, int len, int flags);
+int scif_user_recv(scif_epd_t epd, void __user *msg, int len, int flags);
void scif_cnctreq(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_cnctgnt(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_cnctgnt_ack(struct scif_dev *scifdev, struct scifmsg *msg);
@@ -151,6 +153,8 @@ void scif_cnctgnt_nack(struct scif_dev *scifdev, struct
scifmsg *msg);
void scif_cnctrej(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_discnct(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_discnt_ack(struct scif_dev *scifdev, struct scifmsg *msg);
+void scif_clientsend(struct scif_dev *scifdev, struct scifmsg *msg);
+void scif_clientrcvd(struct scif_dev *scifdev, struct scifmsg *msg);
int __scif_connect(scif_epd_t epd, struct scif_port_id *dst, bool non_block);
int __scif_flush(scif_epd_t epd);
#endif /* SCIF_EPD_H */
diff --git a/drivers/misc/mic/scif/scif_api.c b/drivers/misc/mic/scif/scif_api.c
index da64cee..f39d313 100644
--- a/drivers/misc/mic/scif/scif_api.c
+++ b/drivers/misc/mic/scif/scif_api.c
@@ -872,3 +872,405 @@ scif_accept_error_epalloc:
return err;
}
EXPORT_SYMBOL_GPL(scif_accept);
+
+/*
+ * scif_msg_param_check:
+ * @epd: The end point returned from scif_open()
+ * @len: Length to receive
+ * @flags: blocking or non blocking
+ *
+ * Validate parameters for messaging APIs scif_send(..)/scif_recv(..).
+ */
+static inline int scif_msg_param_check(scif_epd_t epd, int len, int flags)
+{
+ int ret = -EINVAL;
+
+ if (len 0)
+ goto err_ret;
+ if (flags (!(flags SCIF_RECV_BLOCK)))
+ goto err_ret;
+ ret = 0;
+err_ret:
+ return ret;
+}
+
+static int _scif_send(scif_epd_t epd, void *msg, int len, int flags)
+{
+ struct scif_endpt *ep = (struct scif_endpt *)epd;
+ struct scifmsg notif_msg;
+ int curr_xfer_len = 0, sent_len = 0, write_count;
+ int ret = 0;
+ struct scif_qp *qp = ep-qp_info.qp;
+
+ if (flags SCIF_SEND_BLOCK)
+ might_sleep();
+
+ spin_lock(ep-lock);
+ while (sent_len != len SCIFEP_CONNECTED == ep-state) {
+ write_count = scif_rb_space(qp-outbound_q);
+ if (write_count) {
+ /* Best effort to send as much data as possible */
+ curr_xfer_len = min(len - sent_len, write_count);
+ ret = scif_rb_write(qp-outbound_q, msg,
+ curr_xfer_len);
+ if (ret 0)
+ break;
+ /* Success. Update write pointer */
+ scif_rb_commit(qp-outbound_q);
+ /*
+* Send a notification to the peer about the
+* produced data message.
+*/
+ notif_msg.src = ep-port;
+ notif_msg.uop = SCIF_CLIENT_SENT;
+ notif_msg.payload[0] = ep-remote_ep;
+ ret = _scif_nodeqp_send(ep-remote_dev, notif_msg);
+ if (ret)
+ break;
+ sent_len += curr_xfer_len;
+ msg = msg + curr_xfer_len;
+ continue;
+ }
+ curr_xfer_len = min(len - sent_len, SCIF_ENDPT_QP_SIZE - 1);
+ /* Not enough RB space. return for the Non Blocking case */
+ if (!(flags SCIF_SEND_BLOCK))
+ break;
+
+
[PATCH char-misc-next v4 10/13] misc: mic: SCIF messaging and node enumeration APIs
SCIF messaging APIs which allow sending messages between the SCIF
endpoints via a byte stream based ring buffer which has been
optimized to avoid reads across PCIe. The SCIF messaging APIs
are typically used for short < 1024 byte messages for best
performance while the RDMA APIs which will be submitted in a future
patch series is recommended for larger transfers. The node
enumeration API enables a user to query for the number of nodes
online in the SCIF network and their node ids.
Reviewed-by: Ashutosh Dixit
Signed-off-by: Nikhil Rao
Signed-off-by: Sudeep Dutt
---
drivers/misc/mic/scif/scif_epd.h| 4 +
drivers/misc/mic/scif/scif_api.c| 402
drivers/misc/mic/scif/scif_epd.c| 32 +++
drivers/misc/mic/scif/scif_fd.c | 93 +
drivers/misc/mic/scif/scif_nm.c | 17 ++
drivers/misc/mic/scif/scif_nodeqp.c | 36 +++-
6 files changed, 583 insertions(+), 1 deletion(-)
diff --git a/drivers/misc/mic/scif/scif_epd.h b/drivers/misc/mic/scif/scif_epd.h
index 65c628e..331322a 100644
--- a/drivers/misc/mic/scif/scif_epd.h
+++ b/drivers/misc/mic/scif/scif_epd.h
@@ -144,6 +144,8 @@ int scif_rsrv_port(u16 port);
void scif_get_port(u16 port);
int scif_get_new_port(void);
void scif_put_port(u16 port);
+int scif_user_send(scif_epd_t epd, void __user *msg, int len, int flags);
+int scif_user_recv(scif_epd_t epd, void __user *msg, int len, int flags);
void scif_cnctreq(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_cnctgnt(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_cnctgnt_ack(struct scif_dev *scifdev, struct scifmsg *msg);
@@ -151,6 +153,8 @@ void scif_cnctgnt_nack(struct scif_dev *scifdev, struct
scifmsg *msg);
void scif_cnctrej(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_discnct(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_discnt_ack(struct scif_dev *scifdev, struct scifmsg *msg);
+void scif_clientsend(struct scif_dev *scifdev, struct scifmsg *msg);
+void scif_clientrcvd(struct scif_dev *scifdev, struct scifmsg *msg);
int __scif_connect(scif_epd_t epd, struct scif_port_id *dst, bool non_block);
int __scif_flush(scif_epd_t epd);
#endif /* SCIF_EPD_H */
diff --git a/drivers/misc/mic/scif/scif_api.c b/drivers/misc/mic/scif/scif_api.c
index da64cee..f39d313 100644
--- a/drivers/misc/mic/scif/scif_api.c
+++ b/drivers/misc/mic/scif/scif_api.c
@@ -872,3 +872,405 @@ scif_accept_error_epalloc:
return err;
}
EXPORT_SYMBOL_GPL(scif_accept);
+
+/*
+ * scif_msg_param_check:
+ * @epd: The end point returned from scif_open()
+ * @len: Length to receive
+ * @flags: blocking or non blocking
+ *
+ * Validate parameters for messaging APIs scif_send(..)/scif_recv(..).
+ */
+static inline int scif_msg_param_check(scif_epd_t epd, int len, int flags)
+{
+ int ret = -EINVAL;
+
+ if (len < 0)
+ goto err_ret;
+ if (flags && (!(flags & SCIF_RECV_BLOCK)))
+ goto err_ret;
+ ret = 0;
+err_ret:
+ return ret;
+}
+
+static int _scif_send(scif_epd_t epd, void *msg, int len, int flags)
+{
+ struct scif_endpt *ep = (struct scif_endpt *)epd;
+ struct scifmsg notif_msg;
+ int curr_xfer_len = 0, sent_len = 0, write_count;
+ int ret = 0;
+ struct scif_qp *qp = ep->qp_info.qp;
+
+ if (flags & SCIF_SEND_BLOCK)
+ might_sleep();
+
+ spin_lock(>lock);
+ while (sent_len != len && SCIFEP_CONNECTED == ep->state) {
+ write_count = scif_rb_space(>outbound_q);
+ if (write_count) {
+ /* Best effort to send as much data as possible */
+ curr_xfer_len = min(len - sent_len, write_count);
+ ret = scif_rb_write(>outbound_q, msg,
+ curr_xfer_len);
+ if (ret < 0)
+ break;
+ /* Success. Update write pointer */
+ scif_rb_commit(>outbound_q);
+ /*
+* Send a notification to the peer about the
+* produced data message.
+*/
+ notif_msg.src = ep->port;
+ notif_msg.uop = SCIF_CLIENT_SENT;
+ notif_msg.payload[0] = ep->remote_ep;
+ ret = _scif_nodeqp_send(ep->remote_dev, _msg);
+ if (ret)
+ break;
+ sent_len += curr_xfer_len;
+ msg = msg + curr_xfer_len;
+ continue;
+ }
+ curr_xfer_len = min(len - sent_len, SCIF_ENDPT_QP_SIZE - 1);
+ /* Not enough RB space. return for the Non Blocking case */
+ if (!(flags & SCIF_SEND_BLOCK))
+ break;
+
+ spin_unlock(>lock);
+ /* Wait for a SCIF_CLIENT_RCVD message in
[PATCH char-misc-next v4 10/13] misc: mic: SCIF messaging and node enumeration APIs
SCIF messaging APIs which allow sending messages between the SCIF
endpoints via a byte stream based ring buffer which has been
optimized to avoid reads across PCIe. The SCIF messaging APIs
are typically used for short 1024 byte messages for best
performance while the RDMA APIs which will be submitted in a future
patch series is recommended for larger transfers. The node
enumeration API enables a user to query for the number of nodes
online in the SCIF network and their node ids.
Reviewed-by: Ashutosh Dixit ashutosh.di...@intel.com
Signed-off-by: Nikhil Rao nikhil@intel.com
Signed-off-by: Sudeep Dutt sudeep.d...@intel.com
---
drivers/misc/mic/scif/scif_epd.h| 4 +
drivers/misc/mic/scif/scif_api.c| 402
drivers/misc/mic/scif/scif_epd.c| 32 +++
drivers/misc/mic/scif/scif_fd.c | 93 +
drivers/misc/mic/scif/scif_nm.c | 17 ++
drivers/misc/mic/scif/scif_nodeqp.c | 36 +++-
6 files changed, 583 insertions(+), 1 deletion(-)
diff --git a/drivers/misc/mic/scif/scif_epd.h b/drivers/misc/mic/scif/scif_epd.h
index 65c628e..331322a 100644
--- a/drivers/misc/mic/scif/scif_epd.h
+++ b/drivers/misc/mic/scif/scif_epd.h
@@ -144,6 +144,8 @@ int scif_rsrv_port(u16 port);
void scif_get_port(u16 port);
int scif_get_new_port(void);
void scif_put_port(u16 port);
+int scif_user_send(scif_epd_t epd, void __user *msg, int len, int flags);
+int scif_user_recv(scif_epd_t epd, void __user *msg, int len, int flags);
void scif_cnctreq(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_cnctgnt(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_cnctgnt_ack(struct scif_dev *scifdev, struct scifmsg *msg);
@@ -151,6 +153,8 @@ void scif_cnctgnt_nack(struct scif_dev *scifdev, struct
scifmsg *msg);
void scif_cnctrej(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_discnct(struct scif_dev *scifdev, struct scifmsg *msg);
void scif_discnt_ack(struct scif_dev *scifdev, struct scifmsg *msg);
+void scif_clientsend(struct scif_dev *scifdev, struct scifmsg *msg);
+void scif_clientrcvd(struct scif_dev *scifdev, struct scifmsg *msg);
int __scif_connect(scif_epd_t epd, struct scif_port_id *dst, bool non_block);
int __scif_flush(scif_epd_t epd);
#endif /* SCIF_EPD_H */
diff --git a/drivers/misc/mic/scif/scif_api.c b/drivers/misc/mic/scif/scif_api.c
index da64cee..f39d313 100644
--- a/drivers/misc/mic/scif/scif_api.c
+++ b/drivers/misc/mic/scif/scif_api.c
@@ -872,3 +872,405 @@ scif_accept_error_epalloc:
return err;
}
EXPORT_SYMBOL_GPL(scif_accept);
+
+/*
+ * scif_msg_param_check:
+ * @epd: The end point returned from scif_open()
+ * @len: Length to receive
+ * @flags: blocking or non blocking
+ *
+ * Validate parameters for messaging APIs scif_send(..)/scif_recv(..).
+ */
+static inline int scif_msg_param_check(scif_epd_t epd, int len, int flags)
+{
+ int ret = -EINVAL;
+
+ if (len 0)
+ goto err_ret;
+ if (flags (!(flags SCIF_RECV_BLOCK)))
+ goto err_ret;
+ ret = 0;
+err_ret:
+ return ret;
+}
+
+static int _scif_send(scif_epd_t epd, void *msg, int len, int flags)
+{
+ struct scif_endpt *ep = (struct scif_endpt *)epd;
+ struct scifmsg notif_msg;
+ int curr_xfer_len = 0, sent_len = 0, write_count;
+ int ret = 0;
+ struct scif_qp *qp = ep-qp_info.qp;
+
+ if (flags SCIF_SEND_BLOCK)
+ might_sleep();
+
+ spin_lock(ep-lock);
+ while (sent_len != len SCIFEP_CONNECTED == ep-state) {
+ write_count = scif_rb_space(qp-outbound_q);
+ if (write_count) {
+ /* Best effort to send as much data as possible */
+ curr_xfer_len = min(len - sent_len, write_count);
+ ret = scif_rb_write(qp-outbound_q, msg,
+ curr_xfer_len);
+ if (ret 0)
+ break;
+ /* Success. Update write pointer */
+ scif_rb_commit(qp-outbound_q);
+ /*
+* Send a notification to the peer about the
+* produced data message.
+*/
+ notif_msg.src = ep-port;
+ notif_msg.uop = SCIF_CLIENT_SENT;
+ notif_msg.payload[0] = ep-remote_ep;
+ ret = _scif_nodeqp_send(ep-remote_dev, notif_msg);
+ if (ret)
+ break;
+ sent_len += curr_xfer_len;
+ msg = msg + curr_xfer_len;
+ continue;
+ }
+ curr_xfer_len = min(len - sent_len, SCIF_ENDPT_QP_SIZE - 1);
+ /* Not enough RB space. return for the Non Blocking case */
+ if (!(flags SCIF_SEND_BLOCK))
+ break;
+
+
