[PATCH v3 6/9] NTB: Add Messaging NTB API

[Serge Semin]

Some IDT NTB-capable PCIe-switches have message registers to communicate with
peer devices. This patch adds new NTB API callback methods, which can be used
to utilize these registers functionality:
 ntb_msg_count(); - get number of message registers
 ntb_msg_inbits(); - get bitfield of inbound message registers status
 ntb_msg_outbits(); - get bitfield of outbound message registers status
 ntb_msg_read_sts(); - read the inbound and outbound message registers status
 ntb_msg_clear_sts(); - clear status bits of message registers
 ntb_msg_set_mask(); - mask interrupts raised by status bits of message
registers.
 ntb_msg_clear_mask(); - clear interrupts mask bits of message registers
 ntb_msg_read(midx, *pidx); - read message register with specified index,
additionally getting peer port index which data received from
 ntb_msg_write(midx, pidx); - write data to the specified message register
sending it to the passed peer device connected over a pidx port
 ntb_msg_event(); - notify driver context of a new message event

Of course there is hardware which doesn't support Message registers, so
this API is made optional.

Acked-by: Allen Hubbe 
Signed-off-by: Serge Semin 
---
 drivers/ntb/ntb.c   |  13 
 include/linux/ntb.h | 205 
 2 files changed, 218 insertions(+)

diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index 2551bb2..03b80d8 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -193,6 +193,19 @@ void ntb_db_event(struct ntb_dev *ntb, int vector)
 }
 EXPORT_SYMBOL(ntb_db_event);
 
+void ntb_msg_event(struct ntb_dev *ntb)
+{
+   unsigned long irqflags;
+
+   spin_lock_irqsave(>ctx_lock, irqflags);
+   {
+   if (ntb->ctx_ops && ntb->ctx_ops->msg_event)
+   ntb->ctx_ops->msg_event(ntb->ctx);
+   }
+   spin_unlock_irqrestore(>ctx_lock, irqflags);
+}
+EXPORT_SYMBOL(ntb_msg_event);
+
 int ntb_default_port_number(struct ntb_dev *ntb)
 {
switch (ntb->topo) {
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index a54e2be..76c56d5 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -178,10 +178,12 @@ static inline int ntb_client_ops_is_valid(const struct 
ntb_client_ops *ops)
  * struct ntb_ctx_ops - ntb driver context operations
  * @link_event:See ntb_link_event().
  * @db_event:  See ntb_db_event().
+ * @msg_event: See ntb_msg_event().
  */
 struct ntb_ctx_ops {
void (*link_event)(void *ctx);
void (*db_event)(void *ctx, int db_vector);
+   void (*msg_event)(void *ctx);
 };
 
 static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
@@ -190,6 +192,7 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
return
/* ops->link_event  && */
/* ops->db_event&& */
+   /* ops->msg_event   && */
1;
 }
 
@@ -234,6 +237,15 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @peer_spad_addr:See ntb_peer_spad_addr().
  * @peer_spad_read:See ntb_peer_spad_read().
  * @peer_spad_write:   See ntb_peer_spad_write().
+ * @msg_count: See ntb_msg_count().
+ * @msg_inbits:See ntb_msg_inbits().
+ * @msg_outbits:   See ntb_msg_outbits().
+ * @msg_read_sts:  See ntb_msg_read_sts().
+ * @msg_clear_sts: See ntb_msg_clear_sts().
+ * @msg_set_mask:  See ntb_msg_set_mask().
+ * @msg_clear_mask:See ntb_msg_clear_mask().
+ * @msg_read:  See ntb_msg_read().
+ * @msg_write: See ntb_msg_write().
  */
 struct ntb_dev_ops {
int (*port_number)(struct ntb_dev *ntb);
@@ -296,6 +308,16 @@ struct ntb_dev_ops {
u32 (*peer_spad_read)(struct ntb_dev *ntb, int pidx, int sidx);
int (*peer_spad_write)(struct ntb_dev *ntb, int pidx, int sidx,
   u32 val);
+
+   int (*msg_count)(struct ntb_dev *ntb);
+   u64 (*msg_inbits)(struct ntb_dev *ntb);
+   u64 (*msg_outbits)(struct ntb_dev *ntb);
+   u64 (*msg_read_sts)(struct ntb_dev *ntb);
+   int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits);
+   int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_read)(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg);
+   int (*msg_write)(struct ntb_dev *ntb, int midx, int pidx, u32 msg);
 };
 
 static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
@@ -342,6 +364,15 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* !ops->peer_spad_addr == !ops->spad_count && */
/* !ops->peer_spad_read == !ops->spad_count && */
!ops->peer_spad_write == !ops->spad_count   &&
+
+   !ops->msg_inbits == !ops->msg_count &&
+   

[PATCH v3 6/9] NTB: Add Messaging NTB API

[Serge Semin]

Some IDT NTB-capable PCIe-switches have message registers to communicate with
peer devices. This patch adds new NTB API callback methods, which can be used
to utilize these registers functionality:
 ntb_msg_count(); - get number of message registers
 ntb_msg_inbits(); - get bitfield of inbound message registers status
 ntb_msg_outbits(); - get bitfield of outbound message registers status
 ntb_msg_read_sts(); - read the inbound and outbound message registers status
 ntb_msg_clear_sts(); - clear status bits of message registers
 ntb_msg_set_mask(); - mask interrupts raised by status bits of message
registers.
 ntb_msg_clear_mask(); - clear interrupts mask bits of message registers
 ntb_msg_read(midx, *pidx); - read message register with specified index,
additionally getting peer port index which data received from
 ntb_msg_write(midx, pidx); - write data to the specified message register
sending it to the passed peer device connected over a pidx port
 ntb_msg_event(); - notify driver context of a new message event

Of course there is hardware which doesn't support Message registers, so
this API is made optional.

Acked-by: Allen Hubbe 
Signed-off-by: Serge Semin 
---
 drivers/ntb/ntb.c   |  13 
 include/linux/ntb.h | 205 
 2 files changed, 218 insertions(+)

diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index 2551bb2..03b80d8 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -193,6 +193,19 @@ void ntb_db_event(struct ntb_dev *ntb, int vector)
 }
 EXPORT_SYMBOL(ntb_db_event);
 
+void ntb_msg_event(struct ntb_dev *ntb)
+{
+   unsigned long irqflags;
+
+   spin_lock_irqsave(>ctx_lock, irqflags);
+   {
+   if (ntb->ctx_ops && ntb->ctx_ops->msg_event)
+   ntb->ctx_ops->msg_event(ntb->ctx);
+   }
+   spin_unlock_irqrestore(>ctx_lock, irqflags);
+}
+EXPORT_SYMBOL(ntb_msg_event);
+
 int ntb_default_port_number(struct ntb_dev *ntb)
 {
switch (ntb->topo) {
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index a54e2be..76c56d5 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -178,10 +178,12 @@ static inline int ntb_client_ops_is_valid(const struct 
ntb_client_ops *ops)
  * struct ntb_ctx_ops - ntb driver context operations
  * @link_event:See ntb_link_event().
  * @db_event:  See ntb_db_event().
+ * @msg_event: See ntb_msg_event().
  */
 struct ntb_ctx_ops {
void (*link_event)(void *ctx);
void (*db_event)(void *ctx, int db_vector);
+   void (*msg_event)(void *ctx);
 };
 
 static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
@@ -190,6 +192,7 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
return
/* ops->link_event  && */
/* ops->db_event&& */
+   /* ops->msg_event   && */
1;
 }
 
@@ -234,6 +237,15 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @peer_spad_addr:See ntb_peer_spad_addr().
  * @peer_spad_read:See ntb_peer_spad_read().
  * @peer_spad_write:   See ntb_peer_spad_write().
+ * @msg_count: See ntb_msg_count().
+ * @msg_inbits:See ntb_msg_inbits().
+ * @msg_outbits:   See ntb_msg_outbits().
+ * @msg_read_sts:  See ntb_msg_read_sts().
+ * @msg_clear_sts: See ntb_msg_clear_sts().
+ * @msg_set_mask:  See ntb_msg_set_mask().
+ * @msg_clear_mask:See ntb_msg_clear_mask().
+ * @msg_read:  See ntb_msg_read().
+ * @msg_write: See ntb_msg_write().
  */
 struct ntb_dev_ops {
int (*port_number)(struct ntb_dev *ntb);
@@ -296,6 +308,16 @@ struct ntb_dev_ops {
u32 (*peer_spad_read)(struct ntb_dev *ntb, int pidx, int sidx);
int (*peer_spad_write)(struct ntb_dev *ntb, int pidx, int sidx,
   u32 val);
+
+   int (*msg_count)(struct ntb_dev *ntb);
+   u64 (*msg_inbits)(struct ntb_dev *ntb);
+   u64 (*msg_outbits)(struct ntb_dev *ntb);
+   u64 (*msg_read_sts)(struct ntb_dev *ntb);
+   int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits);
+   int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_read)(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg);
+   int (*msg_write)(struct ntb_dev *ntb, int midx, int pidx, u32 msg);
 };
 
 static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
@@ -342,6 +364,15 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* !ops->peer_spad_addr == !ops->spad_count && */
/* !ops->peer_spad_read == !ops->spad_count && */
!ops->peer_spad_write == !ops->spad_count   &&
+
+   !ops->msg_inbits == !ops->msg_count &&
+   !ops->msg_outbits == !ops->msg_count

[PATCH v3 6/9] NTB: Add Messaging NTB API

[Serge Semin]

Some IDT NTB-capable PCIe-switches have message registers to communicate with
peer devices. This patch adds new NTB API callback methods, which can be used
to utilize these registers functionality:
 ntb_msg_count(); - get number of message registers
 ntb_msg_inbits(); - get bitfield of inbound message registers status
 ntb_msg_outbits(); - get bitfield of outbound message registers status
 ntb_msg_read_sts(); - read the inbound and outbound message registers status
 ntb_msg_clear_sts(); - clear status bits of message registers
 ntb_msg_set_mask(); - mask interrupts raised by status bits of message
registers.
 ntb_msg_clear_mask(); - clear interrupts mask bits of message registers
 ntb_msg_read(midx, *pidx); - read message register with specified index,
additionally getting peer port index which data received from
 ntb_msg_write(midx, pidx); - write data to the specified message register
sending it to the passed peer device connected over a pidx port
 ntb_msg_event(); - notify driver context of a new message event

Of course there is hardware which doesn't support Message registers, so
this API is made optional.

Acked-by: Allen Hubbe 
Signed-off-by: Serge Semin 
---
 drivers/ntb/ntb.c   |  13 
 include/linux/ntb.h | 205 
 2 files changed, 218 insertions(+)

diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index 2551bb2..03b80d8 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -193,6 +193,19 @@ void ntb_db_event(struct ntb_dev *ntb, int vector)
 }
 EXPORT_SYMBOL(ntb_db_event);
 
+void ntb_msg_event(struct ntb_dev *ntb)
+{
+   unsigned long irqflags;
+
+   spin_lock_irqsave(>ctx_lock, irqflags);
+   {
+   if (ntb->ctx_ops && ntb->ctx_ops->msg_event)
+   ntb->ctx_ops->msg_event(ntb->ctx);
+   }
+   spin_unlock_irqrestore(>ctx_lock, irqflags);
+}
+EXPORT_SYMBOL(ntb_msg_event);
+
 int ntb_default_port_number(struct ntb_dev *ntb)
 {
switch (ntb->topo) {
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index a54e2be..76c56d5 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -178,10 +178,12 @@ static inline int ntb_client_ops_is_valid(const struct 
ntb_client_ops *ops)
  * struct ntb_ctx_ops - ntb driver context operations
  * @link_event:See ntb_link_event().
  * @db_event:  See ntb_db_event().
+ * @msg_event: See ntb_msg_event().
  */
 struct ntb_ctx_ops {
void (*link_event)(void *ctx);
void (*db_event)(void *ctx, int db_vector);
+   void (*msg_event)(void *ctx);
 };
 
 static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
@@ -190,6 +192,7 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
return
/* ops->link_event  && */
/* ops->db_event&& */
+   /* ops->msg_event   && */
1;
 }
 
@@ -234,6 +237,15 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @peer_spad_addr:See ntb_peer_spad_addr().
  * @peer_spad_read:See ntb_peer_spad_read().
  * @peer_spad_write:   See ntb_peer_spad_write().
+ * @msg_count: See ntb_msg_count().
+ * @msg_inbits:See ntb_msg_inbits().
+ * @msg_outbits:   See ntb_msg_outbits().
+ * @msg_read_sts:  See ntb_msg_read_sts().
+ * @msg_clear_sts: See ntb_msg_clear_sts().
+ * @msg_set_mask:  See ntb_msg_set_mask().
+ * @msg_clear_mask:See ntb_msg_clear_mask().
+ * @msg_read:  See ntb_msg_read().
+ * @msg_write: See ntb_msg_write().
  */
 struct ntb_dev_ops {
int (*port_number)(struct ntb_dev *ntb);
@@ -296,6 +308,16 @@ struct ntb_dev_ops {
u32 (*peer_spad_read)(struct ntb_dev *ntb, int pidx, int sidx);
int (*peer_spad_write)(struct ntb_dev *ntb, int pidx, int sidx,
   u32 val);
+
+   int (*msg_count)(struct ntb_dev *ntb);
+   u64 (*msg_inbits)(struct ntb_dev *ntb);
+   u64 (*msg_outbits)(struct ntb_dev *ntb);
+   u64 (*msg_read_sts)(struct ntb_dev *ntb);
+   int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits);
+   int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_read)(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg);
+   int (*msg_write)(struct ntb_dev *ntb, int midx, int pidx, u32 msg);
 };
 
 static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
@@ -342,6 +364,15 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* !ops->peer_spad_addr == !ops->spad_count && */
/* !ops->peer_spad_read == !ops->spad_count && */
!ops->peer_spad_write == !ops->spad_count   &&
+
+   !ops->msg_inbits == !ops->msg_count &&
+   

[PATCH v3 6/9] NTB: Add Messaging NTB API

[Serge Semin]

Some IDT NTB-capable PCIe-switches have message registers to communicate with
peer devices. This patch adds new NTB API callback methods, which can be used
to utilize these registers functionality:
 ntb_msg_count(); - get number of message registers
 ntb_msg_inbits(); - get bitfield of inbound message registers status
 ntb_msg_outbits(); - get bitfield of outbound message registers status
 ntb_msg_read_sts(); - read the inbound and outbound message registers status
 ntb_msg_clear_sts(); - clear status bits of message registers
 ntb_msg_set_mask(); - mask interrupts raised by status bits of message
registers.
 ntb_msg_clear_mask(); - clear interrupts mask bits of message registers
 ntb_msg_read(midx, *pidx); - read message register with specified index,
additionally getting peer port index which data received from
 ntb_msg_write(midx, pidx); - write data to the specified message register
sending it to the passed peer device connected over a pidx port
 ntb_msg_event(); - notify driver context of a new message event

Of course there is hardware which doesn't support Message registers, so
this API is made optional.

Acked-by: Allen Hubbe 
Signed-off-by: Serge Semin 
---
 drivers/ntb/ntb.c   |  13 
 include/linux/ntb.h | 205 
 2 files changed, 218 insertions(+)

diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index 2551bb2..03b80d8 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -193,6 +193,19 @@ void ntb_db_event(struct ntb_dev *ntb, int vector)
 }
 EXPORT_SYMBOL(ntb_db_event);
 
+void ntb_msg_event(struct ntb_dev *ntb)
+{
+   unsigned long irqflags;
+
+   spin_lock_irqsave(>ctx_lock, irqflags);
+   {
+   if (ntb->ctx_ops && ntb->ctx_ops->msg_event)
+   ntb->ctx_ops->msg_event(ntb->ctx);
+   }
+   spin_unlock_irqrestore(>ctx_lock, irqflags);
+}
+EXPORT_SYMBOL(ntb_msg_event);
+
 int ntb_default_port_number(struct ntb_dev *ntb)
 {
switch (ntb->topo) {
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index a54e2be..76c56d5 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -178,10 +178,12 @@ static inline int ntb_client_ops_is_valid(const struct 
ntb_client_ops *ops)
  * struct ntb_ctx_ops - ntb driver context operations
  * @link_event:See ntb_link_event().
  * @db_event:  See ntb_db_event().
+ * @msg_event: See ntb_msg_event().
  */
 struct ntb_ctx_ops {
void (*link_event)(void *ctx);
void (*db_event)(void *ctx, int db_vector);
+   void (*msg_event)(void *ctx);
 };
 
 static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
@@ -190,6 +192,7 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
return
/* ops->link_event  && */
/* ops->db_event&& */
+   /* ops->msg_event   && */
1;
 }
 
@@ -234,6 +237,15 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @peer_spad_addr:See ntb_peer_spad_addr().
  * @peer_spad_read:See ntb_peer_spad_read().
  * @peer_spad_write:   See ntb_peer_spad_write().
+ * @msg_count: See ntb_msg_count().
+ * @msg_inbits:See ntb_msg_inbits().
+ * @msg_outbits:   See ntb_msg_outbits().
+ * @msg_read_sts:  See ntb_msg_read_sts().
+ * @msg_clear_sts: See ntb_msg_clear_sts().
+ * @msg_set_mask:  See ntb_msg_set_mask().
+ * @msg_clear_mask:See ntb_msg_clear_mask().
+ * @msg_read:  See ntb_msg_read().
+ * @msg_write: See ntb_msg_write().
  */
 struct ntb_dev_ops {
int (*port_number)(struct ntb_dev *ntb);
@@ -296,6 +308,16 @@ struct ntb_dev_ops {
u32 (*peer_spad_read)(struct ntb_dev *ntb, int pidx, int sidx);
int (*peer_spad_write)(struct ntb_dev *ntb, int pidx, int sidx,
   u32 val);
+
+   int (*msg_count)(struct ntb_dev *ntb);
+   u64 (*msg_inbits)(struct ntb_dev *ntb);
+   u64 (*msg_outbits)(struct ntb_dev *ntb);
+   u64 (*msg_read_sts)(struct ntb_dev *ntb);
+   int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits);
+   int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_read)(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg);
+   int (*msg_write)(struct ntb_dev *ntb, int midx, int pidx, u32 msg);
 };
 
 static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
@@ -342,6 +364,15 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* !ops->peer_spad_addr == !ops->spad_count && */
/* !ops->peer_spad_read == !ops->spad_count && */
!ops->peer_spad_write == !ops->spad_count   &&
+
+   !ops->msg_inbits == !ops->msg_count &&
+   !ops->msg_outbits == !ops->msg_count