Re: [PATCH 04/24] ibtrs: client: private header with client structs and functions

[Roman Penyaev]

Hi Sagi,

On Mon, Feb 5, 2018 at 11:59 AM, Sagi Grimberg  wrote:
> Hi Roman,
>
>
>> +struct ibtrs_clt_io_req {
>> +   struct list_headlist;
>> +   struct ibtrs_iu *iu;
>> +   struct scatterlist  *sglist; /* list holding user data */
>> +   unsigned intsg_cnt;
>> +   unsigned intsg_size;
>> +   unsigned intdata_len;
>> +   unsigned intusr_len;
>> +   void*priv;
>> +   boolin_use;
>> +   struct ibtrs_clt_con*con;
>> +   union {
>> +   struct ib_pool_fmr  **fmr_list;
>> +   struct ibtrs_fr_desc**fr_list;
>> +   };
>
>
> We are pretty much stuck with fmrs for legacy devices, it has
> no future support plans, please don't add new dependencies
> on it. Its already hard enough to get rid of it.

Got it, we have a plan to get rid of fmr.  But as I remember our
internal tests: fr is slower.  The question: why that can be
according to your experience?  I will retest, but still that is
interesting to know.

>> +   void*map_page;
>> +   struct ibtrs_tag*tag;
>
>
> Can I ask why do you need another tag that is not the request
> tag?

Once I responded already, the summary is the following:

1. Indeed mq supports tags sharing, but only between hw queues, not
globally, so for us that means tags->nr_hw_queues = 1, which kills
performance.

2. We need tags sharing in the transport library, which should not
be tightly coupled with block device.


>> +   u16 nmdesc;
>> +   enum dma_data_direction dir;
>> +   ibtrs_conf_fn   *conf;
>> +   unsigned long   start_time;
>> +};
>> +
>
>
>> +static inline struct ibtrs_clt_con *to_clt_con(struct ibtrs_con *c)
>> +{
>> +   if (unlikely(!c))
>> +   return NULL;
>> +
>> +   return container_of(c, struct ibtrs_clt_con, c);
>> +}
>> +
>> +static inline struct ibtrs_clt_sess *to_clt_sess(struct ibtrs_sess *s)
>> +{
>> +   if (unlikely(!s))
>> +   return NULL;
>> +
>> +   return container_of(s, struct ibtrs_clt_sess, s);
>> +}
>
>
> Seems a bit awkward that container_of wrappers check pointer validity...

That can be fixed, frankly, I don't remember code paths where I
implicitly rely on that returned null: session or connection are
always expected as valid pointers.

>> +/**
>> + * list_next_or_null_rr - get next list element in round-robin fashion.
>> + * @pos: entry, starting cursor.
>> + * @head:head of the list to examine. This list must have at least
>> one
>> + *   element, namely @pos.
>> + * @member:  name of the list_head structure within typeof(*pos).
>> + *
>> + * Important to understand that @pos is a list entry, which can be
>> already
>> + * removed using list_del_rcu(), so if @head has become empty NULL will
>> be
>> + * returned. Otherwise next element is returned in round-robin fashion.
>> + */
>> +#define list_next_or_null_rcu_rr(pos, head, member) ({ \
>> +   typeof(pos) next = NULL;\
>> +   \
>> +   if (!list_empty(head))  \
>> +   next = (pos)->member.next != (head) ?   \
>> +   list_entry_rcu((pos)->member.next,  \
>> +  typeof(*pos), member) :  \
>> +   list_entry_rcu((pos)->member.next->next,\
>> +  typeof(*pos), member);   \
>> +   next;   \
>> +})
>
>
> Why is this local to your driver?

Yeah, of course I can try to extend list.h

>> +
>> +/* See ibtrs-log.h */
>> +#define TYPES_TO_SESSNAME(obj) \
>> +   LIST(CASE(obj, struct ibtrs_clt_sess *, s.sessname),\
>> +CASE(obj, struct ibtrs_clt *, sessname))
>> +
>> +#define TAG_SIZE(clt) (sizeof(struct ibtrs_tag) + (clt)->pdu_sz)
>> +#define GET_TAG(clt, idx) ((clt)->tags + TAG_SIZE(clt) * idx)
>
>
> Still don't understand why this is even needed..

--
Roman

Re: [PATCH 04/24] ibtrs: client: private header with client structs and functions

[Sagi Grimberg]

Hi Roman,



+struct ibtrs_clt_io_req {
+   struct list_headlist;
+   struct ibtrs_iu *iu;
+   struct scatterlist  *sglist; /* list holding user data */
+   unsigned intsg_cnt;
+   unsigned intsg_size;
+   unsigned intdata_len;
+   unsigned intusr_len;
+   void*priv;
+   boolin_use;
+   struct ibtrs_clt_con*con;
+   union {
+   struct ib_pool_fmr  **fmr_list;
+   struct ibtrs_fr_desc**fr_list;
+   };


We are pretty much stuck with fmrs for legacy devices, it has
no future support plans, please don't add new dependencies
on it. Its already hard enough to get rid of it.


+   void*map_page;
+   struct ibtrs_tag*tag;


Can I ask why do you need another tag that is not the request
tag?


+   u16 nmdesc;
+   enum dma_data_direction dir;
+   ibtrs_conf_fn   *conf;
+   unsigned long   start_time;
+};
+



+static inline struct ibtrs_clt_con *to_clt_con(struct ibtrs_con *c)
+{
+   if (unlikely(!c))
+   return NULL;
+
+   return container_of(c, struct ibtrs_clt_con, c);
+}
+
+static inline struct ibtrs_clt_sess *to_clt_sess(struct ibtrs_sess *s)
+{
+   if (unlikely(!s))
+   return NULL;
+
+   return container_of(s, struct ibtrs_clt_sess, s);
+}


Seems a bit awkward that container_of wrappers check pointer validity...


+/**
+ * list_next_or_null_rr - get next list element in round-robin fashion.
+ * @pos: entry, starting cursor.
+ * @head:head of the list to examine. This list must have at least one
+ *   element, namely @pos.
+ * @member:  name of the list_head structure within typeof(*pos).
+ *
+ * Important to understand that @pos is a list entry, which can be already
+ * removed using list_del_rcu(), so if @head has become empty NULL will be
+ * returned. Otherwise next element is returned in round-robin fashion.
+ */
+#define list_next_or_null_rcu_rr(pos, head, member) ({ \
+   typeof(pos) next = NULL;\
+   \
+   if (!list_empty(head))  \
+   next = (pos)->member.next != (head) ?\
+   list_entry_rcu((pos)->member.next,   \
+  typeof(*pos), member) :  \
+   list_entry_rcu((pos)->member.next->next,  \
+  typeof(*pos), member);   \
+   next;   \
+})


Why is this local to your driver?


+
+/* See ibtrs-log.h */
+#define TYPES_TO_SESSNAME(obj) \
+   LIST(CASE(obj, struct ibtrs_clt_sess *, s.sessname),\
+CASE(obj, struct ibtrs_clt *, sessname))
+
+#define TAG_SIZE(clt) (sizeof(struct ibtrs_tag) + (clt)->pdu_sz)
+#define GET_TAG(clt, idx) ((clt)->tags + TAG_SIZE(clt) * idx)


Still don't understand why this is even needed..

[PATCH 04/24] ibtrs: client: private header with client structs and functions

[Roman Pen]

This header describes main structs and functions used by ibtrs-client
module, mainly for managing IBTRS sessions, creating/destroying sysfs
entries, accounting statistics on client side.

Signed-off-by: Roman Pen 
Signed-off-by: Danil Kipnis 
Cc: Jack Wang 
---
 drivers/infiniband/ulp/ibtrs/ibtrs-clt.h | 338 +++
 1 file changed, 338 insertions(+)

diff --git a/drivers/infiniband/ulp/ibtrs/ibtrs-clt.h 
b/drivers/infiniband/ulp/ibtrs/ibtrs-clt.h
new file mode 100644
index ..b57af19ac833
--- /dev/null
+++ b/drivers/infiniband/ulp/ibtrs/ibtrs-clt.h
@@ -0,0 +1,338 @@
+/*
+ * InfiniBand Transport Layer
+ *
+ * Copyright (c) 2014 - 2017 ProfitBricks GmbH. All rights reserved.
+ * Authors: Fabian Holler 
+ *  Jack Wang 
+ *  Kleber Souza 
+ *  Danil Kipnis 
+ *  Roman Penyaev 
+ *  Milind Dumbare 
+ *
+ * Copyright (c) 2017 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Authors: Danil Kipnis 
+ *  Roman Penyaev 
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see .
+ */
+
+#ifndef IBTRS_CLT_H
+#define IBTRS_CLT_H
+
+#include "ibtrs-pri.h"
+
+/**
+ * enum ibtrs_clt_state - Client states.
+ */
+enum ibtrs_clt_state {
+   IBTRS_CLT_CONNECTING,
+   IBTRS_CLT_CONNECTING_ERR,
+   IBTRS_CLT_RECONNECTING,
+   IBTRS_CLT_CONNECTED,
+   IBTRS_CLT_CLOSING,
+   IBTRS_CLT_CLOSED,
+   IBTRS_CLT_DEAD,
+};
+
+static inline const char *ibtrs_clt_state_str(enum ibtrs_clt_state state)
+{
+   switch (state) {
+   case IBTRS_CLT_CONNECTING:
+   return "IBTRS_CLT_CONNECTING";
+   case IBTRS_CLT_CONNECTING_ERR:
+   return "IBTRS_CLT_CONNECTING_ERR";
+   case IBTRS_CLT_RECONNECTING:
+   return "IBTRS_CLT_RECONNECTING";
+   case IBTRS_CLT_CONNECTED:
+   return "IBTRS_CLT_CONNECTED";
+   case IBTRS_CLT_CLOSING:
+   return "IBTRS_CLT_CLOSING";
+   case IBTRS_CLT_CLOSED:
+   return "IBTRS_CLT_CLOSED";
+   case IBTRS_CLT_DEAD:
+   return "IBTRS_CLT_DEAD";
+   default:
+   return "UNKNOWN";
+   }
+}
+
+enum ibtrs_fast_reg {
+   IBTRS_FAST_MEM_NONE,
+   IBTRS_FAST_MEM_FR,
+   IBTRS_FAST_MEM_FMR
+};
+
+enum ibtrs_mp_policy {
+   MP_POLICY_RR,
+   MP_POLICY_MIN_INFLIGHT,
+};
+
+struct ibtrs_clt_stats_reconnects {
+   int successful_cnt;
+   int fail_cnt;
+};
+
+struct ibtrs_clt_stats_wc_comp {
+   u32 cnt;
+   u64 total_cnt;
+};
+
+struct ibtrs_clt_stats_cpu_migr {
+   atomic_t from;
+   int to;
+};
+
+struct ibtrs_clt_stats_rdma {
+   struct {
+   u64 cnt;
+   u64 size_total;
+   } dir[2];
+
+   u64 failover_cnt;
+};
+
+struct ibtrs_clt_stats_rdma_lat {
+   u64 read;
+   u64 write;
+};
+
+#define MIN_LOG_SG 2
+#define MAX_LOG_SG 5
+#define MAX_LIN_SG BIT(MIN_LOG_SG)
+#define SG_DISTR_SZ (MAX_LOG_SG - MIN_LOG_SG + MAX_LIN_SG + 2)
+
+#define MAX_LOG_LAT 16
+#define MIN_LOG_LAT 0
+#define LOG_LAT_SZ (MAX_LOG_LAT - MIN_LOG_LAT + 2)
+
+struct ibtrs_clt_stats_pcpu {
+   struct ibtrs_clt_stats_cpu_migr cpu_migr;
+   struct ibtrs_clt_stats_rdma rdma;
+   u64 sg_list_total;
+   u64 sg_list_distr[SG_DISTR_SZ];
+   struct ibtrs_clt_stats_rdma_lat rdma_lat_distr[LOG_LAT_SZ];
+   struct ibtrs_clt_stats_rdma_lat rdma_lat_max;
+   struct ibtrs_clt_stats_wc_comp  wc_comp;
+};
+
+struct ibtrs_clt_stats {
+   boolenable_rdma_lat;
+   struct ibtrs_clt_stats_pcpu__percpu *pcpu_stats;
+   struct ibtrs_clt_stats_reconnects   reconnects;
+   atomic_tinflight;
+};
+
+struct ibtrs_clt_con {
+   struct ibtrs_conc;
+   unsignedcpu;
+   atomic_tio_cnt;
+   struct ibtrs_fr_pool*fr_pool;
+   int cm_err;
+};
+