[PATCH] opensm/osm_perfmgr.c: Remove unnecessary lock reference from Performance Manager object
From: Ira Weiny
Date: Mon, 22 Dec 2008 11:03:59 -0800
Subject: [PATCH] opensm/osm_perfmgr.c: Remove unnecessary lock reference from
Performance Manager object
Signed-off-by: Ira Weiny
---
opensm/include/opensm/osm_perfmgr.h |1 -
opensm/opensm/osm_perfmgr.c | 25 -
2 files changed, 12 insertions(+), 14 deletions(-)
diff --git a/opensm/include/opensm/osm_perfmgr.h
b/opensm/include/opensm/osm_perfmgr.h
index 108b691..c26c141 100644
--- a/opensm/include/opensm/osm_perfmgr.h
+++ b/opensm/include/opensm/osm_perfmgr.h
@@ -119,7 +119,6 @@ typedef struct osm_perfmgr {
struct osm_opensm *osm;
osm_subn_t *subn;
osm_sm_t *sm;
- cl_plock_t *lock;
osm_log_t *log;
osm_mad_pool_t *mad_pool;
atomic32_t trans_id;
diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c
index 1952d05..398b463 100644
--- a/opensm/opensm/osm_perfmgr.c
+++ b/opensm/opensm/osm_perfmgr.c
@@ -214,10 +214,10 @@ static void perfmgr_mad_send_err_callback(void
*bind_context,
if (pm->subn->opt.perfmgr_redir && p_madw->status == IB_TIMEOUT) {
/* First, find the node in the monitored map */
- cl_plock_acquire(pm->lock);
+ cl_plock_acquire(&pm->osm->lock);
/* Now, validate port number */
if (port >= p_mon_node->num_ports) {
- cl_plock_release(pm->lock);
+ cl_plock_release(&pm->osm->lock);
OSM_LOG(pm->log, OSM_LOG_ERROR, "ERR 4C16: "
"Invalid port num %u for %s (GUID 0x%016"
PRIx64 ") num ports %u\n", port,
@@ -228,7 +228,7 @@ static void perfmgr_mad_send_err_callback(void
*bind_context,
/* Clear redirection info */
p_mon_node->redir_port[port].redir_lid = 0;
p_mon_node->redir_port[port].redir_qp = 0;
- cl_plock_release(pm->lock);
+ cl_plock_release(&pm->osm->lock);
}
Exit:
@@ -469,7 +469,7 @@ static void perfmgr_query_counters(cl_map_item_t *
p_map_item, void *context)
OSM_LOG_ENTER(pm->log);
- cl_plock_acquire(pm->lock);
+ cl_plock_acquire(&pm->osm->lock);
node = osm_get_node_by_guid(pm->subn, cl_hton64(mon_node->guid));
if (!node) {
OSM_LOG(pm->log, OSM_LOG_ERROR,
@@ -531,7 +531,7 @@ static void perfmgr_query_counters(cl_map_item_t *
p_map_item, void *context)
node->print_desc);
}
Exit:
- cl_plock_release(pm->lock);
+ cl_plock_release(&pm->osm->lock);
OSM_LOG_EXIT(pm->log);
}
@@ -778,9 +778,9 @@ void osm_perfmgr_process(osm_perfmgr_t * pm)
* and not have to sweep the node_guid_tbl each pass
*/
OSM_LOG(pm->log, OSM_LOG_VERBOSE, "Gathering PerfMgr stats\n");
- cl_plock_acquire(pm->lock);
+ cl_plock_acquire(&pm->osm->lock);
cl_qmap_apply_func(&pm->subn->node_guid_tbl, collect_guids, pm);
- cl_plock_release(pm->lock);
+ cl_plock_release(&pm->osm->lock);
/* then for each node query their counters */
cl_qmap_apply_func(&pm->monitored_map, perfmgr_query_counters, pm);
@@ -965,11 +965,11 @@ static void perfmgr_check_overflow(osm_perfmgr_t * pm,
") port %d; clearing counters\n",
mon_node->name, mon_node->guid, port);
- cl_plock_acquire(pm->lock);
+ cl_plock_acquire(&pm->osm->lock);
p_node =
osm_get_node_by_guid(pm->subn, cl_hton64(mon_node->guid));
lid = get_lid(p_node, port, mon_node);
- cl_plock_release(pm->lock);
+ cl_plock_release(&pm->osm->lock);
if (lid == 0) {
OSM_LOG(pm->log, OSM_LOG_ERROR, "PerfMgr: ERR 4C0C: "
"Failed to clear counters for %s (0x%"
@@ -1112,10 +1112,10 @@ static void pc_recv_process(void *context, void *data)
}
/* LID redirection support (easier than GID redirection) */
- cl_plock_acquire(pm->lock);
+ cl_plock_acquire(&pm->osm->lock);
/* Now, validate port number */
if (port >= p_mon_node->num_ports) {
- cl_plock_release(pm->lock);
+ cl_plock_release(&pm->osm->lock);
OSM_LOG(pm->log, OSM_LOG_ERROR, "ERR 4C13: "
"Invalid port num %d for GUID 0x%016"
PRIx64 " num ports %d\n", port, node_guid,
@@ -1124,7 +1124,7 @@ static void pc_recv_process(void *context, void *data)
}
p_mon_node->redir_port[port].redir_lid = cpi->redir_lid;
p_mon_node->redir_port[port].redir_qp = cpi->redir_qp;
- cl_plock_release(pm->lock);
+
librdmacm: select() gives false positive on cm channel file descriptor
Hi, I'm looking for some help with a problem I'm having with librdmacm. I have been using the select() call to detect when a disconnect event is pending on the rdma_cm event channel (while also waiting for completion queue events). On the client side, things seem to work fine. On the server side, I get a false positive from select() if I'm also waiting for completion notification on a completion channel. If I comment out any one piece of the code involving the completion event - the ibv_req_notify_cq(), the ibv_post_send(), or the FD_SET(comp_channel) - then I do not get the false positive. But if I've done all the steps necessary to detect the completion of my send operation, select() returns immediately with an indication that the rdma_cm channel is readable, though a call to rdma_get_cm_event() just blocks because there are no events pending. Any insights into what the problem might be? I can provide code for a simple client and server that demonstrate the problem. I've been testing on a system with SLES 10, and the librdmacm version appears to be 1.0.11. Thanks, -Bryan -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] opensm: fixing compilation issues in some header files
On 00:18 Wed 02 Jun , Yevgeny Kliteynik wrote: > > > > AFAIR (anytype *) to (void *) casting is not needed (doing implicitly) > > in C and this is already part of some basic standards. > > True, but the problem is not (anytype *) to (void *) casting. > It's the other way around: (void *) to (anytype *) casting. It is the same meaning. Otherwise it would be useless. Sasha -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] opensm: fixing compilation issues in some header files
On 01-Jun-10 10:22 PM, Sasha Khapyorsky wrote: Hi Yevgeny, On 14:58 Tue 01 Jun , Yevgeny Kliteynik wrote: On 24-Mar-10 5:50 PM, Yevgeny Kliteynik wrote: All the compilation issues refer to implicit casting from "void*" to "some_struct_t*" This was detected when compiling a code that includes these headers with g++ compiler. The headers should be able to compile by g++ (they have 'extern "C"'). Well let's apply the patch since we have 'extern "C"'. But this raises another question. All affected header files are OpenSM internals and OpenSM is written in C (not C++). So why 'extern "C"' should be here at all? Do you know? But the problem is not only with g++ - it is with typing in general. I may be wrong, but I feel that every new gcc version is more strongly typed. AFAIR (anytype *) to (void *) casting is not needed (doing implicitly) in C and this is already part of some basic standards. True, but the problem is not (anytype *) to (void *) casting. It's the other way around: (void *) to (anytype *) casting. -- Yevgeny Sasha -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] opensm: fixing compilation issues in some header files
On 17:50 Wed 24 Mar , Yevgeny Kliteynik wrote:
> All the compilation issues refer to implicit casting
> from "void*" to "some_struct_t*"
>
> Signed-off-by: Yevgeny Kliteynik
Applied. Thanks. However see a comment below.
> ---
> opensm/include/opensm/osm_pkey.h |8 +---
> opensm/include/opensm/osm_port.h |4 ++--
> opensm/include/opensm/osm_subnet.h |2 +-
> 3 files changed, 8 insertions(+), 6 deletions(-)
>
> diff --git a/opensm/include/opensm/osm_pkey.h
> b/opensm/include/opensm/osm_pkey.h
> index d10479d..53e9657 100644
> --- a/opensm/include/opensm/osm_pkey.h
> +++ b/opensm/include/opensm/osm_pkey.h
> @@ -252,7 +252,8 @@ static inline ib_pkey_table_t
> *osm_pkey_tbl_block_get(const osm_pkey_tbl_t *
> uint16_t block)
> {
> return ((block < cl_ptr_vector_get_size(&p_pkey_tbl->blocks)) ?
> - cl_ptr_vector_get(&p_pkey_tbl->blocks, block) : NULL);
> + (ib_pkey_table_t *)cl_ptr_vector_get(
> + &p_pkey_tbl->blocks, block) : NULL);
> };
>
> /*
> @@ -282,8 +283,9 @@ static inline ib_pkey_table_t
> *osm_pkey_tbl_new_block_get(const osm_pkey_tbl_t *
> p_pkey_tbl,
> uint16_t block)
> {
> - return (block < cl_ptr_vector_get_size(&p_pkey_tbl->new_blocks)) ?
> - cl_ptr_vector_get(&p_pkey_tbl->new_blocks, block) : NULL;
> + return ((block < cl_ptr_vector_get_size(&p_pkey_tbl->new_blocks)) ?
> + (ib_pkey_table_t *)cl_ptr_vector_get(
> + &p_pkey_tbl->new_blocks, block) : NULL);
> };
>
> /f* OpenSM: osm_pkey_tbl_set_new_entry
> diff --git a/opensm/include/opensm/osm_port.h
> b/opensm/include/opensm/osm_port.h
> index ff0a178..8c68c99 100644
> --- a/opensm/include/opensm/osm_port.h
> +++ b/opensm/include/opensm/osm_port.h
> @@ -549,7 +549,7 @@ static inline void osm_physp_set_slvl_tbl(IN osm_physp_t
> * p_physp,
>
> CL_ASSERT(p_slvl_tbl);
> CL_ASSERT(osm_physp_is_valid(p_physp));
> - p_tbl = cl_ptr_vector_get(&p_physp->slvl_by_port, in_port_num);
> + p_tbl = (ib_slvl_table_t *)cl_ptr_vector_get(&p_physp->slvl_by_port,
> in_port_num);
> *p_tbl = *p_slvl_tbl;
> }
>
> @@ -590,7 +590,7 @@ static inline ib_slvl_table_t *osm_physp_get_slvl_tbl(IN
> const osm_physp_t *
> ib_slvl_table_t *p_tbl;
>
> CL_ASSERT(osm_physp_is_valid(p_physp));
> - p_tbl = cl_ptr_vector_get(&p_physp->slvl_by_port, in_port_num);
> + p_tbl = (ib_slvl_table_t *)cl_ptr_vector_get(&p_physp->slvl_by_port,
> in_port_num);
> return p_tbl;
> }
>
> diff --git a/opensm/include/opensm/osm_subnet.h
> b/opensm/include/opensm/osm_subnet.h
> index 3970e98..2eef9c7 100644
> --- a/opensm/include/opensm/osm_subnet.h
> +++ b/opensm/include/opensm/osm_subnet.h
> @@ -1015,7 +1015,7 @@ struct osm_mgrp *osm_get_mgrp_by_mgid(IN osm_subn_t *
> subn, IN ib_gid_t * mgid);
> */
> static inline struct osm_mgrp_box *osm_get_mbox_by_mlid(osm_subn_t const
> *p_subn, ib_net16_t mlid)
> {
> - return p_subn->mboxes[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO];
> + return (struct osm_mgrp_box *)p_subn->mboxes[cl_ntoh16(mlid) -
> IB_LID_MCAST_START_HO];
In this case I would suggest to define p_subn->mboxes[] to have
appropriate type and so to avoid the casting issue.
Sasha
> }
> /*
> * PARAMETERS
> --
> 1.5.1.4
>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] opensm: fixing compilation issues in some header files
Hi Yevgeny, On 14:58 Tue 01 Jun , Yevgeny Kliteynik wrote: > > On 24-Mar-10 5:50 PM, Yevgeny Kliteynik wrote: > > All the compilation issues refer to implicit casting > > from "void*" to "some_struct_t*" > > This was detected when compiling a code that includes these > headers with g++ compiler. The headers should be able to compile > by g++ (they have 'extern "C"'). Well let's apply the patch since we have 'extern "C"'. But this raises another question. All affected header files are OpenSM internals and OpenSM is written in C (not C++). So why 'extern "C"' should be here at all? Do you know? > But the problem is not only > with g++ - it is with typing in general. I may be wrong, but I > feel that every new gcc version is more strongly typed. AFAIR (anytype *) to (void *) casting is not needed (doing implicitly) in C and this is already part of some basic standards. Sasha -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] opensm: Add a rate based mechanism for SMP transactions
On 18:32 Tue 01 Jun , Sasha Khapyorsky wrote:
> Hi Hal,
>
> On 10:11 Wed 16 Dec , Hal Rosenstock wrote:
> >
> > In order to better handle non responsive SMAs (when link is physically up
> > but the SMA does not respond), a rate based mechanism for SMPs is added
> > to better enable forward progress in a more timely fashion. So rather than
> > wait for timeouts and outstanding wire SMPs to drop below some configured
> > value, there is also a periodic rate for transaction based SMPs. These
> > rate based SMPs are capped at a configured maximum value. In order to
> > accomodate these, the vendor layer ibumad match table is increased by
> > that number in order not to overflow due to these added transactions.
> >
> > Two new options are added for this:
> > rate_based_smp_usecs indicates the number of microseconds between rate
> > based SMPs.
> > max_rate_based_smps indicates the maximum number of rate based SMPs
> > supported. When this limit is reached, rate based SMPs are no longer
> > sent (until the number of outstanding ones drops below this limit).
>
> As far as I learned the patch Wouldn't something like below does the
> same work:
>
>
> diff --git a/opensm/opensm/osm_vl15intf.c b/opensm/opensm/osm_vl15intf.c
> index ff9e4db..a16d88e 100644
> --- a/opensm/opensm/osm_vl15intf.c
> +++ b/opensm/opensm/osm_vl15intf.c
> @@ -113,6 +113,8 @@ static void vl15_poller(IN void *p_ptr)
> osm_madw_t *p_madw;
> osm_vl15_t *p_vl = p_ptr;
> cl_qlist_t *p_fifo;
> + int32_t max_smps = p_vl->max_wire_smps;
> + int32_t max_wire_smps2 = 2 * max_smps; /* FIXME: make configurable */
>
> OSM_LOG_ENTER(p_vl->p_log);
>
> @@ -156,16 +158,21 @@ static void vl15_poller(IN void *p_ptr)
> EVENT_NO_TIMEOUT, TRUE);
>
> while (p_vl->p_stats->qp0_mads_outstanding_on_wire >=
> -(int32_t) p_vl->max_wire_smps &&
> +max_smps &&
> p_vl->thread_state == OSM_THREAD_STATE_RUN) {
> status = cl_event_wait_on(&p_vl->signal,
> EVENT_NO_TIMEOUT, TRUE);
Sure, with a real timeout value here.
> - if (status != CL_SUCCESS) {
> + if (status == CL_TIMEOUT &&
> + max_smps < max_wire_smps2) {
> + max_smps++;
> + break;
> + } else if (status != CL_SUCCESS) {
> OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E02: "
> "Event wait failed (%s)\n",
> CL_STATUS_MSG(status));
> break;
> }
> + max_smps = p_vl->max_wire_smps;
> }
> }
>
>
> If yes, we will need only have two configurable max_wire_smps limits.
>
> Sasha
>
> >
> > The rate based SMP mechanism can be disabled by setting rate_based_smp_usecs
> > to 0. This is equivalent to the (current) algorithm prior to this change.
> >
> > Test results:
> >
> > Subnet consists of 55 switches (all 36-port IS4) and couple of HCAs.
> > OpenSM configuration to enlarge the fabric: LMC=7, LMC of
> > extended port 0 = TRUE.
> >
> > It takes ~8K SMPs to configure this fabric (no QoS).
> >
> > Measured section of the code: LFTs configuration, which is
> > the most SMP-intense phase of the sweep.
> >
> > Existing OpenSM code:
> >max_wire_smps=1: LFT configuration took ~0.27 sec
> >max_wire_smps=4: LFT configuration took ~0.13 sec
> >
> > OpenSM with rate-based SMPs
> >no difference from the existing OpenSM was observed.
> >
> > Further testing showed that when subnet is OK (no timeouts),
> > SM doesn't send rate-based SMPs at all, or sends just a couple
> > of them (out of total 8K SMPs).
> >
> > Experimenting with "bad" fabric:
> > With 480 timeouts in a row, all the timeouts were failed Set() commands.
> > OpenSM configuration was as follows:
> >max_wire_smps=1
> >rate_based_smp_usec=1 (10 msec)
> >max_rate_based_smps=100
> >
> > Whole sweep time: 21 seconds
> > Virtually all the SMPs were rate-based.
> > Calculating how much this should have taken w/o rate-based SMPs:
> > (480 timeouts) * (3 retries) * (0.2 sec timeout) = 4.8 minutes
> > so this is a big improvement in the presence of errors.
> >
> > Signed-off-by: Hal Rosenstock
> > ---
> > diff --git a/opensm/include/opensm/osm_base.h
> > b/opensm/include/opensm/osm_base.h
> > index 4e9aaa9..ddb1265 100644
> > --- a/opensm/include/opensm/osm_base.h
> > +++ b/opensm/include/opensm/osm_base.h
> > @@ -448,6 +448,30 @@ BEGIN_C_DECLS
> > */
> > #define OSM_DEFAULT_SMP_MAX_ON_WIRE 4
> > /***/
> > +/d* OpenSM: Base/OSM_DEFAULT_SMP_RATE
> > +* NAME
> > +* OSM_DEFAULT_SMP_RATE
> > +*
> > +* DESCRIPTION
> > +* Specifies the de
Re: [PATCH v2] opensm/qos.c: Revert port ranges for calls to sl2vl_update_table().
On 08:56 Tue 01 Jun , Jim Schutt wrote: > Before commit 051a1dd5 (opensm/osm_qos.c: split switch external and end > ports setup), osm_qos_setup() would end up calling sl2vl_update_table() > for output ports 1-N, and inport ports 0-N. > > Commit 051a1dd5 changed this around to be output ports 0-N, and input > ports 1-N, and an InfiniScale IV-based fabric would log lots of errors > like these: > > log_rcv_cb_error: ERR 3111: Received MAD with error status = 0x1C > SubnGetResp(SLtoVLMappingTable), attr_mod 0x2300, TID 0xad069 > Initial path: 0,1,1,4,13 Return path: 0,25,1,7,10 > > The attr_mod in every such message has 0x00 in the least significant > byte, which specifies the output port. > > With the port ranges restored to their old values, the above log messages > stop. Hal Rosenstock pointed out that we should not be attempting > to program a base SP0 with SL2VL maps; see, e.g., IBA 1.2.1, section > 14.2.5.8, page 844. So, this patch is a full reversion for > switches supporting base SP0, but only a partial reversion for > switches supporting enhanced SP0. > > Signed-off-by: Jim Schutt Applied. Thanks. Sasha -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v2] opensm/osmeventplugin: added couple of events to monitor SM
Hi Yevgeny,
On 12:20 Wed 07 Apr , Yevgeny Kliteynik wrote:
>
> I've added a couple of new events that allow event
> plug-in to see what SM is doing, when it is sweeping
> and when it updates dump files:
>
> OSM_EVENT_ID_L_SWEEP_STARTED,
> OSM_EVENT_ID_L_SWEEP_DONE,
> OSM_EVENT_ID_H_SWEEP_STARTED,
> OSM_EVENT_ID_H_SWEEP_DONE,
> OSM_EVENT_ID_REROUTE_DONE,
> OSM_EVENT_ID_ENTERING_STANDBY,
> OSM_EVENT_ID_SM_PORT_DOWN,
> OSM_EVENT_ID_SA_DB_DUMPED
>
> The last event is reported when SA DB was actually dumped.
> I'm thinking of similar optimization for guid2lid file - it
> doesn't have to be dumped at the end of each heavy sweep,
> as many heavy sweeps don't really happen because of nodes
> appearing/disappearing.
I don't think that having a lot of events and spamming OpenSM core code
with osm_opensm_report_event() calls were an original goals. The
plugin interface is done so that it has full access to OpenSM internal
data structures, etc.. So only *really* important things (such as SUBNET
UP) will be transferred as events.
Also when sending the patch like this it would be really nice to have
some description about why this and that events are needed, where this
will be used, etc..
Sasha
>
> Signed-off-by: Yevgeny Kliteynik
> ---
>
> Changes from V1:
> - added reporting OSM_EVENT_ID_H_SWEEP_DONE event
> - rebased to latest master
>
> opensm/include/opensm/osm_event_plugin.h | 10 +-
> opensm/opensm/osm_state_mgr.c | 22 +-
> opensm/osmeventplugin/src/osmeventplugin.c | 24
> 3 files changed, 54 insertions(+), 2 deletions(-)
>
> diff --git a/opensm/include/opensm/osm_event_plugin.h
> b/opensm/include/opensm/osm_event_plugin.h
> index 33d1920..f5a57d7 100644
> --- a/opensm/include/opensm/osm_event_plugin.h
> +++ b/opensm/include/opensm/osm_event_plugin.h
> @@ -72,7 +72,15 @@ typedef enum {
> OSM_EVENT_ID_PORT_SELECT,
> OSM_EVENT_ID_TRAP,
> OSM_EVENT_ID_SUBNET_UP,
> - OSM_EVENT_ID_MAX
> + OSM_EVENT_ID_MAX,
> + OSM_EVENT_ID_L_SWEEP_STARTED,
> + OSM_EVENT_ID_L_SWEEP_DONE,
> + OSM_EVENT_ID_H_SWEEP_STARTED,
> + OSM_EVENT_ID_H_SWEEP_DONE,
> + OSM_EVENT_ID_REROUTE_DONE,
> + OSM_EVENT_ID_ENTERING_STANDBY,
> + OSM_EVENT_ID_SM_PORT_DOWN,
> + OSM_EVENT_ID_SA_DB_DUMPED
> } osm_epi_event_id_t;
>
> typedef struct osm_epi_port_id {
> diff --git a/opensm/opensm/osm_state_mgr.c b/opensm/opensm/osm_state_mgr.c
> index e43463f..d5dff14 100644
> --- a/opensm/opensm/osm_state_mgr.c
> +++ b/opensm/opensm/osm_state_mgr.c
> @@ -1076,6 +1076,9 @@ static void do_sweep(osm_sm_t * sm)
> sm->p_subn->sm_state != IB_SMINFO_STATE_DISCOVERING)
> return;
>
> + osm_opensm_report_event(sm->p_subn->p_osm,
> + OSM_EVENT_ID_L_SWEEP_STARTED, NULL);
> +
> if (sm->p_subn->coming_out_of_standby)
> /*
>* Need to force re-write of sm_base_lid to all ports
> @@ -,6 +1114,8 @@ static void do_sweep(osm_sm_t * sm)
> osm_sa_db_file_dump(sm->p_subn->p_osm);
> OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
> "LIGHT SWEEP COMPLETE");
> + osm_opensm_report_event(sm->p_subn->p_osm,
> + OSM_EVENT_ID_L_SWEEP_DONE, NULL);
> return;
> }
> }
> @@ -1151,6 +1156,8 @@ static void do_sweep(osm_sm_t * sm)
> if (!sm->p_subn->subnet_initialization_error) {
> OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
> "REROUTE COMPLETE");
> + osm_opensm_report_event(sm->p_subn->p_osm,
> + OSM_EVENT_ID_REROUTE_DONE, NULL);
> return;
> }
> }
> @@ -1158,6 +1165,9 @@ static void do_sweep(osm_sm_t * sm)
> /* go to heavy sweep */
> repeat_discovery:
>
> + osm_opensm_report_event(sm->p_subn->p_osm,
> + OSM_EVENT_ID_H_SWEEP_STARTED, NULL);
> +
> /* First of all - unset all flags */
> sm->p_subn->force_heavy_sweep = FALSE;
> sm->p_subn->force_reroute = FALSE;
> @@ -1185,6 +1195,8 @@ repeat_discovery:
>
> /* Move to DISCOVERING state */
> osm_sm_state_mgr_process(sm, OSM_SM_SIGNAL_DISCOVER);
> + osm_opensm_report_event(sm->p_subn->p_osm,
> + OSM_EVENT_ID_SM_PORT_DOWN, NULL);
> return;
> }
>
> @@ -1205,6 +1217,8 @@ repeat_discovery:
> "ENTERING STANDBY STATE");
> /* notify master SM about us */
> osm_send_trap144(sm, 0);
> + osm_opensm_report_event(sm->p_subn->p_osm,
> + OSM_EVENT_ID_ENTERING_STANDBY, NULL);
> return;
> }
>
> @@ -1212,6
[PATCH] librdmacm: support 2.6.9
Redhat 4.x is based on 2.6.9. Add support for older kernels.
Signed-off-by: Sean Hefty
---
This should fix the OFED build errors on RH 4.x. When testing this on a
RH 4.x system, I noticed additional build warnings on 32-bit systems.
I'll add a fix for these warnings separately.
include/infiniband/ib.h | 10 ++
1 files changed, 10 insertions(+), 0 deletions(-)
diff --git a/include/infiniband/ib.h b/include/infiniband/ib.h
index 3a97322..2e5029a 100644
--- a/include/infiniband/ib.h
+++ b/include/infiniband/ib.h
@@ -43,6 +43,16 @@
#define PF_IB AF_IB
#endif
+#ifndef __be16
+#define __be16 __u16
+#endif
+#ifndef __be32
+#define __be32 __u32
+#endif
+#ifndef __be64
+#define __be64 __u64
+#endif
+
struct ib_addr {
union {
__u8uib_addr8[16];
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] opensm: Add a rate based mechanism for SMP transactions
Hi Hal,
On 10:11 Wed 16 Dec , Hal Rosenstock wrote:
>
> In order to better handle non responsive SMAs (when link is physically up
> but the SMA does not respond), a rate based mechanism for SMPs is added
> to better enable forward progress in a more timely fashion. So rather than
> wait for timeouts and outstanding wire SMPs to drop below some configured
> value, there is also a periodic rate for transaction based SMPs. These
> rate based SMPs are capped at a configured maximum value. In order to
> accomodate these, the vendor layer ibumad match table is increased by
> that number in order not to overflow due to these added transactions.
>
> Two new options are added for this:
> rate_based_smp_usecs indicates the number of microseconds between rate
> based SMPs.
> max_rate_based_smps indicates the maximum number of rate based SMPs
> supported. When this limit is reached, rate based SMPs are no longer
> sent (until the number of outstanding ones drops below this limit).
As far as I learned the patch Wouldn't something like below does the
same work:
diff --git a/opensm/opensm/osm_vl15intf.c b/opensm/opensm/osm_vl15intf.c
index ff9e4db..a16d88e 100644
--- a/opensm/opensm/osm_vl15intf.c
+++ b/opensm/opensm/osm_vl15intf.c
@@ -113,6 +113,8 @@ static void vl15_poller(IN void *p_ptr)
osm_madw_t *p_madw;
osm_vl15_t *p_vl = p_ptr;
cl_qlist_t *p_fifo;
+ int32_t max_smps = p_vl->max_wire_smps;
+ int32_t max_wire_smps2 = 2 * max_smps; /* FIXME: make configurable */
OSM_LOG_ENTER(p_vl->p_log);
@@ -156,16 +158,21 @@ static void vl15_poller(IN void *p_ptr)
EVENT_NO_TIMEOUT, TRUE);
while (p_vl->p_stats->qp0_mads_outstanding_on_wire >=
- (int32_t) p_vl->max_wire_smps &&
+ max_smps &&
p_vl->thread_state == OSM_THREAD_STATE_RUN) {
status = cl_event_wait_on(&p_vl->signal,
EVENT_NO_TIMEOUT, TRUE);
- if (status != CL_SUCCESS) {
+ if (status == CL_TIMEOUT &&
+ max_smps < max_wire_smps2) {
+ max_smps++;
+ break;
+ } else if (status != CL_SUCCESS) {
OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E02: "
"Event wait failed (%s)\n",
CL_STATUS_MSG(status));
break;
}
+ max_smps = p_vl->max_wire_smps;
}
}
If yes, we will need only have two configurable max_wire_smps limits.
Sasha
>
> The rate based SMP mechanism can be disabled by setting rate_based_smp_usecs
> to 0. This is equivalent to the (current) algorithm prior to this change.
>
> Test results:
>
> Subnet consists of 55 switches (all 36-port IS4) and couple of HCAs.
> OpenSM configuration to enlarge the fabric: LMC=7, LMC of
> extended port 0 = TRUE.
>
> It takes ~8K SMPs to configure this fabric (no QoS).
>
> Measured section of the code: LFTs configuration, which is
> the most SMP-intense phase of the sweep.
>
> Existing OpenSM code:
>max_wire_smps=1: LFT configuration took ~0.27 sec
>max_wire_smps=4: LFT configuration took ~0.13 sec
>
> OpenSM with rate-based SMPs
>no difference from the existing OpenSM was observed.
>
> Further testing showed that when subnet is OK (no timeouts),
> SM doesn't send rate-based SMPs at all, or sends just a couple
> of them (out of total 8K SMPs).
>
> Experimenting with "bad" fabric:
> With 480 timeouts in a row, all the timeouts were failed Set() commands.
> OpenSM configuration was as follows:
>max_wire_smps=1
>rate_based_smp_usec=1 (10 msec)
>max_rate_based_smps=100
>
> Whole sweep time: 21 seconds
> Virtually all the SMPs were rate-based.
> Calculating how much this should have taken w/o rate-based SMPs:
> (480 timeouts) * (3 retries) * (0.2 sec timeout) = 4.8 minutes
> so this is a big improvement in the presence of errors.
>
> Signed-off-by: Hal Rosenstock
> ---
> diff --git a/opensm/include/opensm/osm_base.h
> b/opensm/include/opensm/osm_base.h
> index 4e9aaa9..ddb1265 100644
> --- a/opensm/include/opensm/osm_base.h
> +++ b/opensm/include/opensm/osm_base.h
> @@ -448,6 +448,30 @@ BEGIN_C_DECLS
> */
> #define OSM_DEFAULT_SMP_MAX_ON_WIRE 4
> /***/
> +/d* OpenSM: Base/OSM_DEFAULT_SMP_RATE
> +* NAME
> +*OSM_DEFAULT_SMP_RATE
> +*
> +* DESCRIPTION
> +*Specifies the default rate (in usec) for rate based SMPs.
> +*The default rate is 1 msec (1000 usec). A value of 0
> +*(or EVENT_NO_TIMEOUT) disables the rate based SMP mechanism.
> +*
> +* SYNOPSIS
> +*/
> +#define OSM_DEFAULT_SMP_RATE 1000
> +/***/
> +/d* OpenSM: Base/O
RE: [ANNOUNCE] librdmacm 1.0.12
> gcc -DHAVE_CONFIG_H -I. -I. -I. -I./include -g -Wall -D_GNU_SOURCE -O2 -g > -pipe -m64 -MT src_librdmacm_la-acm.lo -MD -MP -MF .deps/src_librdmacm_la- > acm.Tpo -c src/acm.c -fPIC -DPIC -o > .libs/src_librdmacm_la-acm.o > In file included from src/acm.c:44: > ./include/infiniband/ib.h:49: error: syntax error before "__be16" At least the following definitions are missing from types.h in redhat 4.x, which is based on 2.6.9. (These are from RH 5.x.) 158 #ifdef __CHECKER__ 159 #define __bitwise__ __attribute__((bitwise)) 160 #else 161 #define __bitwise__ 162 #endif 163 #ifdef __CHECK_ENDIAN__ 164 #define __bitwise __bitwise__ 165 #else 166 #define __bitwise 167 #endif 168 169 typedef __u16 __bitwise __le16; 170 typedef __u16 __bitwise __be16; 171 typedef __u32 __bitwise __le32; 172 typedef __u32 __bitwise __be32; 173 #if defined(__GNUC__) && !defined(__STRICT_ANSI__) 174 typedef __u64 __bitwise __le64; 175 typedef __u64 __bitwise __be64; 176 #endif Has OFED handled a similar problem in the past? If so, how? (I can think of a couple ways to deal with this.) - Sean -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH v2] opensm/qos.c: Revert port ranges for calls to sl2vl_update_table().
Before commit 051a1dd5 (opensm/osm_qos.c: split switch external and end
ports setup), osm_qos_setup() would end up calling sl2vl_update_table()
for output ports 1-N, and inport ports 0-N.
Commit 051a1dd5 changed this around to be output ports 0-N, and input
ports 1-N, and an InfiniScale IV-based fabric would log lots of errors
like these:
log_rcv_cb_error: ERR 3111: Received MAD with error status = 0x1C
SubnGetResp(SLtoVLMappingTable), attr_mod 0x2300, TID 0xad069
Initial path: 0,1,1,4,13 Return path: 0,25,1,7,10
The attr_mod in every such message has 0x00 in the least significant
byte, which specifies the output port.
With the port ranges restored to their old values, the above log messages
stop. Hal Rosenstock pointed out that we should not be attempting
to program a base SP0 with SL2VL maps; see, e.g., IBA 1.2.1, section
14.2.5.8, page 844. So, this patch is a full reversion for
switches supporting base SP0, but only a partial reversion for
switches supporting enhanced SP0.
Signed-off-by: Jim Schutt
---
opensm/opensm/osm_qos.c |5 +++--
1 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/opensm/opensm/osm_qos.c b/opensm/opensm/osm_qos.c
index f814ea8..cce59ee 100644
--- a/opensm/opensm/osm_qos.c
+++ b/opensm/opensm/osm_qos.c
@@ -230,10 +230,11 @@ static int qos_extports_setup(osm_sm_t * sm, osm_node_t
*node,
&qcfg->sl2vl);
}
- for (i = 1; i < num_ports; i++) {
+ for (i = 0; i < num_ports; i++) {
p = osm_node_get_physp_ptr(node, i);
force_update = p->need_update || sm->p_subn->need_update;
- for (j = 0; j < num_ports; j++)
+ j = ib_switch_info_is_enhanced_port0(&node->sw->switch_info) ?
0 : 1;
+ for (; j < num_ports; j++)
if (sl2vl_update_table(sm, p, i, i << 8 | j,
force_update, &qcfg->sl2vl))
ret = -1;
--
1.6.2.2
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH] ipoib pkey race condition
IPoIB is coded to use the 1st PKey in the PKey table as its ib0 interface.
Additional ib0.pkey interfaces may be created using the /sys/class/...
add_child interface.
However, there is a race. During normal boot, IPoIB will be started before the
port is Active. Hence the pkey table has not yet been programmed and has a
default pkey table (with 0x as only pkey).
Later when the SM moves the port to Active, the SM may program the pkey table
differently. However at this point IPoIB has already started using the
incorrect pkey.
It appears that the initially formatted 'broadcast' mgid is never updated to
supply actual pkey value if ipoib comes up before hca port. Proposed patch
targets two issues:
1. Suppress activation of interface and join multicast group queries (it will
fail anyway) until hca port is initialized. When port becomes active - update
pkey value and move on.
2. Update broadcast mgid based on actual pkey, then issue join broadcast group
request.
Signed-Off-By: Michael Heinz
---
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index ec6b4fb..496d96c 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -51,6 +51,7 @@ MODULE_PARM_DESC(data_debug_level,
#endif
static DEFINE_MUTEX(pkey_mutex);
+static void ipoib_pkey_dev_check_presence(struct net_device *dev);
struct ipoib_ah *ipoib_create_ah(struct net_device *dev,
struct ib_pd *pd, struct ib_ah_attr *attr)
@@ -654,12 +655,13 @@ int ipoib_ib_dev_open(struct net_device *dev)
struct ipoib_dev_priv *priv = netdev_priv(dev);
int ret;
- if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &priv->pkey_index)) {
+ ipoib_pkey_dev_check_presence(dev);
+
+ if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
ipoib_warn(priv, "P_Key 0x%04x not found\n", priv->pkey);
clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
return -1;
}
- set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
ret = ipoib_init_qp(dev);
if (ret) {
@@ -694,9 +696,26 @@ int ipoib_ib_dev_open(struct net_device *dev)
static void ipoib_pkey_dev_check_presence(struct net_device *dev)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
- u16 pkey_index = 0;
+ struct ib_port_attrport_attr;
+
+ if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
+ clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
+ if (ib_query_port(priv->ca, priv->port, &port_attr)) {
+ ipoib_warn(priv, "Query port attrs failed\n");
+ return;
+ }
+
+ if (port_attr.state != IB_PORT_ACTIVE)
+ return;
+
+ if (ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey)) {
+ ipoib_warn(priv, "Query P_Key table entry 0 failed\n");
+ return;
+ }
+ set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
+ }
- if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &pkey_index))
+ if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &priv->pkey_index))
clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
else
set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
@@ -955,7 +974,8 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv
*priv,
}
/* restart QP only if P_Key index is changed */
- if (test_and_set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags) &&
+ if (test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags) &&
+ test_and_set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags) &&
new_index == priv->pkey_index) {
ipoib_dbg(priv, "Not flushing - P_Key index not
changed.\n");
return;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index 3871ac6..6fe6527 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -552,6 +552,13 @@ void ipoib_mcast_join_task(struct work_struct *work)
}
spin_lock_irq(&priv->lock);
+
+ if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
+ /* fix broadcast gid in case if pkey was changed */
+ priv->pkey |= 0x8000;
+ priv->dev->broadcast[8] = priv->pkey >> 8;
+ priv->dev->broadcast[9] = priv->pkey & 0xff;
+ }
memcpy(broadcast->mcmember.mgid.raw, priv->dev->broadcast + 4,
sizeof (union ib_gid));
priv->broadcast = broadcast;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info
Re: [ewg] libibverbs without an HCA
On 6/1/2010 2:39 PM, Albert Strasheim wrote: Hello all Having just reviewed the various on-host shared memory mechanisms available in Linux and being quite unimpressed, I have the following question: Has anyone considered making a kernel and userspace driver so that libibverbs can be used on a single machine without any Infiniband hardware? The Verbs API seems like a very nice way to handle shared memory between processes where the number of shared buffers can vary in size and quantity. Regards Albert These kind of questions should be addressed by the Linux RDMA List (already CC) Tziporet -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] opensm: fixing compilation issues in some header files
Sasha,
On 24-Mar-10 5:50 PM, Yevgeny Kliteynik wrote:
All the compilation issues refer to implicit casting
from "void*" to "some_struct_t*"
This was detected when compiling a code that includes these
headers with g++ compiler. The headers should be able to compile
by g++ (they have 'extern "C"'). But the problem is not only
with g++ - it is with typing in general. I may be wrong, but I
feel that every new gcc version is more strongly typed.
Moreover, it appears that it would be more strongly typed in the
future, as GCC folks themselves are starting to compile GCC
with G++:
http://gcc.gnu.org/ml/gcc/2010-05/msg00705.html
-- Yevgeny
Signed-off-by: Yevgeny Kliteynik
---
opensm/include/opensm/osm_pkey.h |8 +---
opensm/include/opensm/osm_port.h |4 ++--
opensm/include/opensm/osm_subnet.h |2 +-
3 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/opensm/include/opensm/osm_pkey.h b/opensm/include/opensm/osm_pkey.h
index d10479d..53e9657 100644
--- a/opensm/include/opensm/osm_pkey.h
+++ b/opensm/include/opensm/osm_pkey.h
@@ -252,7 +252,8 @@ static inline ib_pkey_table_t *osm_pkey_tbl_block_get(const
osm_pkey_tbl_t *
uint16_t block)
{
return ((block< cl_ptr_vector_get_size(&p_pkey_tbl->blocks)) ?
- cl_ptr_vector_get(&p_pkey_tbl->blocks, block) : NULL);
+ (ib_pkey_table_t *)cl_ptr_vector_get(
+ &p_pkey_tbl->blocks, block) : NULL);
};
/*
@@ -282,8 +283,9 @@ static inline ib_pkey_table_t
*osm_pkey_tbl_new_block_get(const osm_pkey_tbl_t *
p_pkey_tbl,
uint16_t block)
{
- return (block< cl_ptr_vector_get_size(&p_pkey_tbl->new_blocks)) ?
- cl_ptr_vector_get(&p_pkey_tbl->new_blocks, block) : NULL;
+ return ((block< cl_ptr_vector_get_size(&p_pkey_tbl->new_blocks)) ?
+ (ib_pkey_table_t *)cl_ptr_vector_get(
+ &p_pkey_tbl->new_blocks, block) : NULL);
};
/f* OpenSM: osm_pkey_tbl_set_new_entry
diff --git a/opensm/include/opensm/osm_port.h b/opensm/include/opensm/osm_port.h
index ff0a178..8c68c99 100644
--- a/opensm/include/opensm/osm_port.h
+++ b/opensm/include/opensm/osm_port.h
@@ -549,7 +549,7 @@ static inline void osm_physp_set_slvl_tbl(IN osm_physp_t *
p_physp,
CL_ASSERT(p_slvl_tbl);
CL_ASSERT(osm_physp_is_valid(p_physp));
- p_tbl = cl_ptr_vector_get(&p_physp->slvl_by_port, in_port_num);
+ p_tbl = (ib_slvl_table_t *)cl_ptr_vector_get(&p_physp->slvl_by_port,
in_port_num);
*p_tbl = *p_slvl_tbl;
}
@@ -590,7 +590,7 @@ static inline ib_slvl_table_t *osm_physp_get_slvl_tbl(IN
const osm_physp_t *
ib_slvl_table_t *p_tbl;
CL_ASSERT(osm_physp_is_valid(p_physp));
- p_tbl = cl_ptr_vector_get(&p_physp->slvl_by_port, in_port_num);
+ p_tbl = (ib_slvl_table_t *)cl_ptr_vector_get(&p_physp->slvl_by_port,
in_port_num);
return p_tbl;
}
diff --git a/opensm/include/opensm/osm_subnet.h
b/opensm/include/opensm/osm_subnet.h
index 3970e98..2eef9c7 100644
--- a/opensm/include/opensm/osm_subnet.h
+++ b/opensm/include/opensm/osm_subnet.h
@@ -1015,7 +1015,7 @@ struct osm_mgrp *osm_get_mgrp_by_mgid(IN osm_subn_t *
subn, IN ib_gid_t * mgid);
*/
static inline struct osm_mgrp_box *osm_get_mbox_by_mlid(osm_subn_t const
*p_subn, ib_net16_t mlid)
{
- return p_subn->mboxes[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO];
+ return (struct osm_mgrp_box *)p_subn->mboxes[cl_ntoh16(mlid) -
IB_LID_MCAST_START_HO];
}
/*
* PARAMETERS
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
