Tamir, I have this in a for-next branch. Can someone from Mellanox verify this will work on all their switches. I did not think they always labeled their switches this way but perhaps they do.
Can the user program node descriptors in those chassis? I thought this was possible but I am unable to verify this today. Ira On Wed, 9 Feb 2011 07:41:09 -0800 Tamir Ronen <tam...@voltaire.com> wrote: > > From f1c92f4cae5605dec338b52ac6b261ddbe6e5609 Mon Sep 17 00:00:00 2001 > From: Tamir Ronen <tam...@voltaire.com> > Date: Tue, 8 Feb 2011 13:01:22 +0200 > Subject: [PATCH] add support for ibnetdiscover grouping for Mellanox switches > > Group Mellanox switches to the same chassis using SYSTEM_GUID > Use the node description format: > 'MF0;<system name>:<system type>/<system slot name>[:board type]/U<node > index>' > to identify system type, slot type Line/Spine, slotnum and anafanum. > Map Line ports 1-18 as external ports 1-18. > --- > .../libibnetdisc/include/infiniband/ibnetdisc.h | 3 + > infiniband-diags/libibnetdisc/src/chassis.c | 220 ++++++++++++++++++- > infiniband-diags/libibnetdisc/src/chassis.h | 3 + > 3 files changed, 214 insertions(+), 12 deletions(-) > > diff --git a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h > b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h > index 935e427..1efd1d6 100644 > --- a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h > +++ b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h > @@ -43,6 +43,8 @@ > struct ibnd_chassis; /* forward declare */ > struct ibnd_port; /* forward declare */ > > +#define CHASSIS_TYPE_SIZE 20 > + > /** ========================================================================= > * Node > */ > @@ -80,6 +82,7 @@ typedef struct ibnd_node { > struct ibnd_node *next_chassis_node; /* next node in > ibnd_chassis_t->nodes */ > struct ibnd_chassis *chassis; /* if != NULL the chassis this node > belongs to */ > unsigned char ch_type; > + char ch_type_str[CHASSIS_TYPE_SIZE]; > unsigned char ch_anafanum; > unsigned char ch_slotnum; > unsigned char ch_slot; > diff --git a/infiniband-diags/libibnetdisc/src/chassis.c > b/infiniband-diags/libibnetdisc/src/chassis.c > index 6bfc45b..3d90034 100644 > --- a/infiniband-diags/libibnetdisc/src/chassis.c > +++ b/infiniband-diags/libibnetdisc/src/chassis.c > @@ -62,30 +62,53 @@ typedef struct chassis_scan { > > char *ibnd_get_chassis_type(ibnd_node_t * node) > { > + int chassis_type; > + > if (!node) { > IBND_DEBUG("node parameter NULL\n"); > return NULL; > } > > - /* Currently, only if Voltaire chassis */ > - if (mad_get_field(node->info, 0, IB_NODE_VENDORID_F) != VTR_VENDOR_ID) > - return NULL; > if (!node->chassis) > return NULL; > - if (node->ch_type == UNRESOLVED_CT || node->ch_type > ISR4200_CT) > - return NULL; > - return ChassisTypeStr[node->ch_type]; > + > + chassis_type = mad_get_field(node->info, 0, IB_NODE_VENDORID_F); > + > + switch (chassis_type) > + { > + case VTR_VENDOR_ID: /* Voltaire chassis */ > + { > + if (node->ch_type == UNRESOLVED_CT || node->ch_type > > ISR4200_CT) > + return NULL; > + return ChassisTypeStr[node->ch_type]; > + } > + case MLX_VENDOR_ID: > + { > + if (node->ch_type_str[0] == '\0') > + return NULL; > + return node->ch_type_str; > + } > + default: > + { > + break; > + } > + } > + return NULL; > } > > char *ibnd_get_chassis_slot_str(ibnd_node_t * node, char *str, size_t size) > { > + int vendor_id; > + > if (!node) { > IBND_DEBUG("node parameter NULL\n"); > return NULL; > } > > - /* Currently, only if Voltaire chassis */ > - if (mad_get_field(node->info, 0, IB_NODE_VENDORID_F) != VTR_VENDOR_ID) > + /* Currently, only if Voltaire or Mellanox chassis */ > + vendor_id = mad_get_field(node->info, 0,IB_NODE_VENDORID_F); > + > + if ((vendor_id != VTR_VENDOR_ID) && (vendor_id != MLX_VENDOR_ID)) > return NULL; > if (!node->chassis) > return NULL; > @@ -574,6 +597,166 @@ static int get_slb_slot(ibnd_node_t * n, ibnd_port_t * > spineport) > return 0; > } > > + > +/* > + This function called for every Mellanox node in fabric > +*/ > +static int fill_mellanox_chassis_record(ibnd_node_t * node) > +{ > + int p = 0; > + ibnd_port_t *port; > + > + char node_desc[IB_SMP_DATA_SIZE]; > + char *system_name; > + char *system_type; > + char *system_slot_name; > + char *node_index; > + char *iter; > + int dev_id; > + > + /* > + The node description has the following format: > + > + 'MF0;<system name>:<system type>/<system slot name>[:board type]/U<node > index>' > + > + - System slot name in our systems can be L[01-36] , S[01-18] > + - Node index is always 1 (we don.t have boards with multiple IS4 chips). > + - System name is taken from the currently configured host name. > + -The board type is optional and we don.t set it currently - A leaf or > spine slot can currently hold a single type of board. > + */ > + > + memcpy(node_desc, node->nodedesc, IB_SMP_DATA_SIZE); > + > + IBND_DEBUG("fill_mellanox_chassis_record: node_desc:%s \n",node_desc); > + > + if (node->ch_found) /* somehow this node has already been passed */ > + return 0; > + > + /* All mellanox IS4 switches have the same vendor id*/ > + dev_id = mad_get_field(node->info, 0,IB_NODE_DEVID_F); > + if (dev_id != MLX_DEVID_IS4) > + return 0; > + > + if((node_desc[0] != 'M') || > + (node_desc[1] != 'F') || > + (node_desc[2] != '0') || > + (node_desc[3] != ';')) { > + IBND_DEBUG("fill_mellanox_chassis_record: Unsupported node > description format:%s \n",node_desc); > + return 0; > + } > + > + /* parse system name*/ > + system_name = &node_desc[4]; > + for (iter = system_name ; (*iter != ':') && (*iter != '\0') ; iter++); > + if(*iter == '\0'){ > + IBND_DEBUG("fill_mellanox_chassis_record: Unsupported node > description format:%s - (get system_name failed) \n",node_desc); > + return 0; > + } > + *iter = '\0'; > + iter++; > + /* parse system type*/ > + system_type = iter; > + for ( ; (*iter != '/') && (*iter != '\0') ; iter++); > + if(*iter == '\0'){ > + IBND_DEBUG("fill_mellanox_chassis_record: Unsupported node > description format:%s - (get system_type failed) \n",node_desc); > + return 0; > + } > + *iter = '\0'; > + iter++; > + /* parse system slot name*/ > + system_slot_name = iter; > + for ( ; (*iter != '/') && (*iter != ':') && (*iter != '\0') ; iter++); > + if(*iter == '\0'){ > + IBND_DEBUG("fill_mellanox_chassis_record: Unsupported node > description format:%s - (get system_slot_name failed) \n",node_desc); > + return 0; > + } > + if(*iter == ':'){ > + *iter = '\0'; > + iter++; > + for ( ; (*iter != '/') && (*iter != '\0') ; iter++); > + if(*iter == '\0'){ > + IBND_DEBUG("fill_mellanox_chassis_record: Unsupported > node description format:%s - (get board type failed) \n",node_desc); > + return 0; > + } > + } > + *iter = '\0'; > + iter++; > + node_index = iter; > + if(node_index[0] != 'U'){ > + IBND_DEBUG("fill_mellanox_chassis_record: Unsupported node > description format:%s - (get node index) \n",node_desc); > + return 0; > + } > + > + /* set Chip number (node index) */ > + node->ch_anafanum = atoi(&node_index[1]); > + if(node->ch_anafanum != 1){ > + IBND_DEBUG("Unexpected Chip number:%d \n",node->ch_anafanum); > + } > + > + > + /* set Line Spine numbers */ > + if(system_slot_name[0] == 'L') > + node->ch_slot = LINE_CS; > + else if(system_slot_name[0] == 'S') > + node->ch_slot = SPINE_CS; > + else{ > + IBND_DEBUG("fill_mellanox_chassis_record: Unsupported > system_slot_name:%s \n",system_slot_name); > + return 0; > + } > + > + /* The switch will be displayed under Line or Spine and not under > Chassis switches */ > + node->ch_found = 1; > + > + node->ch_slotnum = atoi(&system_slot_name[1]); > + if((node->ch_slot == LINE_CS && (node->ch_slotnum > (LINES_MAX_NUM + > 1))) || > + (node->ch_slot == SPINE_CS && (node->ch_slotnum > (SPINES_MAX_NUM + > 1)))){ > + IBND_ERROR("fill_mellanox_chassis_record: invalid slot > number:%d \n",node->ch_slotnum); > + node->ch_slotnum = 0; > + return 0; > + } > + > + /*set ch_type_str*/ > + strncpy(node->ch_type_str , system_type, sizeof(node->ch_type_str)-1); > + > + /* Line ports 1-18 are mapped to external ports 1-18*/ > + if(node->ch_slot == LINE_CS) > + { > + for (p = 1; p <= node->numports && p <= 18 ; p++) { > + port = node->ports[p]; > + if (!port) > + continue; > + port->ext_portnum = p; > + } > + } > + > + return 0; > +} > + > +static int insert_mellanox_line_and_spine(ibnd_node_t * node, ibnd_chassis_t > * chassis) > +{ > + if (node->ch_slot == LINE_CS){ > + > + if (chassis->linenode[node->ch_slotnum]) > + return 0; /* already filled slot */ > + > + chassis->linenode[node->ch_slotnum] = node; > + } > + else if (node->ch_slot == SPINE_CS){ > + > + if (chassis->spinenode[node->ch_slotnum]) > + return 0; /* already filled slot */ > + > + chassis->spinenode[node->ch_slotnum] = node; > + } > + else > + return 0; > + > + node->chassis = chassis; > + > + return 0; > +} > + > + > /* forward declare this */ > static void voltaire_portmap(ibnd_port_t * port); > /* > @@ -1064,15 +1247,23 @@ int group_nodes(ibnd_fabric_t * fabric) > chassis_scan.current_chassis = NULL; > chassis_scan.last_chassis = NULL; > > + int vendor_id; > + > /* first pass on switches and build for every Voltaire node */ > /* an appropriate chassis record (slotnum and position) */ > /* according to internal connectivity */ > /* not very efficient but clear code so... */ > for (node = fabric->switches; node; node = node->type_next) { > - if (mad_get_field(node->info, 0, > - IB_NODE_VENDORID_F) == VTR_VENDOR_ID > + > + vendor_id = mad_get_field(node->info, 0,IB_NODE_VENDORID_F); > + > + if (vendor_id == VTR_VENDOR_ID > && fill_voltaire_chassis_record(node)) > goto cleanup; > + else if (vendor_id == MLX_VENDOR_ID > + && fill_mellanox_chassis_record(node)) > + goto cleanup; > + > } > > /* separate every Voltaire chassis from each other and build linked > list of them */ > @@ -1118,8 +1309,10 @@ int group_nodes(ibnd_fabric_t * fabric) > /* now, make another pass to see which nodes are part of chassis */ > /* (defined as chassis->nodecount > 1) */ > for (node = fabric->nodes; node; node = node->next) { > - if (mad_get_field(node->info, 0, > - IB_NODE_VENDORID_F) == VTR_VENDOR_ID) > + > + vendor_id = mad_get_field(node->info, 0,IB_NODE_VENDORID_F); > + > + if (vendor_id == VTR_VENDOR_ID) > continue; > if (mad_get_field64(node->info, 0, IB_NODE_SYSTEM_GUID_F)) { > chassis = find_chassisguid(fabric, node); > @@ -1130,6 +1323,9 @@ int group_nodes(ibnd_fabric_t * fabric) > node->ch_found = 1; > add_node_to_chassis(chassis, node); > } > + else if (vendor_id == MLX_VENDOR_ID){ > + insert_mellanox_line_and_spine(node, > chassis); > + } > } > } > } > diff --git a/infiniband-diags/libibnetdisc/src/chassis.h > b/infiniband-diags/libibnetdisc/src/chassis.h > index 004156e..7a91be3 100644 > --- a/infiniband-diags/libibnetdisc/src/chassis.h > +++ b/infiniband-diags/libibnetdisc/src/chassis.h > @@ -75,8 +75,11 @@ > #define VTR_DEVID_SFB4700X2 0x5a5d > #define VTR_DEVID_SFB4200 0x5a60 > > +#define MLX_DEVID_IS4 0xbd36 > + > /* Vendor IDs (for chassis based systems) */ > #define VTR_VENDOR_ID 0x8f1 /* Voltaire */ > +#define MLX_VENDOR_ID 0x2c9 /* Mellanox */ > #define TS_VENDOR_ID 0x5ad /* Cisco */ > #define SS_VENDOR_ID 0x66a /* InfiniCon */ > #define XS_VENDOR_ID 0x1397 /* Xsigo */ > -- > 1.5.5.6 > -- Ira Weiny Math Programmer/Computer Scientist Lawrence Livermore National Lab 925-423-8008 wei...@llnl.gov -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html