rather than starting from VL 0 Signed-off-by: Robert Pearson <[email protected]> Signed-off-by: Hal Rosenstock <[email protected]> --- diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 59a32ad..da8cc5e 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -221,6 +222,7 @@ typedef struct osm_subn_opt { char *prefix_routes_file; boolean_t consolidate_ipv6_snm_req; struct osm_subn_opt *file_opts; /* used for update */ + uint8_t lash_start_vl; /* starting vl to use in lash */ } osm_subn_opt_t; /* * FIELDS diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in index 66d2fe6..e8801c9 100644 --- a/opensm/man/opensm.8.in +++ b/opensm/man/opensm.8.in @@ -1,4 +1,4 @@ -.TH OPENSM 8 "April 22, 2009" "OpenIB" "OpenIB Management" +.TH OPENSM 8 "May 28, 2009" "OpenIB" "OpenIB Management" .SH NAME opensm \- InfiniBand subnet manager and administration (SM/SA) @@ -15,6 +15,7 @@ opensm \- InfiniBand subnet manager and administration (SM/SA) [\-r(eassign_lids)] [\-R <engine name(s)> | \-\-routing_engine <engine name(s)>] [\-\-do_mesh_analysis] +[\-\-lash_start_vl <vl number>] [\-A | \-\-ucast_cache] [\-z | \-\-connect_roots] [\-M <file name> | \-\-lid_matrix_file <file name>] @@ -147,6 +148,10 @@ This option enables additional analysis for the lash routing engine to precondition switch port assignments in regular cartesian meshes which may reduce the number of SLs required to give a deadlock free routing. .TP +\fB\-\-lash_start_vl\fR <vl number> +This option sets the starting VL to use for the lash routing algorithm. +Defaults to 0. +.TP \fB\-A\fR, \fB\-\-ucast_cache\fR This option enables unicast routing cache and prevents routing recalculation (which is a heavy task in a large cluster) when diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c index 296d5d5..d682ff5 100644 --- a/opensm/opensm/main.c +++ b/opensm/opensm/main.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009 HNR Consulting. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -181,6 +182,9 @@ static void show_usage(void) " routing engine to precondition switch port assignments\n" " in regular cartesian meshes which may reduce the number\n" " of SLs required to give a deadlock free routing\n\n"); + printf("--lash_start_vl <vl number>\n" + " Sets the starting VL to use for the lash routing algorithm.\n" + " Defaults to 0.\n"); printf("--connect_roots, -z\n" " This option enforces a routing engine (currently\n" " up/down only) to make connectivity between root switches\n" @@ -601,6 +605,7 @@ int main(int argc, char *argv[]) {"prefix_routes_file", 1, NULL, 3}, {"consolidate_ipv6_snm_req", 0, NULL, 4}, {"do_mesh_analysis", 0, NULL, 5}, + {"lash_start_vl", 1, NULL, 6}, {NULL, 0, NULL, 0} /* Required at the end of the array */ }; @@ -951,6 +956,15 @@ int main(int argc, char *argv[]) case 5: opt.do_mesh_analysis = TRUE; break; + case 6: + temp = strtol(optarg, NULL, 0); + if (temp < 0 || temp >= IB_MAX_NUM_VLS) { + fprintf(stderr, + "ERROR: starting lash vl must be between 0 and 15\n"); + return (-1); + } + opt.lash_start_vl = (uint8_t) temp; + break; case 'h': case '?': case ':': diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index ec15f8a..fda2eb0 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -389,6 +390,7 @@ static const opt_rec_t opt_tbl[] = { { "no_clients_rereg", OPT_OFFSET(no_clients_rereg), opts_parse_boolean, NULL, 1 }, { "prefix_routes_file", OPT_OFFSET(prefix_routes_file), opts_parse_charp, NULL, 0 }, { "consolidate_ipv6_snm_req", OPT_OFFSET(consolidate_ipv6_snm_req), opts_parse_boolean, NULL, 1 }, + { "lash_start_vl", OPT_OFFSET(lash_start_vl), opts_parse_uint8, NULL, 1 }, {0} }; @@ -749,6 +751,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt) p_opt->no_clients_rereg = FALSE; p_opt->prefix_routes_file = strdup(OSM_DEFAULT_PREFIX_ROUTES_FILE); p_opt->consolidate_ipv6_snm_req = FALSE; + p_opt->lash_start_vl = 0; subn_init_qos_options(&p_opt->qos_options, NULL); subn_init_qos_options(&p_opt->qos_ca_options, NULL); subn_init_qos_options(&p_opt->qos_sw0_options, NULL); @@ -1432,6 +1435,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) p_opts->do_mesh_analysis ? "TRUE" : "FALSE"); fprintf(out, + "# Starting VL for LASH algorithm\n" + "lash_start_vl %d\n\n", + p_opts->lash_start_vl); + + fprintf(out, "# SA database file name\nsa_db_file %s\n\n", p_opts->sa_db_file ? p_opts->sa_db_file : null_str); diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 12b5e34..540214c 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -478,7 +478,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix; int *num_mst_in_lane = p_lash->num_mst_in_lane; int ***virtual_location = p_lash->virtual_location; - int min_filled_lane, max_filled_lane, trials; + int min_filled_lane, max_filled_lane, trials, max_vl; int old_min_filled_lane, old_max_filled_lane, new_num_min_lane, new_num_max_lane; unsigned int i, j; @@ -486,9 +486,13 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) int next_switch2, output_link2; int stop = 0, cycle_found; int cycle_found2; + osm_subn_opt_t *opt = &p_lash->p_osm->subn.opt; - max_filled_lane = 0; - min_filled_lane = lanes_needed - 1; + max_filled_lane = opt->lash_start_vl; + max_vl = lanes_needed + opt->lash_start_vl; + if (max_vl > IB_MAX_NUM_VLS) + max_vl = IB_MAX_NUM_VLS; + min_filled_lane = max_vl - 1; trials = num_mst_in_lane[max_filled_lane]; if (lanes_needed == 1) @@ -590,7 +594,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) new_num_min_lane = MAX_INT; new_num_max_lane = 0; - for (i = 0; i < lanes_needed; i++) { + for (i = opt->lash_start_vl; i < max_vl; i++) { if (num_mst_in_lane[i] < new_num_min_lane) { new_num_min_lane = num_mst_in_lane[i]; @@ -673,12 +677,18 @@ static void free_lash_structures(lash_t * p_lash) { unsigned int i, j, k; unsigned num_switches = p_lash->num_switches; + int max_vl; osm_log_t *p_log = &p_lash->p_osm->log; + osm_subn_opt_t *opt = &p_lash->p_osm->subn.opt; OSM_LOG_ENTER(p_log); + max_vl = opt->lash_start_vl + p_lash->vl_min; + if (max_vl > IB_MAX_NUM_VLS) + max_vl = IB_MAX_NUM_VLS; + /* free cdg_vertex_matrix */ - for (i = 0; i < p_lash->vl_min; i++) { + for (i = opt->lash_start_vl; i < max_vl; i++) { for (j = 0; j < num_switches; j++) { for (k = 0; k < num_switches; k++) if (p_lash->cdg_vertex_matrix[i][j][k]) @@ -715,13 +725,19 @@ static int init_lash_structures(lash_t * p_lash) osm_log_t *p_log = &p_lash->p_osm->log; int status = 0; unsigned int i, j, k; + int max_vl; + osm_subn_opt_t *opt = &p_lash->p_osm->subn.opt; OSM_LOG_ENTER(p_log); + max_vl = vl_min + opt->lash_start_vl; + if (max_vl > IB_MAX_NUM_VLS) + max_vl = IB_MAX_NUM_VLS; + /* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */ p_lash->cdg_vertex_matrix = - (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****)); - for (i = 0; i < vl_min; i++) { + (cdg_vertex_t ****) calloc(max_vl, sizeof(cdg_vertex_t ****)); + for (i = opt->lash_start_vl; i < max_vl; i++) { p_lash->cdg_vertex_matrix[i] = (cdg_vertex_t ***) malloc(num_switches * sizeof(cdg_vertex_t ***)); @@ -730,7 +746,7 @@ static int init_lash_structures(lash_t * p_lash) goto Exit_Mem_Error; } - for (i = 0; i < vl_min; i++) { + for (i = opt->lash_start_vl; i < max_vl; i++) { for (j = 0; j < num_switches; j++) { p_lash->cdg_vertex_matrix[i][j] = (cdg_vertex_t **) malloc(num_switches * @@ -763,7 +779,7 @@ static int init_lash_structures(lash_t * p_lash) for (i = 0; i < num_switches; i++) { for (j = 0; j < num_switches; j++) { p_lash->virtual_location[i][j] = - (int *)malloc(vl_min * sizeof(int *)); + (int *)calloc(max_vl, sizeof(int *)); if (p_lash->virtual_location[i][j] == NULL) goto Exit_Mem_Error; for (k = 0; k < vl_min; k++) { @@ -804,10 +820,12 @@ static int lash_core(lash_t * p_lash) int cycle_found2 = 0; int status = 0; int *switch_bitmap = NULL; /* Bitmap to check if we have processed this pair */ + int max_vl; + osm_subn_opt_t *opt = &p_lash->p_osm->subn.opt; OSM_LOG_ENTER(p_log); - if (p_lash->p_osm->subn.opt.do_mesh_analysis && osm_do_mesh_analysis(p_lash)) { + if (opt->do_mesh_analysis && osm_do_mesh_analysis(p_lash)) { OSM_LOG(p_log, OSM_LOG_ERROR, "Mesh analysis failed\n"); goto Exit; } @@ -838,11 +856,14 @@ static int lash_core(lash_t * p_lash) } for (i = 0; i < num_switches; i++) { - for (dest_switch = 0; dest_switch < num_switches; dest_switch++) + for (dest_switch = 0; dest_switch < num_switches; dest_switch++) { + max_vl = lanes_needed + opt->lash_start_vl; + if (max_vl > IB_MAX_NUM_VLS) + max_vl = IB_MAX_NUM_VLS; if (dest_switch != i && switch_bitmap[i * num_switches + dest_switch] == 0) { - v_lane = 0; + v_lane = opt->lash_start_vl; stop = 0; - while (v_lane < lanes_needed && stop == 0) { + while (v_lane < max_vl && stop == 0) { generate_cdg_for_sp(p_lash, i, dest_switch, v_lane); generate_cdg_for_sp(p_lash, dest_switch, i, v_lane); @@ -906,7 +927,9 @@ static int lash_core(lash_t * p_lash) switches[dest_switch]->routing_table[i].lane = v_lane; if (cycle_found == 1 || cycle_found2 == 1) { - if (++lanes_needed > p_lash->vl_min) + lanes_needed++; + if (lanes_needed > p_lash->vl_min || + opt->lash_start_vl + lanes_needed - 1 >= IB_DROP_VL) goto Error_Not_Enough_Lanes; generate_cdg_for_sp(p_lash, i, dest_switch, v_lane); @@ -926,19 +949,24 @@ static int lash_core(lash_t * p_lash) switch_bitmap[i * num_switches + dest_switch] = 1; switch_bitmap[dest_switch * num_switches + i] = 1; } + } } OSM_LOG(p_log, OSM_LOG_INFO, "Lanes needed: %d, Balancing\n", lanes_needed); - for (i = 0; i < lanes_needed; i++) { + max_vl = lanes_needed + opt->lash_start_vl; + if (max_vl > IB_MAX_NUM_VLS) + max_vl = IB_MAX_NUM_VLS; + + for (i = opt->lash_start_vl; i < max_vl; i++) { OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", i, p_lash->num_mst_in_lane[i]); } balance_virtual_lanes(p_lash, lanes_needed); - for (i = 0; i < lanes_needed; i++) { + for (i = opt->lash_start_vl; i < max_vl; i++) { OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", i, p_lash->num_mst_in_lane[i]); } @@ -1271,6 +1299,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port, unsigned dst_id; unsigned src_id; osm_switch_t *p_sw; + osm_subn_opt_t *opt = &p_osm->subn.opt; if (p_osm->routing_engine_used != OSM_ROUTING_ENGINE_TYPE_LASH) return OSM_DEFAULT_SL; @@ -1286,7 +1315,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port, src_id = get_lash_id(p_sw); if (src_id == dst_id) - return OSM_DEFAULT_SL; + return opt->lash_start_vl; return (uint8_t) ((switch_t *) p_sw->priv)->routing_table[dst_id].lane; } _______________________________________________ general mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
