--- a/write_bw.c +++ b/write_bw.c @@ -75,6 +75,7 @@ struct user_parameters { int maxpostsofqpiniteration; int inline_size; int qp_timeout; + int gid_index; /* if value not negative, we use gid AND gid_index=value */ }; struct extended_qp { struct ibv_qp *qp; @@ -98,6 +99,7 @@ struct pingpong_context { struct ibv_send_wr wr; int *scnt; int *ccnt; + union ibv_gid dgid; }; struct pingpong_dest { @@ -106,6 +108,7 @@ struct pingpong_dest { int psn; unsigned rkey; unsigned long long vaddr; + union ibv_gid dgid; }; @@ -160,14 +163,20 @@ static int pp_client_connect(const char *servername, int port) } struct pingpong_dest * pp_client_exch_dest(int sockfd, - const struct pingpong_dest *my_dest) + const struct pingpong_dest *my_dest, struct user_parameters *user_parm) { struct pingpong_dest *rem_dest = NULL; - char msg[sizeof "0000:000000:000000:00000000:0000000000000000"]; + char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"]; int parsed; - sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn, - my_dest->psn,my_dest->rkey,my_dest->vaddr); + sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x", + my_dest->lid, my_dest->qpn, my_dest->psn,my_dest->rkey,my_dest->vaddr, + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2], + my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5], + my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8], + my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11], + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14], + my_dest->dgid.raw[15]); if (write(sockfd, msg, sizeof msg) != sizeof msg) { perror("client write"); fprintf(stderr, "Couldn't send local address\n"); @@ -184,15 +193,59 @@ struct pingpong_dest * pp_client_exch_dest(int sockfd, if (!rem_dest) goto out; - parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn, - &rem_dest->psn,&rem_dest->rkey,&rem_dest->vaddr); - - if (parsed != 5) { - fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, - msg); - free(rem_dest); - rem_dest = NULL; - goto out; + if (user_parm->gid_index < 0) { + parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn, + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr); + if (parsed != 5) { + fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg); + free(rem_dest); + rem_dest = NULL; + goto out; + } + }else{ + char *pstr = msg, *term; + char tmp[20]; + int i; + + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA + + for (i = 0; i < 15; ++i) { + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16); + } + pstr += term - pstr + 1; + strcpy(tmp, pstr); + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16); } out: return rem_dest; @@ -254,9 +307,9 @@ int pp_server_connect(int port) return connfd; } -static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest) +static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm) { - char msg[sizeof "0000:000000:000000:00000000:0000000000000000"]; + char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"]; struct pingpong_dest *rem_dest = NULL; int parsed; int n; @@ -272,18 +325,69 @@ static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpo if (!rem_dest) goto out; - parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn, - &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr); - if (parsed != 5) { - fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, - msg); - free(rem_dest); - rem_dest = NULL; - goto out; + if (user_parm->gid_index < 0) { + parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn, + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr); + if (parsed != 5) { + fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg); + free(rem_dest); + rem_dest = NULL; + goto out; + } + }else{ + char *pstr = msg, *term; + char tmp[20]; + int i; + + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA + + for (i = 0; i < 15; ++i) { + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16); + } + pstr += term - pstr + 1; + strcpy(tmp, pstr); + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16); } - sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn, - my_dest->psn, my_dest->rkey, my_dest->vaddr); + sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x", + my_dest->lid, my_dest->qpn, my_dest->psn, my_dest->rkey, my_dest->vaddr, + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2], + my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5], + my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8], + my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11], + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14], + my_dest->dgid.raw[15]); if (write(connfd, msg, sizeof msg) != sizeof msg) { perror("server write"); fprintf(stderr, "Couldn't send local address\n"); @@ -341,7 +445,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, fprintf(stderr, "Failed to query device props"); return NULL; } - if (device_attr.vendor_part_id == 23108) { + if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) { user_parm->mtu = 1024; } else { user_parm->mtu = 2048; @@ -444,9 +548,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, attr.max_dest_rd_atomic = 1; attr.min_rnr_timer = 12; } - attr.ah_attr.is_global = 0; - attr.ah_attr.dlid = dest->lid; - attr.ah_attr.sl = sl; + if (user_parm->gid_index<0) { + attr.ah_attr.is_global = 0; + attr.ah_attr.dlid = dest->lid; + attr.ah_attr.sl = sl; + } else { + attr.ah_attr.is_global = 1; + attr.ah_attr.grh.dgid = dest->dgid; + attr.ah_attr.grh.hop_limit = 1; + attr.ah_attr.sl = 0; + } attr.ah_attr.src_path_bits = 0; attr.ah_attr.port_num = port; if (user_parm->connection_type == RC) { @@ -524,6 +635,7 @@ static void usage(const char *argv0) printf(" -I, --inline_size=<size> max size of message to be sent in inline mode (default 400)\n"); printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n"); printf(" -S, --sl=<sl> SL (default 0)\n"); + printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n"); printf(" -b, --bidirectional measure bidirectional bandwidth (default unidirectional)\n"); printf(" -V, --version display version number\n"); printf(" -N, --no peak-bw cancel peak-bw calculation (default with peak-bw)\n"); @@ -687,6 +799,7 @@ int main(int argc, char *argv[]) int inline_given_in_cmd = 0; struct ibv_context *context; int no_cpu_freq_fail = 0; + union ibv_gid gid; /* init default values to user's parameters */ memset(&user_param, 0, sizeof(struct user_parameters)); @@ -698,6 +811,7 @@ int main(int argc, char *argv[]) user_param.maxpostsofqpiniteration = 100; user_param.inline_size = MAX_INLINE; user_param.qp_timeout = 14; + user_param.gid_index = -1; /*gid will not be used*/ /* Parameter parsing. */ while (1) { int c; @@ -716,6 +830,7 @@ int main(int argc, char *argv[]) { .name = "inline_size", .has_arg = 1, .val = 'I' }, { .name = "qp-timeout", .has_arg = 1, .val = 'u' }, { .name = "sl", .has_arg = 1, .val = 'S' }, + { .name = "gid-index", .has_arg = 1, .val = 'x' }, { .name = "all", .has_arg = 0, .val = 'a' }, { .name = "bidirectional", .has_arg = 0, .val = 'b' }, { .name = "version", .has_arg = 0, .val = 'V' }, @@ -724,7 +839,7 @@ int main(int argc, char *argv[]) { 0 } }; - c = getopt_long(argc, argv, "p:d:i:m:q:g:c:s:n:t:I:u:S:baVNF", long_options, NULL); + c = getopt_long(argc, argv, "p:d:i:m:q:g:c:s:n:t:I:u:S:x:baVNF", long_options, NULL); if (c == -1) break; @@ -821,6 +936,14 @@ int main(int argc, char *argv[]) if (sl > 15) { usage(argv[0]); return 1; } break; + case 'x': + user_param.gid_index = strtol(optarg, NULL, 0); + if (user_param.gid_index > 63) { + usage(argv[0]); + return 1; + } + break; + default: usage(argv[0]); return 1; @@ -855,6 +978,9 @@ int main(int argc, char *argv[]) printf("Can not post more than iterations per qp , adjusting max number of post to num of iteration\n"); user_param.maxpostsofqpiniteration = user_param.iters; } + if (user_param.gid_index > -1) { + printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n"); + } printf("Each Qp will post up to %d messages each time\n",user_param.maxpostsofqpiniteration); /* Done with parameter parsing. Perform setup. */ if (user_param.all == ALL) { @@ -901,7 +1027,15 @@ int main(int argc, char *argv[]) if (!ctx) return 1; - + if (user_param.gid_index != -1) { + int err=0; + err = ibv_query_gid (ctx->context, ib_port, user_param.gid_index, &gid); + if (err) { + return -1; + } + ctx->dgid=gid; + } + if (user_param.servername) { sockfd = pp_client_connect(user_param.servername, port); if (sockfd < 0) @@ -928,10 +1062,13 @@ int main(int argc, char *argv[]) * We do it by exchanging data over a TCP socket connection. */ my_dest[i].lid = pp_get_local_lid(ctx, ib_port); my_dest[i].psn = lrand48() & 0xffffff; - if (!my_dest[i].lid) { - fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n"); - return 1; - } + if (user_param.gid_index < 0) {/*We do not fail test upon lid in RDMA0E/Eth conf*/ + if (!my_dest[i].lid) { + fprintf(stderr, "Local lid 0x0 detected. Is an SM running? If you are running on an RMDAoE interface you must use GIDs\n"); + return 1; + } + } + my_dest[i].dgid = gid; my_dest[i].qpn = ctx->qp[i]->qp_num; /* TBD this should be changed inot VA and different key to each qp */ my_dest[i].rkey = ctx->mr->rkey; @@ -941,10 +1078,19 @@ int main(int argc, char *argv[]) "RKey %#08x VAddr %#016Lx\n", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, my_dest[i].rkey, my_dest[i].vaddr); + if (user_param.gid_index > -1) { + printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n", + my_dest[i].dgid.raw[0],my_dest[i].dgid.raw[1], + my_dest[i].dgid.raw[2], my_dest[i].dgid.raw[3], my_dest[i].dgid.raw[4], + my_dest[i].dgid.raw[5], my_dest[i].dgid.raw[6], my_dest[i].dgid.raw[7], + my_dest[i].dgid.raw[8], my_dest[i].dgid.raw[9], my_dest[i].dgid.raw[10], + my_dest[i].dgid.raw[11], my_dest[i].dgid.raw[12], my_dest[i].dgid.raw[13], + my_dest[i].dgid.raw[14], my_dest[i].dgid.raw[15]); + } if (user_param.servername) { - rem_dest[i] = pp_client_exch_dest(sockfd, &my_dest[i]); + rem_dest[i] = pp_client_exch_dest(sockfd, &my_dest[i], &user_param); } else { - rem_dest[i] = pp_server_exch_dest(sockfd, &my_dest[i]); + rem_dest[i] = pp_server_exch_dest(sockfd, &my_dest[i], &user_param); } if (!rem_dest[i]) return 1; @@ -952,15 +1098,24 @@ int main(int argc, char *argv[]) "RKey %#08x VAddr %#016Lx\n", rem_dest[i]->lid, rem_dest[i]->qpn, rem_dest[i]->psn, rem_dest[i]->rkey, rem_dest[i]->vaddr); + if (user_param.gid_index > -1) { + printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n", + rem_dest[i]->dgid.raw[0],rem_dest[i]->dgid.raw[1], + rem_dest[i]->dgid.raw[2], rem_dest[i]->dgid.raw[3], rem_dest[i]->dgid.raw[4], + rem_dest[i]->dgid.raw[5], rem_dest[i]->dgid.raw[6], rem_dest[i]->dgid.raw[7], + rem_dest[i]->dgid.raw[8], rem_dest[i]->dgid.raw[9], rem_dest[i]->dgid.raw[10], + rem_dest[i]->dgid.raw[11], rem_dest[i]->dgid.raw[12], rem_dest[i]->dgid.raw[13], + rem_dest[i]->dgid.raw[14], rem_dest[i]->dgid.raw[15]); + } if (pp_connect_ctx(ctx, ib_port, my_dest[i].psn, rem_dest[i], &user_param, i)) return 1; /* An additional handshake is required *after* moving qp to RTR. Arbitrarily reuse exch_dest for this purpose. */ if (user_param.servername) { - rem_dest[i] = pp_client_exch_dest(sockfd, &my_dest[i]); + rem_dest[i] = pp_client_exch_dest(sockfd, &my_dest[i], &user_param); } else { - rem_dest[i] = pp_server_exch_dest(sockfd, &my_dest[i]); + rem_dest[i] = pp_server_exch_dest(sockfd, &my_dest[i], &user_param); } } @@ -969,7 +1124,7 @@ int main(int argc, char *argv[]) /* For half duplex tests, server just waits for client to exit */ /* the 0th place is arbitrary to signal finish ... */ if (!user_param.servername && !duplex) { - rem_dest[0] = pp_server_exch_dest(sockfd, &my_dest[0]); + rem_dest[0] = pp_server_exch_dest(sockfd, &my_dest[0], &user_param); if (write(sockfd, "done", sizeof "done") != sizeof "done"){ perror("server write"); fprintf(stderr, "Couldn't write to socket\n"); @@ -1007,9 +1162,9 @@ int main(int argc, char *argv[]) } /* the 0th place is arbitrary to signal finish ... */ if (user_param.servername) { - rem_dest[0] = pp_client_exch_dest(sockfd, &my_dest[0]); + rem_dest[0] = pp_client_exch_dest(sockfd, &my_dest[0], &user_param); } else { - rem_dest[0] = pp_server_exch_dest(sockfd, &my_dest[0]); + rem_dest[0] = pp_server_exch_dest(sockfd, &my_dest[0], &user_param); } if (write(sockfd, "done", sizeof "done") != sizeof "done"){
--- a/write_lat.c +++ b/write_lat.c @@ -73,6 +73,7 @@ struct user_parameters { int tx_depth; int inline_size; int qp_timeout; + int gid_index; /* if value not negative, we use gid AND gid_index=value */ }; struct report_options { int unsorted; @@ -94,6 +95,7 @@ struct pingpong_context { int tx_depth; struct ibv_sge list; struct ibv_send_wr wr; + union ibv_gid dgid; }; struct pingpong_dest { @@ -102,6 +104,7 @@ struct pingpong_dest { int psn; unsigned rkey; unsigned long long vaddr; + union ibv_gid dgid; }; @@ -137,14 +140,33 @@ static struct ibv_device *pp_find_dev(const char *ib_devname) { #define KEY_MSG_SIZE (sizeof "0000:000000:000000:00000000:0000000000000000") #define KEY_PRINT_FMT "%04x:%06x:%06x:%08x:%016Lx" +#define KEY_MSG_SIZE_GID (sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00") +#define KEY_PRINT_FMT_GID "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x" -static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest) +static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm) { - char msg[KEY_MSG_SIZE]; + if (user_parm->gid_index < 0) { + char msg[KEY_MSG_SIZE]; - sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn, - my_dest->psn, my_dest->rkey, my_dest->vaddr); + sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn, + my_dest->psn, my_dest->rkey, my_dest->vaddr); + if (write(sockfd, msg, sizeof msg) != sizeof msg) { + perror("client write"); + fprintf(stderr, "Couldn't send local address\n"); + return -1; + } + + return 0; + } else { + char msg[KEY_MSG_SIZE_GID]; + + sprintf(msg, KEY_PRINT_FMT_GID, my_dest->lid, my_dest->qpn, + my_dest->psn, my_dest->rkey, my_dest->vaddr, + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2], my_dest->dgid.raw[3], + my_dest->dgid.raw[4], my_dest->dgid.raw[5], my_dest->dgid.raw[6], my_dest->dgid.raw[7], + my_dest->dgid.raw[8], my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11], + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14], my_dest->dgid.raw[15]); if (write(sockfd, msg, sizeof msg) != sizeof msg) { perror("client write"); fprintf(stderr, "Couldn't send local address\n"); @@ -152,30 +174,84 @@ static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest) } return 0; + } } static int pp_read_keys(int sockfd, const struct pingpong_dest *my_dest, - struct pingpong_dest *rem_dest) + struct pingpong_dest *rem_dest, struct user_parameters *user_parm) { - int parsed; - char msg[KEY_MSG_SIZE]; - - if (read(sockfd, msg, sizeof msg) != sizeof msg) { - perror("pp_read_keys"); - fprintf(stderr, "Couldn't read remote address\n"); - return -1; - } + if (user_parm->gid_index < 0) { + int parsed; + char msg[KEY_MSG_SIZE]; + + if (read(sockfd, msg, sizeof msg) != sizeof msg) { + perror("pp_read_keys"); + fprintf(stderr, "Couldn't read remote address\n"); + return -1; + } - parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn, - &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr); + parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn, + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr); - if (parsed != 5) { - fprintf(stderr, "Couldn't parse line <%.*s>\n", - (int)sizeof msg, msg); - return -1; - } + if (parsed != 5) { + fprintf(stderr, "Couldn't parse line <%.*s>\n", + (int)sizeof msg, msg); + return -1; + } - return 0; + return 0; + } else { + char msg[KEY_MSG_SIZE_GID]; + if (read(sockfd, msg, sizeof msg) != sizeof msg) { + perror("pp_read_keys"); + fprintf(stderr, "Couldn't read remote address\n"); + return -1; + } + char *pstr = msg, *term; + char tmp[20]; + int i; + + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY + + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA + + for (i = 0; i < 15; ++i) { + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16); + } + pstr += term - pstr + 1; + strcpy(tmp, pstr); + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16); + return 0; + } } static int pp_client_connect(const char *servername, int port) @@ -220,12 +296,12 @@ static int pp_client_connect(const char *servername, int port) } static int pp_client_exch_dest(int sockfd, const struct pingpong_dest *my_dest, - struct pingpong_dest *rem_dest) + struct pingpong_dest *rem_dest, struct user_parameters *user_parm) { - if (pp_write_keys(sockfd, my_dest)) + if (pp_write_keys(sockfd, my_dest, user_parm)) return -1; - return pp_read_keys(sockfd, my_dest, rem_dest); + return pp_read_keys(sockfd, my_dest, rem_dest, user_parm); } static int pp_server_connect(int port) @@ -285,13 +361,13 @@ static int pp_server_connect(int port) } static int pp_server_exch_dest(int sockfd, const struct pingpong_dest *my_dest, - struct pingpong_dest* rem_dest) + struct pingpong_dest* rem_dest, struct user_parameters *user_parm) { - if (pp_read_keys(sockfd, my_dest, rem_dest)) + if (pp_read_keys(sockfd, my_dest, rem_dest, user_parm)) return -1; - return pp_write_keys(sockfd, my_dest); + return pp_write_keys(sockfd, my_dest, user_parm); } static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, @@ -328,7 +404,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, fprintf(stderr, "Failed to query device props"); return NULL; } - if (device_attr.vendor_part_id == 23108) { + if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) { user_parm->mtu = 1024; } else { user_parm->mtu = 2048; @@ -439,9 +515,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, attr.min_rnr_timer = 12; } - attr.ah_attr.is_global = 0; - attr.ah_attr.dlid = dest->lid; - attr.ah_attr.sl = sl; + if (user_parm->gid_index < 0) { + attr.ah_attr.is_global = 0; + attr.ah_attr.dlid = dest->lid; + attr.ah_attr.sl = sl; + } else { + attr.ah_attr.is_global = 1; + attr.ah_attr.grh.dgid = dest->dgid; + attr.ah_attr.grh.hop_limit = 1; + attr.ah_attr.sl = 0; + } attr.ah_attr.src_path_bits = 0; attr.ah_attr.port_num = port; @@ -506,23 +589,46 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername, struct pingpong_dest my_dest; int sockfd; int rc; + union ibv_gid gid; /* Create connection between client and server. * We do it by exchanging data over a TCP socket connection. */ + + if (user_parm->gid_index != -1) { + int err=0; + err = ibv_query_gid (ctx->context, ib_port, user_parm->gid_index, &gid); + if (err) { + return -1; + } + ctx->dgid=gid; + } + my_dest.lid = pp_get_local_lid(ctx, ib_port); + my_dest.dgid = gid; my_dest.qpn = ctx->qp->qp_num; my_dest.psn = lrand48() & 0xffffff; - if (!my_dest.lid) { - fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n"); - return -1; + if (user_parm->gid_index < 0) {/*We do not fail test upon lid in RDMAoE/Eth conf*/ + if (!my_dest.lid) { + fprintf(stderr, "Local lid 0x0 detected. Is an SM running? If you are running on an RMDAoE interface you must use GIDs\n"); + return 1; + } } my_dest.rkey = ctx->mr->rkey; my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size; printf(addr_fmt, "local", my_dest.lid, my_dest.qpn, my_dest.psn, my_dest.rkey, my_dest.vaddr); + if (user_parm->gid_index > -1) { + printf(" GID: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n", + my_dest.dgid.raw[0],my_dest.dgid.raw[1], + my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4], + my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7], + my_dest.dgid.raw[8], my_dest.dgid.raw[9], my_dest.dgid.raw[10], + my_dest.dgid.raw[11], my_dest.dgid.raw[12], my_dest.dgid.raw[13], + my_dest.dgid.raw[14], my_dest.dgid.raw[15]); + } sockfd = servername ? pp_client_connect(servername, port) : pp_server_connect(port); @@ -533,13 +639,22 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername, return sockfd; } - rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest) : - pp_server_exch_dest(sockfd, &my_dest, rem_dest); + rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest, user_parm) : + pp_server_exch_dest(sockfd, &my_dest, rem_dest, user_parm); if (rc) return rc; printf(addr_fmt, "remote", rem_dest->lid, rem_dest->qpn, rem_dest->psn, rem_dest->rkey, rem_dest->vaddr); + if (user_parm->gid_index > -1) { + printf(" GID: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n", + rem_dest->dgid.raw[0],rem_dest->dgid.raw[1], + rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4], + rem_dest->dgid.raw[5], rem_dest->dgid.raw[6], rem_dest->dgid.raw[7], + rem_dest->dgid.raw[8], rem_dest->dgid.raw[9], rem_dest->dgid.raw[10], + rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13], + rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]); + } if ((rc = pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest,user_parm))) return rc; @@ -548,8 +663,8 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername, * Arbitrarily reuse exch_dest for this purpose. */ - rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest) : - pp_server_exch_dest(sockfd, &my_dest, rem_dest); + rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest, user_parm) : + pp_server_exch_dest(sockfd, &my_dest, rem_dest, user_parm); if (rc) return rc; @@ -582,6 +697,7 @@ static void usage(const char *argv0) printf(" -I, --inline_size=<size> max size of message to be sent in inline mode (default 400)\n"); printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n"); printf(" -S, --sl=<sl> SL (default 0)\n"); + printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n"); printf(" -C, --report-cycles report times in cpu cycle units (default microseconds)\n"); printf(" -H, --report-histogram print out all results (default print summary only)\n"); printf(" -U, --report-unsorted (implies -H) print out unsorted results (default sorted)\n"); @@ -777,6 +893,7 @@ int main(int argc, char *argv[]) user_param.servername = NULL; user_param.inline_size = MAX_INLINE; user_param.qp_timeout = 14; + user_param.gid_index = -1; /*gid will not be used*/ /* Parameter parsing. */ while (1) { int c; @@ -793,6 +910,7 @@ int main(int argc, char *argv[]) { .name = "inline_size", .has_arg = 1, .val = 'I' }, { .name = "qp-timeout", .has_arg = 1, .val = 'u' }, { .name = "sl", .has_arg = 1, .val = 'S' }, + { .name = "gid-index", .has_arg = 1, .val = 'x' }, { .name = "all", .has_arg = 0, .val = 'a' }, { .name = "report-cycles", .has_arg = 0, .val = 'C' }, { .name = "report-histogram",.has_arg = 0, .val = 'H' }, @@ -802,7 +920,7 @@ int main(int argc, char *argv[]) { 0 } }; - c = getopt_long(argc, argv, "p:c:m:d:i:s:n:t:I:u:S:aCHUVF", long_options, NULL);///cpufreq + c = getopt_long(argc, argv, "p:c:m:d:i:s:n:t:I:u:S:x:aCHUVF", long_options, NULL);///cpufreq if (c == -1) break; @@ -896,6 +1014,14 @@ int main(int argc, char *argv[]) if (sl > 15) { usage(argv[0]); return 6; } break; + case 'x': + user_param.gid_index = strtol(optarg, NULL, 0); + if (user_param.gid_index > 63) { + usage(argv[0]); + return 1; + } + break; + default: usage(argv[0]); return 7; @@ -926,6 +1052,9 @@ int main(int argc, char *argv[]) } else { printf("Connection type : UC\n"); } + if (user_param.gid_index > -1) { + printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n"); + } if (user_param.all == ALL) { /*since we run all sizes */ size = 8388608; /*2^23 */ _______________________________________________ ewg mailing list ewg@lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg