No, it doesn't work.
Attaching the applications being used.
"Select" also has similar kind of issue when called from non-main thread
Thread 9 "nstack_select" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fffd77fe700 (LWP 63170)]
0x00007ffff4e1d032 in ldp_select_init_maps (original=0x7fffbc0008c0,
resultb=0x7fffe002e514, libcb=0x7fffe002e544, vclb=0x7fffe002e52c, nfds=34,
minbits=64, n_bytes=5, si_bits=0x7fffd77fdc20,
libc_bits=0x7fffd77fdc28) at
/home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/ldp.c:601
601 clib_bitmap_validate (*vclb, minbits);
(gdb) bt
#0 0x00007ffff4e1d032 in ldp_select_init_maps (original=0x7fffbc0008c0,
resultb=0x7fffe002e514, libcb=0x7fffe002e544, vclb=0x7fffe002e52c, nfds=34,
minbits=64, n_bytes=5, si_bits=0x7fffd77fdc20,
libc_bits=0x7fffd77fdc28) at
/home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/ldp.c:601
#1 0x00007ffff4e1db47 in ldp_pselect (nfds=34, readfds=0x7fffbc0008c0,
writefds=0x7fffbc000cd0, exceptfds=0x7fffbc0010e0, timeout=0x7fffd77fdcb0,
sigmask=0x0)
at
/home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/ldp.c:723
#2 0x00007ffff4e1e5d5 in select (nfds=34, readfds=0x7fffbc0008c0,
writefds=0x7fffbc000cd0, exceptfds=0x7fffbc0010e0, timeout=0x7fffd77fdd20)
at
/home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/ldp.c:857
#3 0x00007ffff7b4c42a in nstack_select_thread (arg=0x0) at
/home/root1/sharath/2019/vpp_ver/19.04/dmm/src/nSocket/nstack/event/select/nstack_select.c:651
#4 0x00007ffff78ed6ba in start_thread (arg=0x7fffd77fe700) at
pthread_create.c:333
#5 0x00007ffff741b41d in clone () at
../sysdeps/unix/sysv/linux/x86_64/clone.S:109
Before https://gerrit.fd.io/r/#/c/18597/ I have tried to fix the issue.
The below changes fixed epoll_wait and epoll_ctl issues for me.[doesn't
include the changes of https://gerrit.fd.io/r/#/c/18597/]
diff --git a/src/vcl/vcl_locked.c b/src/vcl/vcl_locked.c
index fb19b5d..e6c891b 100644
--- a/src/vcl/vcl_locked.c
+++ b/src/vcl/vcl_locked.c
@@ -564,7 +564,10 @@ vls_attr (vls_handle_t vlsh, uint32_t op, void
*buffer, uint32_t * buflen)
if (!(vls = vls_get_w_dlock (vlsh)))
return VPPCOM_EBADFD;
+
+ vls_mt_guard (0, VLS_MT_OP_XPOLL);
rv = vppcom_session_attr (vls_to_sh_tu (vls), op, buffer, buflen);
+ vls_mt_unguard ();
vls_get_and_unlock (vlsh);
return rv;
}
@@ -773,8 +776,10 @@ vls_epoll_ctl (vls_handle_t ep_vlsh, int op,
vls_handle_t vlsh,
vls_table_rlock ();
ep_vls = vls_get_and_lock (ep_vlsh);
vls = vls_get_and_lock (vlsh);
+ vls_mt_guard (0, VLS_MT_OP_XPOLL);
ep_sh = vls_to_sh (ep_vls);
sh = vls_to_sh (vls);
+ vls_mt_unguard ();
if (PREDICT_FALSE (!vlsl->epoll_mp_check))
vls_epoll_ctl_mp_checks (vls, op);
Thanks,
Sharath.
On Fri, Mar 29, 2019 at 9:15 PM Florin Coras <fcoras.li...@gmail.com> wrote:
> Interesting. What application are you running and does this [1] fix the
> issue for you?
>
> In short, many of vls’ apis check if the call is coming in on a new
> pthread and program vcl accordingly if yes. The patch makes sure vls_attr
> does that as well.
>
> Thanks,
> Florin
>
> [1] https://gerrit.fd.io/r/#/c/18597/
>
> On Mar 29, 2019, at 4:29 AM, Dave Barach via Lists.Fd.Io <
> dbarach=cisco....@lists.fd.io> wrote:
>
> For whatever reason, the vls layer received an event notification which
> didn’t end well. vcl_worker_get (wrk_index=4294967295) [aka 0xFFFFFFFF]
> will never work.
>
> I’ll let Florin comment further. He’s in the PDT time zone, so don’t
> expect to hear from him for a few hours.
>
> D.
>
> *From:* vpp-dev@lists.fd.io <vpp-dev@lists.fd.io> *On Behalf Of *sharath
> kumar
> *Sent:* Friday, March 29, 2019 12:18 AM
> *To:* vpp-dev@lists.fd.io; csit-...@lists.fd.io
> *Subject:* [vpp-dev] multi-threaded application, "epoll_wait" and
> "epoll_ctl" have "received signal SIGABRT, Aborted".
>
> Hello all,
>
> I am a newbie to VPP.
>
> I am trying to run VPP with a multi-threaded application.
> "recv" works fine from non-main threads,
> whereas "epoll_wait" and "epoll_ctl" have "received signal SIGABRT,
> Aborted".
>
> Is this a known issue?
> Or am I doing something wrong?
>
> Attaching backtrace for "epoll_wait" and "epoll_ctl"
>
> Thread 9 "dmm_vcl_epoll" received signal SIGABRT, Aborted.
> [Switching to Thread 0x7fffd67fe700 (LWP 56234)]
> 0x00007ffff7349428 in __GI_raise (sig=sig@entry=6) at
> ../sysdeps/unix/sysv/linux/raise.c:54
> 54 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
> (gdb) bt
> #0 0x00007ffff7349428 in __GI_raise (sig=sig@entry=6) at
> ../sysdeps/unix/sysv/linux/raise.c:54
> #1 0x00007ffff734b02a in __GI_abort () at abort.c:89
> #2 0x00007ffff496d873 in os_panic () at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vppinfra/unix-misc.c:176
> #3 0x00007ffff48ce42c in debugger () at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vppinfra/error.c:84
> #4 0x00007ffff48ce864 in _clib_error (how_to_die=2, function_name=0x0,
> line_number=0, fmt=0x7ffff4bfe0e0 "%s:%d (%s) assertion `%s' fails")
> at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vppinfra/error.c:143
> #5 0x00007ffff4bcca7d in vcl_worker_get (wrk_index=4294967295) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/vcl_private.h:540
> #6 0x00007ffff4bccabe in vcl_worker_get_current () at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/vcl_private.h:554
> #7 0x00007ffff4bd7c49 in vppcom_session_attr (session_handle=4278190080,
> op=6, buffer=0x0, buflen=0x0) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/vppcom.c:2606
> #8 0x00007ffff4bfc7fd in vls_attr (vlsh=0, op=6, buffer=0x0, buflen=0x0)
> at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/vcl_locked.c:569
> #9 0x00007ffff4e21736 in ldp_epoll_pwait (epfd=32, events=0x7fffd67fad20,
> maxevents=1024, timeout=100, sigmask=0x0) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/ldp.c:2203
> #10 0x00007ffff4e21948 in epoll_wait (epfd=32, events=0x7fffd67fad20,
> maxevents=1024, timeout=100) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/ldp.c:2257
> #11 0x00007ffff4e13041 in dmm_vcl_epoll_thread (arg=0x0) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/dmm_vcl_adpt.c:75
> #12 0x00007ffff78ed6ba in start_thread (arg=0x7fffd67fe700) at
> pthread_create.c:333
> #13 0x00007ffff741b41d in clone () at
> ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
>
>
>
>
> Thread 11 "vs_epoll" received signal SIGABRT, Aborted.
> 0x00007ffff7349428 in __GI_raise (sig=sig@entry=6) at
> ../sysdeps/unix/sysv/linux/raise.c:54
> 54 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
> (gdb) bt
> #0 0x00007ffff7349428 in __GI_raise (sig=sig@entry=6) at
> ../sysdeps/unix/sysv/linux/raise.c:54
> #1 0x00007ffff734b02a in __GI_abort () at abort.c:89
> #2 0x00007ffff496d873 in os_panic () at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vppinfra/unix-misc.c:176
> #3 0x00007ffff48ce42c in debugger () at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vppinfra/error.c:84
> #4 0x00007ffff48ce864 in _clib_error (how_to_die=2, function_name=0x0,
> line_number=0, fmt=0x7ffff4bfe1a0 "%s:%d (%s) assertion `%s' fails")
> at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vppinfra/error.c:143
> #5 0x00007ffff4bcca7d in vcl_worker_get (wrk_index=4294967295) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/vcl_private.h:540
> #6 0x00007ffff4bccabe in vcl_worker_get_current () at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/vcl_private.h:554
> #7 0x00007ffff4bd597a in vppcom_epoll_ctl (vep_handle=4278190080, op=1,
> session_handle=4278190082, event=0x7fffd4dfb3b0)
> at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/vppcom.c:2152
> #8 0x00007ffff4bfd061 in vls_epoll_ctl (ep_vlsh=0, op=1, vlsh=2,
> event=0x7fffd4dfb3b0) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/vcl_locked.c:787
> #9 0x00007ffff4e213b6 in epoll_ctl (epfd=32, op=1, fd=34,
> event=0x7fffd4dfb3b0) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/ldp.c:2118
> #10 0x00007ffff4e12f88 in vpphs_ep_ctl_ops (epFD=-1, proFD=34, ctl_ops=0,
> events=0x7fffd5190078, pdata=0x7fffd53f01d0)
> at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/stacks/vpp/vpp/src/vcl/dmm_vcl_adpt.c:48
> #11 0x00007ffff7b4d502 in nsep_epctl_triggle (epi=0x7fffd5190018,
> info=0x7fffd53f01d0, triggle_ops=0) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/src/nSocket/nstack/event/epoll/nstack_eventpoll.c:134
> #12 0x00007ffff7b4de31 in nsep_insert_node (ep=0x7fffd50bffa8,
> event=0x7fffd4dfb5a0, fdInfo=0x7fffd53f01d0)
> at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/src/nSocket/nstack/event/epoll/nstack_eventpoll.c:250
> #13 0x00007ffff7b4e480 in nsep_epctl_add (ep=0x7fffd50bffa8, fd=22,
> events=0x7fffd4dfb5a0) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/src/nSocket/nstack/event/epoll/nstack_eventpoll.c:294
> #14 0x00007ffff7b44db0 in nstack_epoll_ctl (epfd=21, op=1, fd=22,
> event=0x7fffd4dfb630) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/src/nSocket/nstack/nstack_socket.c:2499
> #15 0x0000000000401e65 in process_server_msg_thread (pArgv=<optimized
> out>) at
> /home/root1/sharath/2019/vpp_ver/19.04/dmm/app_example/perf-test/multi_tcp_epoll_app_Ser.c:369
> #16 0x00007ffff78ed6ba in start_thread (arg=0x7fffd4dff700) at
> pthread_create.c:333
> #17 0x00007ffff741b41d in clone () at
> ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
>
> Thanks and Regards,
> Sharath.
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
>
> View/Reply Online (#12665): https://lists.fd.io/g/vpp-dev/message/12665
> Mute This Topic: https://lists.fd.io/mt/30819724/675152
> Group Owner: vpp-dev+ow...@lists.fd.io
> Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [fcoras.li...@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>
>
>
/*
*
* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _GNU_SOURCE
#include <sys/socket.h>
#include <fcntl.h>
#include <memory.h>
#include <signal.h>
#include <stdio.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <netinet/in.h>
#include <time.h>
#include <linux/tcp.h>
#include <sched.h>
#include <pthread.h>
#include <arpa/inet.h>
#include <sys/epoll.h>
#include <errno.h>
#define _BIND bind
#define _LISTEN listen
#define _SOCKET socket
#define _ACCEPT accept
#define _SEND send
#define _RECV recv
#define _CLOSE close
#define _CONNECT connect
#define _PROTOCOL IPPROTO_TCP
#define MAX_TEST_TIME 1000
#define MSG_LENGTH 256
#define CORE_NUM 8
#define START_CPU_ID 2
#define MAX_CONN_LIMIT 256
#define MAX_PORT_NUM 65535
//1:A-B ;0:A-B-A
#define SEND_RECV_MODE 1
#define MAX_EVENTS 1000
#define Default_PortID 12345
#define Default_SleepCnt 10000
#define Default_SleepTime 1000000
#define Flag_Print 1
#define Fd_Number 1
static struct sockaddr_in g_dest;
static struct sockaddr_in g_src;
static struct sockaddr_in g_recv;
int times = MAX_TEST_TIME;
int msg_length = MSG_LENGTH;
int coreNum = CORE_NUM;
int startCPUId = START_CPU_ID;
int connectNum = MAX_CONN_LIMIT;
int msgMode = SEND_RECV_MODE;
int sleepCnt = Default_SleepCnt;
int sleepTime = Default_SleepTime; //us
int fdNumber = Fd_Number;
int flagPrint = Flag_Print;
int unitPrint = 0;
int waitTime = 0;
int srcPort = Default_PortID;
int destPort = 0;
int recvPort = 0;
char sendarr[256] = "";
char recvarr[256] = "";
char destarr[256] = "";
static void
setArgsDefault ()
{
memset (&g_dest, 0, sizeof (g_dest));
g_dest.sin_family = AF_INET;
g_dest.sin_addr.s_addr = inet_addr ("127.0.0.1");
g_dest.sin_port = htons (12345);
bzero (&(g_dest.sin_zero), 8);
memset (&g_src, 0, sizeof (g_src));
g_src.sin_family = AF_INET;
g_src.sin_addr.s_addr = inet_addr ("0.0.0.0");
g_src.sin_port = htons (7895);
bzero (&(g_src.sin_zero), 8);
memset (&g_recv, 0, sizeof (g_recv));
g_recv.sin_family = AF_INET;
g_recv.sin_addr.s_addr = inet_addr ("0.0.0.0");
g_recv.sin_port = htons (7895);
bzero (&(g_recv.sin_zero), 8);
}
static int
process_arg (int argc, char *argv[])
{
int opt = 0;
int error = 0;
const char *optstring = "p:d:s:a:l:t:e:i:f:r:n:w:u:x:";
int rw_mark = 0;
if (argc < 4)
{
printf
("Param info :-p dest_portid; -d dest_serverIP; -a src_portid; -s src_clientIP; -l msg_length; \n");
printf
("\t-t MAX_TEST_TIME; -i msg_interval ; -f client fd number ; -r receive port; -n connectNum(one server vs n client)\n");
return 0;
}
while ((opt = getopt (argc, argv, optstring)) != -1)
{
switch (opt)
{
case 'p':
destPort = atoi (optarg);
g_dest.sin_port = htons (atoi (optarg));
break;
case 'd':
stpcpy (destarr, optarg);
g_dest.sin_addr.s_addr = inet_addr (optarg);
break;
case 's':
stpcpy (sendarr, optarg);
g_src.sin_addr.s_addr = inet_addr (optarg);
g_recv.sin_addr.s_addr = inet_addr (optarg);
break;
case 'a':
//g_src.sin_port = htons(atoi(optarg));
srcPort = atoi (optarg);
break;
case 'l':
msg_length = atoi (optarg);
break;
case 't':
times = atoi (optarg);
break;
case 'e':
sleepCnt = atoi (optarg);
break;
case 'i':
sleepTime = atoi (optarg);
break;
case 'f':
fdNumber = atoi (optarg);
break;
case 'r':
recvPort = atoi (optarg);
g_recv.sin_port = htons (atoi (optarg));
break;
case 'n':
connectNum = atoi (optarg);
break;
case 'w':
waitTime = atoi (optarg);
break;
case 'u':
unitPrint = atoi (optarg);
break;
case 'x':
flagPrint = atoi (optarg);
break;
}
}
return 1;
}
void
process_client (void)
{
int sendLen = 0;
int i = 0, t = 0, p = 0, optval = 0, ret = 0;
char send_buf[1000] = "";
int c_socketfd[100] = { 0 };
int errbind[1000] = { 0 };
int errconn[1000] = { 0 };
int send_count[1000] = { 0 };
int pps = 0;
long pps_time = 0;
struct timespec startTime, endTime;
struct timespec countStart;
struct timespec countEnd;
memset (&countStart, 0, sizeof (countStart));
memset (&countEnd, 0, sizeof (countEnd));
for (i = 0; i < fdNumber; i++)
{
c_socketfd[i] = _SOCKET (PF_INET, SOCK_STREAM, _PROTOCOL);
if (0 > c_socketfd[i])
{
printf ("client %d failed,err %d\n", i, errno);
}
else
{
printf ("client %d created success\n", i);
}
}
for (i = 0; i < fdNumber; i++)
{
optval = 1;
ret =
setsockopt (c_socketfd[i], SOL_SOCKET, SO_REUSEADDR, (void *) &optval,
sizeof (optval));
if (ret == -1)
{
printf ("Couldn't setsockopt(SO_REUSEADDR)\n");
break;
}
}
for (i = 0; i < fdNumber; i++)
{
g_src.sin_port = htons (srcPort + i);
errbind[i] =
_BIND (c_socketfd[i], (struct sockaddr *) &g_src, sizeof (g_src));
if (errbind[i] < 0)
{
printf ("client %d bind Failed %d\n", i, errno);
_CLOSE (c_socketfd[i]);
c_socketfd[i] = -1;
continue;
}
else
{
printf ("client %d bind Success port:%d IP:%s\n", i,
ntohs (g_src.sin_port),
inet_ntoa (*((struct in_addr *) &(g_src.sin_addr.s_addr))));
}
}
for (i = 0; i < fdNumber; i++)
{
if (errbind[i] >= 0)
{
errconn[i] =
_CONNECT (c_socketfd[i], (struct sockaddr *) &g_dest,
sizeof (g_dest));
if (errconn[i] < 0)
{
printf ("client %d Connect Failed %d\n", i, errno);
_CLOSE (c_socketfd[i]);
c_socketfd[i] = -1;
continue;
}
else
{
printf ("client %d Connect Success port:%d, IP:%s\n", i,
ntohs (g_dest.sin_port),
inet_ntoa (*
((struct in_addr *)
&(g_dest.sin_addr.s_addr))));
}
}
}
sleep (1);
clock_gettime (CLOCK_MONOTONIC, &startTime);
clock_gettime (CLOCK_MONOTONIC, &countStart);
for (t = 0; t < times; t++)
{
for (i = 0; i < fdNumber; i++)
{
if (c_socketfd[i] < 0)
{
continue;
}
do
{
sendLen = _SEND (c_socketfd[i], send_buf, msg_length, 0);
}
while (sendLen <= 0);
send_count[i]++;
}
if (0 != sleepTime)
{
if ((t % sleepCnt) == 0)
{
usleep (sleepTime);
}
}
if ((send_count[0] % unitPrint) == 0)
{
clock_gettime (CLOCK_MONOTONIC, &countEnd);
pps_time =
(countEnd.tv_sec - countStart.tv_sec) * 1000000000 +
countEnd.tv_nsec - countStart.tv_nsec;
pps = ((float) 1000000000 / pps_time) * unitPrint * fdNumber;
if ((flagPrint != 0))
{
printf (" sendcount %d, time: %ld ns\n",
send_count[0] * fdNumber, pps_time);
}
printf ("sendcount %d , pps=%d\n", send_count[0] * fdNumber, pps);
clock_gettime (CLOCK_MONOTONIC, &countStart);
}
}
clock_gettime (CLOCK_MONOTONIC, &endTime);
for (i = 0; i < fdNumber; i++)
{
printf ("client %d send %d , sendtime :%ld s %ld ns\n", i,
send_count[i], endTime.tv_sec - startTime.tv_sec,
endTime.tv_nsec - startTime.tv_nsec);
}
for (i = 0; i < fdNumber; i++)
{
printf ("client %d close!\n", i);
if (c_socketfd[i] > 0)
_CLOSE (c_socketfd[i]);
}
pthread_exit (NULL);
}
/*
using this thread to do recv msg;
*/
void *
process_server_msg_thread (void *pArgv)
{
int recvLen = 0, recvLen2 = 0;
char send_buf[1000];
char recv_buf[1000];
int recv_count = 0;
long recv_ppstime = 0;
int recv_pps = 0;
int ret;
struct epoll_event event;
struct epoll_event eventList[MAX_EVENTS];
struct timespec recvStart;
struct timespec recvEnd;
memset (&recvStart, 0, sizeof (recvStart));
memset (&recvEnd, 0, sizeof (recvEnd));
pthread_detach (pthread_self ());
int msgEpFd = epoll_create (100);
int msgFd = *(int *) pArgv;
event.events = EPOLLIN | EPOLLET;
event.data.fd = msgFd;
if (epoll_ctl (msgEpFd, EPOLL_CTL_ADD, msgFd, &event) < 0)
{
printf ("epoll add fd[%d] to msgEpfd:[%d] failed\n", msgFd, msgEpFd);
close (msgEpFd);
close (msgFd);
return NULL;
}
clock_gettime (CLOCK_MONOTONIC, &recvStart);
while (1)
{
int timeout = -1;
int m;
int sock;
ret = epoll_wait (msgEpFd, eventList, MAX_EVENTS, timeout);
if (ret == 0)
continue;
for (m = 0; m < ret; m++)
{
if ((eventList[m].events & EPOLLERR)
|| (eventList[m].events & EPOLLHUP)
|| !(eventList[m].events & EPOLLIN))
{
printf ("fd %d epoll error event:%d\n", eventList[m].data.fd,
eventList[m].events);
close (eventList[m].data.fd);
continue;
}
else
{
recvLen = 0;
recvLen2 = 0;
sock = eventList[m].data.fd;
do
{
recvLen2 =
recv (sock, recv_buf + recvLen, msg_length - recvLen, 0);
if (recvLen2 <= 0)
{
break;
}
else if (recvLen2 == msg_length - recvLen)
{
recvLen = 0;
recv_count++;
if (msg_length != send (sock, send_buf, msg_length, 0))
{
printf ("send failed!====, need exit\n");
}
}
else if (recvLen2 < msg_length - recvLen)
{
recvLen += recvLen2;
}
if ((flagPrint != 0) && ((recv_count % unitPrint) == 0))
{
clock_gettime (CLOCK_MONOTONIC, &recvEnd);
recv_ppstime =
(recvEnd.tv_sec - recvStart.tv_sec) * 1000000000 +
recvEnd.tv_nsec - recvStart.tv_nsec;
recv_pps =
((float) 1000000000 / recv_ppstime) * unitPrint;
printf ("receive count:%d, receive time: %ld ns\n",
recv_count, recv_ppstime);
printf ("receive pps = %d\n", recv_pps);
clock_gettime (CLOCK_MONOTONIC, &recvStart);
}
}
while (1);
}
}
if (recv_count == times)
{
break;
}
}
close (msgFd);
close (msgEpFd);
}
/*
using this thread to do accept connect
*/
void *
process_server_accept_thread (void *pArgv)
{
int listenFd = 0;
int x, optval, ret = 0;
int acpt_socketfd[1000] = { 0 };
listenFd = socket (PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (0 > listenFd)
{
printf ("ERROR:socket failed,errno [%d]\n", errno);
return NULL;
}
else
{
printf ("INFO:Create listen socket Success, listenFd[%d]\n", listenFd);
}
if (0 > fcntl (listenFd, F_SETFL, O_NONBLOCK))
{
printf ("ERROR:fcntl failed. fd[%d], errno[%d]/n", listenFd, errno);
close (listenFd);
}
optval = 1;
ret =
setsockopt (listenFd, SOL_SOCKET, SO_REUSEADDR, (void *) &optval,
sizeof (optval));
if (ret == -1)
{
printf ("ERROR:setsockopt failed. fd[%d], errno[%d]\n", listenFd,
errno);
close (listenFd);
return NULL;
}
if (0 !=
bind (listenFd, (struct sockaddr *) &g_recv, sizeof (struct sockaddr)))
{
printf ("ERROR:bind failed. fd[%d] errno [%d]\n", listenFd, errno);
printf ("INFO:Bind Failed, port %d IP:%s\n", ntohs (g_recv.sin_port),
inet_ntoa (*((struct in_addr *) &(g_recv.sin_addr.s_addr))));
close (listenFd);
//return NULL;
exit (-1);
}
else
{
printf ("INFO:Bind Success, port %d IP:%s\n", ntohs (g_recv.sin_port),
inet_ntoa (*((struct in_addr *) &(g_recv.sin_addr.s_addr))));
}
if (0 != listen (listenFd, 100))
{
printf ("server socket listen failed. err %d\n", errno);
close (listenFd);
return NULL;
}
printf ("Listen Success\n");
int accEpFd;
struct epoll_event eventList[100];
accEpFd = epoll_create (100);
struct epoll_event event;
event.events = EPOLLIN | EPOLLET;
event.data.fd = listenFd;
if (epoll_ctl (accEpFd, EPOLL_CTL_ADD, listenFd, &event) < 0)
{
printf ("epoll_ctl add [%d] to epfd:%d failed\n", listenFd, accEpFd);
exit (-1);
}
int timeout = -1;
int accpedNum = 0;
while (accpedNum < connectNum)
{
ret = epoll_wait (accEpFd, eventList, connectNum, timeout);
if (ret < 0)
{
printf ("accept_thread epoll_wait error, epfd[%d], errno[%d]\n",
accEpFd, errno);
continue;
}
else if (ret == 0)
continue;
/*loop done ever Event */
for (x = 0; x < ret; x++)
{
printf ("event:%d; ret:%d\n", eventList[x].events, ret);
if ((eventList[x].events & EPOLLERR)
|| !(eventList[x].events & EPOLLIN))
{
printf ("epoll fd %d error\n", eventList[x].data.fd);
close (eventList[x].data.fd);
continue;
}
while (1)
{
acpt_socketfd[accpedNum] =
accept4 (listenFd, NULL, NULL, SOCK_NONBLOCK);
if (acpt_socketfd[accpedNum] < 0)
{
printf ("no more connect\n");
break;
}
/*add new accptFd to MsgEpFD */
pthread_t ser_rcv_thread_id;
if (pthread_create
(&ser_rcv_thread_id, NULL, process_server_msg_thread,
(void *) &acpt_socketfd[accpedNum]) == -1)
{
printf ("create process_server_msg_thread fail\n");
break;
}
printf ("accept cnt [%d], cur accept fd [%d]\n", accpedNum,
acpt_socketfd[accpedNum]);
accpedNum++;
}
}
}
//close (listenFd);
close (accEpFd);
while (1)
{
sleep (10);
}
pthread_exit (NULL);
}
void
main (int argc, char *argv[])
{
socklen_t addrlen = sizeof (struct sockaddr);
int err = 0, result = 0, ret = 0;
int i = 0, j = 0, optval = 0, z = 0, x, listenfd;
cpu_set_t mask;
setArgsDefault ();
ret = process_arg (argc, argv);
if (ret != 1)
{
printf ("The param error\n");
return;
}
pthread_t server_thread_id;
if (pthread_create
(&server_thread_id, NULL, process_server_accept_thread, NULL) == -1)
{
printf ("create process_server_accept_thread fail\n");
return;
}
printf ("create process_server_accept_thread success\n");
if (server_thread_id != 0)
{
printf ("Server Thread join\n");
pthread_join (server_thread_id, NULL);
}
while (1)
{
sleep (10);
}
return;
}
/*
*
* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _GNU_SOURCE
#include <sys/socket.h>
#include <fcntl.h>
#include <memory.h>
#include <signal.h>
#include <stdio.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <netinet/in.h>
#include <time.h>
#include <linux/tcp.h>
#include <sched.h>
#include <pthread.h>
#include <arpa/inet.h>
#include <sys/epoll.h>
#include <errno.h>
#define _BIND bind
#define _LISTEN listen
#define _SOCKET socket
#define _ACCEPT accept
#define _SEND send
#define _RECV recv
#define _CLOSE close
#define _CONNECT connect
#define _PROTOCOL IPPROTO_TCP
#define MAX_TEST_TIME 1000
#define MSG_LENGTH 256
#define CORE_NUM 8
#define START_CPU_ID 2
#define MAX_CONN_LIMIT 256
#define MAX_PORT_NUM 65535
//1:A-B ;0:A-B-A
#define SEND_RECV_MODE 1
#define MAX_EVENTS 1000
#define Default_PortID 12345
#define Default_SleepCnt 10000
#define Default_SleepTime 1000000
#define Flag_Print 1
#define Fd_Number 1
static struct sockaddr_in g_dest;
static struct sockaddr_in g_src;
static struct sockaddr_in g_recv;
int times = MAX_TEST_TIME;
int msg_length = MSG_LENGTH;
int coreNum = CORE_NUM;
int startCPUId = START_CPU_ID;
int connectNum = MAX_CONN_LIMIT;
int msgMode = SEND_RECV_MODE;
int sleepCnt = Default_SleepCnt;
int sleepTime = Default_SleepTime; //us
int fdNumber = Fd_Number;
int flagPrint = Flag_Print;
int unitPrint = 0;
int waitTime = 0;
int srcPort = Default_PortID;
int destPort = 0;
int recvPort = 0;
char sendarr[256] = "";
char recvarr[256] = "";
char destarr[256] = "";
static void
setArgsDefault ()
{
memset (&g_dest, 0, sizeof (g_dest));
g_dest.sin_family = AF_INET;
g_dest.sin_addr.s_addr = inet_addr ("127.0.0.1");
g_dest.sin_port = htons (12345);
bzero (&(g_dest.sin_zero), 8);
memset (&g_src, 0, sizeof (g_src));
g_src.sin_family = AF_INET;
g_src.sin_addr.s_addr = inet_addr ("0.0.0.0");
g_src.sin_port = htons (7895);
bzero (&(g_src.sin_zero), 8);
memset (&g_recv, 0, sizeof (g_recv));
g_recv.sin_family = AF_INET;
g_recv.sin_addr.s_addr = inet_addr ("0.0.0.0");
g_recv.sin_port = htons (7895);
bzero (&(g_recv.sin_zero), 8);
}
static int
process_arg (int argc, char *argv[])
{
int opt = 0;
int error = 0;
const char *optstring = "p:d:s:a:l:t:e:i:f:r:n:w:u:x:";
int rw_mark = 0;
if (argc < 4)
{
printf
("Param info :-p dest_portid; -d dest_serverIP; -a src_portid; -s src_clientIP; -l msg_length; \n");
printf
("\t-t MAX_TEST_TIME; -i msg_interval ; -f client fd number ; -r receive port; -n connectNum(one server vs n client)\n");
return 0;
}
while ((opt = getopt (argc, argv, optstring)) != -1)
{
switch (opt)
{
case 'p':
destPort = atoi (optarg);
g_dest.sin_port = htons (atoi (optarg));
break;
case 'd':
stpcpy (destarr, optarg);
g_dest.sin_addr.s_addr = inet_addr (optarg);
break;
case 's':
stpcpy (sendarr, optarg);
g_src.sin_addr.s_addr = inet_addr (optarg);
g_recv.sin_addr.s_addr = inet_addr (optarg);
break;
case 'a':
//g_src.sin_port = htons(atoi(optarg));
srcPort = atoi (optarg);
break;
case 'l':
msg_length = atoi (optarg);
break;
case 't':
times = atoi (optarg);
break;
case 'e':
sleepCnt = atoi (optarg);
break;
case 'i':
sleepTime = atoi (optarg);
break;
case 'f':
fdNumber = atoi (optarg);
break;
case 'r':
recvPort = atoi (optarg);
g_recv.sin_port = htons (atoi (optarg));
break;
case 'n':
connectNum = atoi (optarg);
break;
case 'w':
waitTime = atoi (optarg);
break;
case 'u':
unitPrint = atoi (optarg);
break;
case 'x':
flagPrint = atoi (optarg);
break;
}
}
return 1;
}
/*
using this thread to do recv msg;
*/
void *
process_server_msg_thread (void *pArgv)
{
int recvLen = 0, recvLen2 = 0;
char send_buf[1000];
char recv_buf[1000];
int recv_count = 0;
long recv_ppstime = 0;
int recv_pps = 0;
int ret;
struct timeval timeout;
fd_set rfds;
int maxfd;
struct timespec recvStart;
struct timespec recvEnd;
memset (&recvStart, 0, sizeof (recvStart));
memset (&recvEnd, 0, sizeof (recvEnd));
pthread_detach (pthread_self ());
int msgFd = *(int *) pArgv;
clock_gettime (CLOCK_MONOTONIC, &recvStart);
while (1)
{
FD_ZERO (&rfds);
FD_SET (msgFd, &rfds);
maxfd = msgFd + 1;
timeout.tv_sec = 10;
timeout.tv_usec = 0;
ret = select (maxfd, &rfds, NULL, NULL, &timeout);
if (ret <= 0)
{
continue;
}
if (FD_ISSET (msgFd, &rfds) == 0)
{
continue;
}
recvLen = 0;
recvLen2 = 0;
do
{
recvLen2 =
recv (msgFd, recv_buf + recvLen, msg_length - recvLen, 0);
if (recvLen2 <= 0)
{
break;
}
else if (recvLen2 == msg_length - recvLen)
{
recvLen = 0;
recv_count++;
if (msg_length != send (msgFd, send_buf, msg_length, 0))
{
printf ("send failed!====, need exit\n");
}
}
else if (recvLen2 < msg_length - recvLen)
{
recvLen += recvLen2;
}
if ((flagPrint != 0) && ((recv_count % unitPrint) == 0))
{
clock_gettime (CLOCK_MONOTONIC, &recvEnd);
recv_ppstime =
(recvEnd.tv_sec - recvStart.tv_sec) * 1000000000 +
recvEnd.tv_nsec - recvStart.tv_nsec;
recv_pps = ((float) 1000000000 / recv_ppstime) * unitPrint;
printf ("receive count:%d, receive time: %ld ns\n",
recv_count, recv_ppstime);
printf ("receive pps = %d\n", recv_pps);
clock_gettime (CLOCK_MONOTONIC, &recvStart);
}
}
while (1);
if (recv_count == times)
{
break;
}
}
close (msgFd);
}
/*
using this thread to do accept connect
*/
void *
process_server_accept_thread (void *pArgv)
{
int listenFd = 0;
int x, optval, ret, m = 0;
int acpt_socketfd[1000] = { 0 };
struct timeval timeout;
fd_set rfds;
int maxfd;
listenFd = socket (PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (0 > listenFd)
{
printf ("ERROR:socket failed,errno [%d]\n", errno);
return NULL;
}
else
{
printf ("INFO:Create listen socket Success, listenFd[%d]\n", listenFd);
}
if (0 > fcntl (listenFd, F_SETFL, O_NONBLOCK))
{
printf ("ERROR:fcntl failed. fd[%d], errno[%d]/n", listenFd, errno);
close (listenFd);
}
optval = 1;
ret =
setsockopt (listenFd, SOL_SOCKET, SO_REUSEADDR, (void *) &optval,
sizeof (optval));
if (ret == -1)
{
printf ("ERROR:setsockopt failed. fd[%d], errno[%d]\n", listenFd,
errno);
close (listenFd);
return NULL;
}
if (0 !=
bind (listenFd, (struct sockaddr *) &g_recv, sizeof (struct sockaddr)))
{
printf ("ERROR:bind failed. fd[%d] errno [%d]\n", listenFd, errno);
printf ("INFO:Bind Failed, port %d IP:%s\n", ntohs (g_recv.sin_port),
inet_ntoa (*((struct in_addr *) &(g_recv.sin_addr.s_addr))));
close (listenFd);
//return NULL;
exit (-1);
}
else
{
printf ("INFO:Bind Success, port %d IP:%s\n", ntohs (g_recv.sin_port),
inet_ntoa (*((struct in_addr *) &(g_recv.sin_addr.s_addr))));
}
if (0 != listen (listenFd, 100))
{
printf ("server socket listen failed. err %d\n", errno);
close (listenFd);
return NULL;
}
printf ("Listen Success\n");
int accpedNum = 0;
while (accpedNum < connectNum)
{
FD_ZERO (&rfds);
FD_SET (listenFd, &rfds);
maxfd = listenFd + 1;
timeout.tv_sec = 10;
timeout.tv_usec = 0;
ret = select (maxfd, &rfds, NULL, NULL, &timeout);
if (ret <= 0)
{
continue;
}
if (FD_ISSET (listenFd, &rfds) == 0)
{
continue;
}
while (1)
{
acpt_socketfd[accpedNum] =
accept4 (listenFd, NULL, NULL, SOCK_NONBLOCK);
if (acpt_socketfd[accpedNum] < 0)
{
printf ("no more connect\n");
break;
}
/*add new accptFd to MsgEpFD */
pthread_t ser_rcv_thread_id;
if (pthread_create
(&ser_rcv_thread_id, NULL, process_server_msg_thread,
(void *) &acpt_socketfd[accpedNum]) == -1)
{
printf ("create process_server_msg_thread fail\n");
break;
}
printf ("accept cnt [%d], cur accept fd [%d]\n", accpedNum,
acpt_socketfd[accpedNum]);
accpedNum++;
}
}
close (listenFd);
while (1)
{
sleep (10);
}
pthread_exit (NULL);
}
void
main (int argc, char *argv[])
{
socklen_t addrlen = sizeof (struct sockaddr);
int err = 0, result = 0, ret = 0;
int i = 0, j = 0, optval = 0, z = 0, x, listenfd;
cpu_set_t mask;
setArgsDefault ();
ret = process_arg (argc, argv);
if (ret != 1)
{
printf ("The param error\n");
return;
}
pthread_t server_thread_id;
if (pthread_create
(&server_thread_id, NULL, process_server_accept_thread, NULL) == -1)
{
printf ("create process_server_accept_thread fail\n");
return;
}
printf ("create process_server_accept_thread success\n");
if (server_thread_id != 0)
{
printf ("Server Thread join\n");
pthread_join (server_thread_id, NULL);
}
while (1)
{
sleep (10);
}
return;
}
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#12669): https://lists.fd.io/g/vpp-dev/message/12669
Mute This Topic: https://lists.fd.io/mt/30819724/21656
Group Owner: vpp-dev+ow...@lists.fd.io
Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
