Hi Paul,
This list isn't really likely to give you free consulting work. My brief
skimming suggested the following tips as possibly relevant:
- libuv is an async library first-and-foremost. You must adapt your mental
model to use callback-managed state machines (inverted control flow).
Learning Javascript can be helpful here for the mental model aspects.
- libuv is not thread-safe. There's some specific operations defined for
concurrent program integration, but you're responsible for the details.
- uv_async_t is not a queue, it is a condition object. You are responsible
for providing your own queue.
-jameson
On Tue, Dec 22, 2020 at 11:56 AM pa...@rcom-software.com <
pa...@rcom-software.com> wrote:
> Hi Folks:
>
> This is what happens at a lower level. Do you have insight about the cause
> ?
> The relevant code segments and the GDB stack traces follow below.
>
> The main() process and IO_Task() are executing concurrently.
>
> NOTE: The IO_Trigger_Task() has been eliminated and the uv_start_poll()
> call is invoked in the IO_Task()
> by a proxy routine and an async. wakeup in the main() process. The wakeup
> is done in the make_incoming_connection()
> callback routine by the following code segment.
>
> poll_handle->data = (void *) cdesc;
> //
> // Start epoll() monitoring of the connection. The poll_start_proxy()
> // routine executes in the IO_Task().
> //
> uv_async_init(&Poll_Loop, &cdesc->async_handle, poll_start_proxy);
> cdesc->async_handle.data = (void *) cdesc;
> uv_async_send(&cdesc->async_handle);
>
>
> * The main() process is executing uv__finish_close() which is invoked via
> uv_run() and uv__run_closing_handles().
> It crashes in the call to uv__finish_close()
>
> This is the code from unix/core.c: Line 210 is the first assert()
> statement.
>
> ------------------------------------------------------------------------------
> static void uv__finish_close(uv_handle_t* handle) {
> /* Note: while the handle is in the UV_CLOSING state now, it's still
> possible
> * for it to be active in the sense that uv__is_active() returns true.
> * A good example is when the user calls uv_shutdown(), immediately
> followed
> * by uv_close(). The handle is considered active at this point because
> the
> * completion of the shutdown req is still pending.
> */
> #ifndef SUPRESS
> assert(handle->flags & UV_CLOSING);
> assert(!(handle->flags & UV_CLOSED));
> #endif /* SUPRESS */
> handle->flags |= UV_CLOSED;
>
> * The IO_Task() thread is executing uv_poll_start() which is called from
> start_poll_proxy() in
> response to an async. request from the main() process.
> (uv__async_event() makes the call to
> start_poll_proxy()).
>
> This is the code that executes in the IO_Task() from network_io.c
> -----------------------------------------------------------------
> //
> // Executes in IO_Task
> //
> ROUTINE void poll_start_proxy(uv_async_t *async_handle)
> {
> int r;
>
> CONN_DESC *cdesc = (CONN_DESC *) async_handle->data;
>
> uv_poll_init(&Poll_Loop, cdesc->poll_handle, cdesc->fd);
> if((r = uv_poll_start(cdesc->poll_handle, UV_READABLE, poll_callback))
> < 0)
> {
> fprintf(stderr, "PROXY: IO_TASK - Polling Initiation Error %d:
> %s\n", r, uv_err_name(r));
> abort();
> }
> uv_close( (uv_handle_t *) async_handle, NULL);
>
> return;
> }
>
> This is the code from unix/poll.c: Line 92 is the call to uv__poll_stop()
> --------------------------------------------------------------------------
> int uv_poll_start(uv_poll_t* handle, int pevents, uv_poll_cb poll_cb) {
> int events;
>
> assert((pevents & ~(UV_READABLE | UV_WRITABLE)) == 0);
> assert(!(handle->flags & (UV_CLOSING | UV_CLOSED)));
>
> uv__poll_stop(handle);
>
> if (pevents == 0)
> return 0;
>
> events = 0;
> if (pevents & UV_READABLE)
> events |= UV__POLLIN;
> if (pevents & UV_WRITABLE)
> events |= UV__POLLOUT;
>
> uv__io_start(handle->loop, &handle->io_watcher, events);
> uv__handle_start(handle);
> handle->poll_cb = poll_cb;
>
> return 0;
> }
>
>
>
> main() Stack Trace
> --------------------
> #0 0x00007ffff751e267 in __GI_raise (sig=sig@entry=6) at
> ../sysdeps/unix/sysv/linux/raise.c:55
> #1 0x00007ffff751feca in __GI_abort () at abort.c:89
> #2 0x00007ffff751703d in __assert_fail_base (fmt=0x7ffff7679028
> "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n",
> assertion=assertion@entry=0x41e678 "handle->flags & UV_CLOSING",
> file=file@entry=0x41e668 "src/unix/core.c",
> line=line@entry=210, function=function@entry=0x41e8b0
> <__PRETTY_FUNCTION__.9522> "uv__finish_close")
> at assert.c:92
> #3 0x00007ffff75170f2 in __GI___assert_fail
> (assertion=assertion@entry=0x41e678
> "handle->flags & UV_CLOSING",
> file=file@entry=0x41e668 "src/unix/core.c", line=line@entry=210,
> function=function@entry=0x41e8b0 <__PRETTY_FUNCTION__.9522>
> "uv__finish_close") at assert.c:101
> #4 0x000000000040cd60 in uv__finish_close (handle=<optimized out>) at
> src/unix/core.c:210
> #5 uv__run_closing_handles (loop=0x830680 <Connect_Loop>) at
> src/unix/core.c:261
> #6 uv_run (loop=0x830680 <Connect_Loop>, mode=UV_RUN_DEFAULT) at
> src/unix/core.c:328
> #7 0x0000000000403e0f in main () at main.c:184
>
> IO_Task() Stack Trace
> -----------------------
> #0 uv__poll_stop (handle=0x831bc0) at src/unix/poll.c:73
> #1 0x000000000040e93f in uv_poll_start (handle=0x831bc0, pevents=1,
> poll_cb=0x404576 <poll_callback>)
> at src/unix/poll.c:92
> #2 0x00000000004047d4 in poll_start_proxy (async_handle=0x639468
> <Conn_Desc_Table+40>) at network_io.c:146
> #3 0x000000000040c1dd in uv__async_event (loop=0x639000 <Poll_Loop>,
> w=<optimized out>, nevents=<optimized out>)
> at src/unix/async.c:89
> #4 0x000000000040c29e in uv__async_io (loop=0x639000 <Poll_Loop>,
> w=0x6391c8 <Poll_Loop+456>,
> events=<optimized out>) at src/unix/async.c:160
> #5 0x000000000041590d in uv__io_poll (loop=loop@entry=0x639000
> <Poll_Loop>, timeout=-1) at src/unix/linux-core.c:319
> #6 0x000000000040cac7 in uv_run (loop=0x639000 <Poll_Loop>,
> mode=UV_RUN_DEFAULT) at src/unix/core.c:326
> #7 0x000000000040484e in IO_Task (arg=0x0) at network_io.c:171
> #8 0x0000000000412eb7 in uv__thread_start (arg=<optimized out>) at
> src/unix/thread.c:49
> #9 0x00007ffff7bc26aa in start_thread (arg=0x7ffff6ce7700) at
> pthread_create.c:333
> #10 0x00007ffff75efeed in clone () at
> ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
>
>
> Best Regards,
>
> Paul R.
>
> On Monday, December 21, 2020 at 5:39:09 PM UTC-8 pa...@rcom-software.com
> wrote:
>
>> Hi Folks:
>>
>> I think I see the heart of my problem. Everything appears to work
>> correctly
>> when you establish an incoming TCP connection and release it and the
>> related
>> Libuv handles--the uv_tcp_t connection handle and the uv_poll_t poll
>> handle.
>> (I revised the release code to do things the right way.)
>>
>> The comments about coalescing of uv_async_send() calls in the
>> documentation is somewhat misleading.
>> They should indicate that call with the same handle are synchronous.
>> Also, I suspect that
>> uv_async_send() is not reentrant.
>>
>> When you attempt another incoming connection the following things occur.
>>
>> Notice in 2.2, below that uv_start_loop() executes without being called.
>> This
>> doesn't make sense to me--at least on the surface. Can you think of a
>> possible
>> reason this occurs ?
>>
>> 1) The connection is successfully established--with uv_accept(), and the
>> socket descriptor fd is the same as was used in the previous connection,
>> in the main() process. (Occurs with uv_loop_t Session_Loop.)
>>
>> conn_handle = (uv_tcp_t *) malloc(sizeof(uv_tcp_t));
>> if(conn_handle == NULL)
>> {
>> fprintf(stderr, "MAIN: No Connect Handle Memory\n");
>> abort();
>> }
>>
>> uv_tcp_init(&Connect_Loop, conn_handle);
>> if(uv_accept(listen_handle, (uv_stream_t *) conn_handle) == 0)
>> {
>> uv_os_fd_t fd;
>>
>> uv_fileno((const uv_handle_t*) conn_handle, &fd);
>> }
>>
>> 2.1) A poll handle is successfully allocated in the IO_Trigger_Task()
>> thread.
>> (No loop involved.)
>>
>> poll_handle = (uv_poll_t *) malloc(sizeof(uv_poll_t));
>> if(poll_handle == NULL)
>> {
>> fprintf(stderr, "IO_TRIGGER_TASK: No Poll HAndle Memory\n");
>> abort();
>> }
>>
>> uv_poll_init(&Poll_Loop, poll_handle, pm->info);
>> if((r = uv_poll_start(poll_handle, UV_READABLE, poll_callback)) <
>> 0)
>> {
>> fprintf(stderr, "IO_TRIGGER_TASK: Polling Initiation Error
>> %d: %s\n", r, uv_err_name(r));
>> abort();
>> }
>>
>> 2.2) uv_poll_start() is invoked via a call.
>>
>> uv_poll_init(&Poll_Loop, poll_handle, pm->info);
>> if((r = uv_poll_start(poll_handle, UV_READABLE, poll_callback)) <
>> 0)
>> {
>> fprintf(stderr, "IO_TRIGGER_TASK: Polling Initiation Error
>> %d: %s\n", r, uv_err_name(r));
>> abort();
>> }
>>
>> 2.3) uv_poll_start() executes again without being called !
>>
>> This is what you see in GDB which is very strange since I know there is
>> only
>> one instance of the IO_Trigger_Task() running and it was not called a
>> second time
>> because the line before line 212 didn't execute a second time.
>>
>> Breakpoint 1, IO_Trigger_Task (arg=0x0) at network_io.c:212
>> 212 if((r = uv_poll_start(poll_handle, UV_READABLE,
>> poll_callback)) < 0)
>> (gdb) bt
>> #0 IO_Trigger_Task (arg=0x0) at network_io.c:212
>> #1 0x0000000000413017 in uv__thread_start (arg=<optimized out>) at
>> src/unix/thread.c:49
>> #2 0x00007ffff7bc26aa in start_thread (arg=0x7ffff64e6700) at
>> pthread_create.c:333
>> #3 0x00007ffff75efeed in clone () at
>> ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
>> (gdb) s
>> uv_poll_start (handle=0x7fffec0008c0, pevents=1, poll_cb=0x404599
>> <poll_callback>) at src/unix/poll.c:89
>> 89 assert((pevents & ~(UV_READABLE | UV_WRITABLE)) == 0);
>> (gdb) s
>> 86 int uv_poll_start(uv_poll_t* handle, int pevents, uv_poll_cb
>> poll_cb) {
>> (gdb) bt
>> #0 uv_poll_start (handle=0x7fffec0008c0, pevents=1, poll_cb=0x404599
>> <poll_callback>) at src/unix/poll.c:86
>> #1 0x00000000004048bb in IO_Trigger_Task (arg=0x0) at network_io.c:212
>> #2 0x0000000000413017 in uv__thread_start (arg=<optimized out>) at
>> src/unix/thread.c:49
>> #3 0x00007ffff7bc26aa in start_thread (arg=0x7ffff64e6700) at
>> pthread_create.c:333
>> #4 0x00007ffff75efeed in clone () at
>> ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
>> (gdb)
>>
>> This is the relevant code of the IO_Trigger_Task() thread.
>>
>>
>> for(;;)
>> {
>> //
>> // Wait for a message from the main() process.
>> //
>> pm = WAIT_IO_Trigger_MSG();
>>
>> poll_handle = (uv_poll_t *) malloc(sizeof(uv_poll_t));
>> if(poll_handle == NULL)
>> {
>> fprintf(stderr, "IO_TRIGGER_TASK: No Poll HAndle Memory\n");
>> abort();
>> }
>>
>> uv_poll_init(&Poll_Loop, poll_handle, pm->info);
>> //
>> // Start epoll() monitoring of the connection.
>> //
>> if((r = uv_poll_start(poll_handle, UV_READABLE, poll_callback)) <
>> 0)
>> {
>> fprintf(stderr, "IO_TRIGGER_TASK: Polling Initiation Error
>> %d: %s\n", r, uv_err_name(r));
>> abort();
>> }
>>
>> MSG_FREE(pm);
>> }
>>
>>
>>
>> 2.4) The polling callback function never executes.
>>
>> NOTE: The polling loop, Poll_Loop, of type uv_loop_t is already running
>> and was started,
>> in the IO_Task() thread, at startup time as follows.
>>
>> uv_loop_init(&Poll_Loop);
>> for(;;)
>> {
>> r = uv_run(&Poll_Loop, UV_RUN_DEFAULT);
>> if(r)
>> fprintf(stderr, "IO_TASK: Run Error %d\n", r);
>> }
>>
>>
>> This is the sequence of operations used to free the first connection.
>>
>> 1) Release the uv_poll_t poll handle in the IO_Task() from the
>> Protocol_Task()
>>
>> //
>> // This causes immediate socket disconnection when it is closed.
>> //
>> spec.l_onoff = TRUE;
>> spec.l_linger = 0;
>> setsockopt(cdesc->fd, SOL_SOCKET, SO_LINGER, &spec, sizeof(spec) );
>> //
>> // poll_release_proxy() executes in the IO_Task() and releases poll
>> handle.
>> //
>> uv_async_init(&Poll_Loop, &cdesc->async_handle, poll_release_proxy);
>> cdesc->async_handle.data = (void *) cdesc;
>> uv_async_send(&cdesc->async_handle);
>>
>> 1.1) Wait for poll handle to be freed and then release the async. handle.
>>
>> uv_close((uv_handle_t *) &cdesc->async_handle, NULL);
>>
>> 2) Release the uv_tcp_t connect handle in the main() process from the
>> Protocol_Task()
>>
>> uv_async_init(&Connect_Loop, &cdesc->async_handle,
>> conn_release_proxy);
>> cdesc->async_handle.data = (void *) cdesc;
>> uv_async_send(&cdesc->async_handle);
>>
>> 2.1) Wait for the connect handle to be free and then release the async.
>> handle.
>>
>> uv_close((uv_handle_t *) &cdesc->async_handle, NULL);
>>
>> 3) Do protocol bookkeeping.
>>
>> This is the code of the proxy callback routines and the close callback
>> routine.
>>
>> //
>> // This routine executes asynchronously and frees a handle.
>> // It is invoked in the follows two cases.
>> //
>> // * When the main process invokes poll_release_proxy()
>> //
>> // * When the IO_Task invokes conn_release_proxy().
>> //
>> ROUTINE void close_callback(uv_handle_t *handle)
>> {
>> SDU *msg;
>>
>> CONN_DESC *cdesc = (CONN_DESC *) handle->data;
>>
>> free(handle);
>>
>> ENTER_MUTEX(&Service_Q_Mutex);
>> //
>> // Set the state correctly and validate the state.
>> //
>> switch(cdesc->release_state)
>> {
>> case RS_POLL_HANDLE_PEND:
>> cdesc->release_state = RS_POLL_HANDLE_FREE;
>> break;
>> case RS_CONN_HANDLE_PEND:
>> cdesc->release_state = RS_CONN_HANDLE_FREE;
>> break;
>> default:
>> fprintf(stderr, "CLOSE_PROXY - BUG: Invalid Release State =
>> %d\n", cdesc->release_state);
>> abort();
>> }
>> EXIT_MUTEX(&Service_Q_Mutex);
>>
>> //
>> // Send a notification message to the Protocol_Task.
>> //
>> msg = MSG_ALLOC(0, FALSE);
>> msg->class = C_NOTIFY;
>> msg->type = T_HANDLE_FREE;
>> msg->info = 0;
>>
>> SEND_SDU(cdesc, msg);
>>
>> return;
>> }
>>
>>
>> //
>> // This routine is invoked by the IO_Task() in response to an async.
>> wakeup by the Protocol_Task()
>> // during TCP connection termination. It release the resources used by
>> the Poll_Loop.
>> //
>> ROUTINE void poll_release_proxy(uv_async_t *async_handle)
>> {
>> CONN_DESC *cdesc = (CONN_DESC *) async_handle->data;
>>
>> //
>> // Stop polling operations before closing the handle.
>> //
>> uv_poll_stop(cdesc->poll_handle);
>> cdesc->poll_handle->data = (void *) cdesc;
>> uv_close((uv_handle_t *) cdesc->poll_handle, close_callback);
>>
>> return;
>> }
>>
>> //
>> // This routine is invoked by the main process in response to an async.
>> wakeup by the Protocol_Task()
>> // during TCP connection termination. It release the resources used by
>> the Connect_Loop.
>> //
>> ROUTINE void conn_release_proxy(uv_async_t *async_handle)
>> {
>> CONN_DESC *cdesc = (CONN_DESC *) async_handle->data;
>>
>> cdesc->conn_handle->data = (void *) cdesc;
>> uv_close((uv_handle_t *) cdesc->conn_handle, close_callback);
>>
>> return;
>> }
>>
>> Best Regards,
>>
>> Paul R.
>>
>> On Sunday, December 20, 2020 at 1:23:52 PM UTC-8 pa...@rcom-software.com
>> wrote:
>>
>>> Hi Folks:
>>>
>>> With limited testing the problem ceases to happen if you force uv_run()
>>> in the IO_Task()
>>> enough to finish its pending work. As an interim measure I do this by
>>> making the
>>> Protocol_Task() to yield the CPU after calling uv_stop() and
>>> up_poll_stop() as follows in
>>> the RELEASE_CONNECTION() routine. This appears to cause IO_Task() to be
>>> scheduled and run
>>> but I am not all all convinced this is a reliable technique.
>>>
>>> //
>>> // Deactive and release the poll handle.
>>> // You have stop the Poll_Loop to deactivate and deallocate the
>>> poll handle.
>>> //
>>> uv_stop(&Poll_Loop);
>>>
>>> uv_poll_stop(cdesc->poll_handle);
>>> #ifdef CLOSE_KLUDGE2
>>> //
>>> // Try to let run() in the IO_Task() finish pending work by
>>> yielding the CPU.
>>> //
>>> for(k = 0; k < 10; k++) pthread_yield();
>>> #endif // CLOSE_KLUDGE2
>>> uv_close((uv_handle_t *) cdesc->poll_handle, close_callback);
>>>
>>>
>>> Best Regards,
>>>
>>> Paul R.
>>>
>>>
>>> On Sunday, December 20, 2020 at 10:13:34 AM UTC-8
>>> pa...@rcom-software.com wrote:
>>>
>>>> Hi Folks:
>>>>
>>>> I made some progress on the problem but it is definitely not solved.
>>>> The updated code
>>>> and more diagnostic code are included in the message.
>>>>
>>>> NOTE: I am using the GIT HUB distribution from the following link on
>>>> Ubuntu Linux version 15.04.
>>>>
>>>> https://github.com/nikhilm/uvbook
>>>>
>>>> The Libuv software package looks like version 1.3.0.
>>>>
>>>> I have had to take extraordinary measures to make connection release
>>>> reliable.
>>>> The relevant code is included at near end of this message and the
>>>> extraordinary
>>>> measures are in the CLOSE_KLUDGE sections. The difficulty arises
>>>> because the
>>>> Libuv loops are not used in the Protocol_Task() yet it must affect
>>>> operations
>>>> on those loops to release handles. It would be nice if Libuv included
>>>> an API
>>>> for releasing handles reliably which could be called from any task.
>>>>
>>>> Connection release still fails about 15% of the time in which case a
>>>> crash occurs
>>>> and the following diagnostic is displayed.
>>>>
>>>> pexd: src/unix/core.c:210: uv__finish_close: Assertion
>>>> `!(handle->flags & UV_CLOSED)' failed.
>>>>
>>>> More diagnostic information follows. Do you know what causes this
>>>> crash ?
>>>>
>>>> Best Regards,
>>>>
>>>> Paul Romero
>>>>
>>>>
>>>> Crash Diagnostics
>>>> -----------------
>>>> The crash occurs when run() is executing in the IO_Task() in
>>>> network_io.c according to the following
>>>> GBD stack trace.
>>>>
>>>> #0 0x00007f281754c267 in __GI_raise (sig=sig@entry=6) at
>>>> ../sysdeps/unix/sysv/linux/raise.c:55
>>>> #1 0x00007f281754deca in __GI_abort () at abort.c:89
>>>> #2 0x00007f281754503d in __assert_fail_base (fmt=0x7f28176a7028
>>>> "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n",
>>>> assertion=assertion@entry=0x41e093 "!(handle->flags & UV_CLOSED)",
>>>> file=file@entry=0x41e068 "src/unix/core.c",
>>>> line=line@entry=210, function=function@entry=0x41e2b0
>>>> <__PRETTY_FUNCTION__.9522> "uv__finish_close") at assert.c:92
>>>> #3 0x00007f28175450f2 in __GI___assert_fail
>>>> (assertion=assertion@entry=0x41e093
>>>> "!(handle->flags & UV_CLOSED)",
>>>> file=file@entry=0x41e068 "src/unix/core.c", line=line@entry=210,
>>>> function=function@entry=0x41e2b0 <__PRETTY_FUNCTION__.9522>
>>>> "uv__finish_close") at assert.c:101
>>>> #4 0x000000000040c967 in uv__finish_close (handle=<optimized out>) at
>>>> src/unix/core.c:210
>>>> #5 uv__run_closing_handles (loop=0x638080 <Poll_Loop>) at
>>>> src/unix/core.c:259
>>>> #6 uv_run (loop=0x638080 <Poll_Loop>, mode=UV_RUN_DEFAULT) at
>>>> src/unix/core.c:326
>>>> #7 0x0000000000404962 in IO_Task (arg=0x0) at network_io.c:226
>>>> #8 0x0000000000412ad7 in uv__thread_start (arg=<optimized out>) at
>>>> src/unix/thread.c:49
>>>> #9 0x00007f2817bf06aa in start_thread (arg=0x7f2816d15700) at
>>>> pthread_create.c:333
>>>> #10 0x00007f281761deed in clone () at
>>>> ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
>>>>
>>>> However, the GDB thread information indicates that
>>>> RELEASE_CONNECTION(), in protocol.c, is executing
>>>> in the Protocol_Task() when the crash occurs.
>>>>
>>>> Id Target Id Frame
>>>> 6 Thread 0x7f2817516700 (LWP 3424) syscall () at
>>>> ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38
>>>> 5 Thread 0x7f2816514700 (LWP 3426) pthread_cond_wait@@GLIBC_2.3.2
>>>> ()
>>>> at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
>>>> 4 Thread 0x7f2818003700 (LWP 3423) syscall () at
>>>> ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38
>>>> 3 Thread 0x7f2815512700 (LWP 3428) pthread_cond_wait@@GLIBC_2.3.2
>>>> ()
>>>> at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
>>>> 2 Thread 0x7f2815d13700 (LWP 3427) 0x0000000000404500 in
>>>> RELEASE_CONNECTION (cdesc=0x6384c0 <Conn_Desc_Table>)
>>>> at protocol.c:357
>>>> * 1 Thread 0x7f2816d15700 (LWP 3425) 0x00007f281754c267 in
>>>> __GI_raise (sig=sig@entry=6)
>>>> at ../sysdeps/unix/sysv/linux/raise.c:55
>>>>
>>>> Line 357 of protocol.c is as follows.
>>>>
>>>> while(WaitClose[cdesc->index]);
>>>>
>>>> Wait_Close[] is only modified in two cases and only in the
>>>> Protocol_Task().
>>>>
>>>> 1) It is initialized to a handle address in RELEASE_CONNECTION() in the
>>>> Protocol_Task().
>>>> 2) It is cleared in the uv_close() callback routine close_callback().
>>>>
>>>>
>>>> Code
>>>> -----
>>>>
>>>> #define CLOSE_KLUDGE
>>>>
>>>> extern uv_loop_t Poll_Loop;
>>>> extern uv_loop_t Connect_Loop;
>>>>
>>>> #ifdef CLOSE_KLUDGE
>>>> uv_handle_t *WaitClose[MAX_CONN_DESC] = { NULL };
>>>> #endif // CLOSE_KLUDGE
>>>>
>>>> ROUTINE void close_callback(uv_handle_t *handle)
>>>> {
>>>> int k;
>>>>
>>>> free(handle);
>>>>
>>>> #ifdef CLOSE_KLUDGE
>>>> //
>>>> // Determine if the handle is being closed.
>>>> //
>>>> for(k = 0; k < MAX_CONN_DESC; k++)
>>>> {
>>>> if(WaitClose[k] == handle)
>>>> {
>>>> //
>>>> // Closure is complete.
>>>> //
>>>> WaitClose[k] = NULL;
>>>> break;
>>>> }
>>>> }
>>>> #endif // CLOSE_KLUDGE
>>>>
>>>> return;
>>>> }
>>>>
>>>> ROUTINE void RELEASE_CONNECTION(CONN_DESC *cdesc)
>>>> {
>>>> uv_async_t as_handle;
>>>> struct linger spec;
>>>>
>>>> if(N_Sockets > 0)
>>>> N_Sockets--;
>>>> //
>>>> // This causes immediate socket disconnection when it is closed.
>>>> //
>>>> spec.l_onoff = TRUE;
>>>> spec.l_linger = 0;
>>>> setsockopt(cdesc->fd, SOL_SOCKET, SO_LINGER, &spec, sizeof(spec) );
>>>>
>>>> if(cdesc->poll_handle)
>>>> {
>>>> #ifdef CLOSE_KLUDGE
>>>> WaitClose[cdesc->index] = (uv_handle_t *) cdesc->poll_handle;
>>>> #endif // CLOSE_KLUDGE
>>>> //
>>>> // Deactive and release the poll handle.
>>>> // You have stop the Poll_Loop to deactivate and deallocate the
>>>> poll handle.
>>>> //
>>>> uv_stop(&Poll_Loop);
>>>>
>>>> uv_poll_stop(cdesc->poll_handle);
>>>> uv_close((uv_handle_t *) cdesc->poll_handle, close_callback);
>>>> //
>>>> // Wake up the Poll_Loop in the IO_Task()
>>>> //
>>>> uv_async_init(&Poll_Loop, &as_handle, NULL);
>>>> uv_async_send(&as_handle);
>>>> uv_close((uv_handle_t *) &as_handle, NULL);
>>>> #ifdef CLOSE_KLUDGE
>>>> //
>>>> // Wait for the handle to be closed and deallocated.
>>>> //
>>>> while(WaitClose[cdesc->index]);
>>>> #endif // CLOSE_KLUDGE
>>>> }
>>>>
>>>> if(cdesc->conn_handle)
>>>> {
>>>> #ifdef CLOSE_KLUDGE
>>>> WaitClose[cdesc->index] = (uv_handle_t *) cdesc->conn_handle;
>>>> #endif // CLOSE_KLUDGE
>>>> //
>>>> // Close and deallocate the connect handle in order to close
>>>> the socket connecction.
>>>> // You have to wake up the Connect_Loop for the close_callback()
>>>> // routine to execute.
>>>> //
>>>> uv_close((uv_handle_t *) cdesc->conn_handle, close_callback);
>>>> //
>>>> // Wake up the Connect_Loop in the main() process.
>>>> //
>>>> uv_async_init(&Connect_Loop, &as_handle, NULL);
>>>> uv_async_send(&as_handle);
>>>> uv_close((uv_handle_t *) &as_handle, NULL);
>>>> #ifdef CLOSE_KLUDGE
>>>> //
>>>> // Wait for the handle and socket connection to be release and
>>>> closed.
>>>> //
>>>> while(WaitClose[cdesc->index]);
>>>> #endif // CLOSE_KLUDGE
>>>> }
>>>>
>>>>
>>>> ENTER_MUTEX(&Service_Q_Mutex);
>>>> DELETE_CONN(cdesc);
>>>> cdesc->fd = -1;
>>>> flush_msg(&cdesc->task_input_q);
>>>> EXIT_MUTEX(&Service_Q_Mutex);
>>>>
>>>> return;
>>>> }
>>>>
>>>> On Sunday, December 20, 2020 at 3:47:07 AM UTC-8
>>>> pa...@rcom-software.com wrote:
>>>>
>>>>> Hi Folks:
>>>>>
>>>>> My Libuv based Server performs all its functions correctly except for
>>>>> TCP connection termination.
>>>>>
>>>>> Each TCP connection has uv_tcp_t connection handle and uv_poll_t
>>>>> handle whose allocation
>>>>> and operation are explained below. When the Protocol_Task() thread
>>>>> needs to terminate
>>>>> a connection, it must stop polling, terminate the TCP socket
>>>>> connection, and deallocate
>>>>> the handles.
>>>>>
>>>>> NOTE: I am using the GIT HUB distribution from the following link on
>>>>> Ubuntu Linux version 15.04.
>>>>>
>>>>> https://github.com/nikhilm/uvbook
>>>>>
>>>>> I have tried the following two approaches.
>>>>>
>>>>> 1) Just use uv_poll_stop() to terminate polling and uv_close() to
>>>>> terminate the TCP connection.
>>>>>
>>>>> 2) Use uv_poll_stop() to terminate polling and the using
>>>>> uv_queue_work() and uv_async_send() to
>>>>> wake up the Connect_Loop, in the main() process described below, so
>>>>> it can terminate the
>>>>> TCP connection, by proxy, with uv_close().
>>>>>
>>>>> In both cases the following problem occurs. The callback routine
>>>>> supplied to uv_close()
>>>>> does not execute until another incoming TCP connection occurs, and in
>>>>> most cases,
>>>>> the Pool_Loop, in the IO_Task() described below, stops invoking it
>>>>> callback routine--
>>>>> poll_callback(). In case 2, a crash almost alway ensues. (I probably
>>>>> am not using
>>>>> uv_async_send() correctly.)
>>>>>
>>>>> Do I have a fundamental misunderstanding of how Libuv works or am I
>>>>> doing something wrong ?
>>>>>
>>>>> Also, I strongly suspect using Linux recv() to read data is not
>>>>> optimal when epoll() is
>>>>> being used. My understanding is that there is a way to pass buffers to
>>>>> epoll() such that
>>>>> data will automatically be inserted in them when a UV_READABLE event
>>>>> occurs. Do you have
>>>>> any advice about this ?
>>>>>
>>>>> An overview of my Server and the relevant code follow.
>>>>>
>>>>> Best Regards,
>>>>>
>>>>> Paul Romero
>>>>>
>>>>> Multi-Connection TCP Server Functional Architecture Overview
>>>>>
>>>>> -----------------------------------------------------------------------------------------
>>>>> There is a connection descriptor for each incoming TCP connection
>>>>> which contains all data
>>>>> needed to manage the connection and perform the relevant functions.
>>>>>
>>>>> When the main() process detects an incoming TCP connection, it sends a
>>>>> notification message to the
>>>>> IO_Trigger_Task(). The IO_Trigger_Task() then sets up epoll()
>>>>> monitoring of incoming TCP data
>>>>> for that connection.
>>>>>
>>>>> Subsequently, the IO_Task() invokes poll_callback() when incoming data
>>>>> is available, reads a chunk
>>>>> of data, and sends a protocol message to the Protocol_Task() when a
>>>>> complete protocol message is
>>>>> recognized.
>>>>>
>>>>> The Timer_Task() sends an expiration notification message to the
>>>>> Protocol_Task() when a protocol
>>>>> timer expires.
>>>>>
>>>>> The Protocol_Task() send messages to the Send_Op_Task() for
>>>>> transmission across the network.
>>>>> It spawns a DB Operation Task to perform slow data base operations and
>>>>> the DB Operation Task
>>>>> notifies the Protocol_Task() when the operation is complete and then
>>>>> terminates.
>>>>>
>>>>> Loops of type uv_loop_t
>>>>> -----------------------
>>>>> * Connect_Loop
>>>>> * Pool_Loop
>>>>> * Timer_Loop`
>>>>>
>>>>> Tasks: All Libuv thread tasks run concurrently and are launched by
>>>>> main() at startup time.
>>>>>
>>>>> ------------------------------------------------------------------------------------------
>>>>> * main(): A Linux process that runs the Connect_Loop to detect
>>>>> incoming TCP connections.
>>>>> The make_incoming_connection() callback routine accepts incoming
>>>>> connections and
>>>>> allocates a uv_tcp_t handle on a per connection basis
>>>>>
>>>>> * IO_Trigger_Task(): A Libuv thread that sets up epoll() plumbing for
>>>>> the IO_Task()
>>>>> when an incoming TCP connection occurs. It allocates a uv_poll_t
>>>>> handle, on a per
>>>>> connection basis, and calls uv_poll_start() to initiate epoll()
>>>>> operation with the
>>>>> Poll_Loop in the IO_Task(). It configures the handle to detect
>>>>> UV_READABLE events and
>>>>> handles them with the poll_callback() routine. However, it does not
>>>>> run the Poll_Loop.
>>>>> (Basically, this task just sets up plumbing.)
>>>>>
>>>>> * IO_Task(): A Libuv thread that runs the Poll_Loop to handle incoming
>>>>> TCP data, on a per
>>>>> connection basis. The poll_callback() routine executes and uses
>>>>> normal Linux recv() to read
>>>>> chunks of data, in non-blocking mode, when a UV_READABLE event
>>>>> occurs.
>>>>>
>>>>> * Timer_Task(): A Libuv thread that runs the Time_Loop to handle
>>>>> ticks, and whose main
>>>>> function is to detect protocol timer expiration. The tick duration
>>>>> is configured with
>>>>> is configured with uv_timer_init() and uv_timer_start(), and ticks
>>>>> are handled by the
>>>>> timer_callback() routine.
>>>>>
>>>>> * Protocol_Task(): A Libuv thread that handles protocol messages sent
>>>>> to it by the following tasks
>>>>> on per connection basis: IO_Task(), Timer_Task(), DB Operation
>>>>> Tasks. DB Operation Libuv thread tasks
>>>>> are spawned by the Protocol_Task() to perform slow database
>>>>> operations and send a notification message
>>>>> to the Protocol_Task() upon completion of the operation.
>>>>>
>>>>> * Send_Op_Task(): A Libuv thread that transmits all network bound
>>>>> messages with normal
>>>>> Linux send() on a per connection basis.
>>>>>
>>>>>
>>>>> Approach 1 Code
>>>>> -------------
>>>>> ROUTINE void close_callback(uv_handle_t *handle)
>>>>> {
>>>>>
>>>>> free(handle);
>>>>> return;
>>>>> }
>>>>>
>>>>> ROUTINE void RELEASE_CONNECTION(CONN_DESC *cdesc)
>>>>> {
>>>>> struct linger spec;
>>>>> int r;
>>>>>
>>>>> if(N_Sockets > 0)
>>>>> N_Sockets--;
>>>>>
>>>>> if(cdesc->poll_handle)
>>>>> {
>>>>> uv_poll_stop(cdesc->poll_handle);
>>>>> free((void *) cdesc->poll_handle);
>>>>> }
>>>>>
>>>>> if(cdesc->conn_handle)
>>>>> {
>>>>> struct linger spec;
>>>>>
>>>>> spec.l_onoff = TRUE;
>>>>> spec.l_linger = 0;
>>>>> setsockopt(cdesc->fd, SOL_SOCKET, SO_LINGER, &spec,
>>>>> sizeof(spec) );
>>>>>
>>>>> uv_close((uv_handle_t *) cdesc->conn_handle, close_callback);
>>>>> }
>>>>>
>>>>> ENTER_MUTEX(&Service_Q_Mutex);
>>>>> DELETE_CONN(cdesc);
>>>>> cdesc->fd = -1;
>>>>> flush_msg(&cdesc->task_input_q);
>>>>> EXIT_MUTEX(&Service_Q_Mutex);
>>>>>
>>>>> return;
>>>>> }
>>>>>
>>>>> Approach 2 Code
>>>>> -----------------
>>>>> ROUTINE void close_callback(uv_handle_t *handle)
>>>>> {
>>>>> free(handle);
>>>>> return;
>>>>> }
>>>>>
>>>>> typedef struct close_template {
>>>>> uv_handle_t *handle;
>>>>> void (*callback) (uv_handle_t *);
>>>>> } CLOSE_TEMPLATE;
>>>>>
>>>>> ROUTINE void close_proxy(uv_work_t *data)
>>>>> {
>>>>> CLOSE_TEMPLATE *cparam = (CLOSE_TEMPLATE *) cparam;
>>>>>
>>>>> uv_close(cparam->handle, cparam->callback);
>>>>> return;
>>>>> }
>>>>>
>>>>>
>>>>> extern uv_loop_t Connect_Loop;
>>>>> static CLOSE_TEMPLATE close_data;
>>>>>
>>>>> ROUTINE void RELEASE_CONNECTION(CONN_DESC *cdesc)
>>>>> {
>>>>> uv_work_t wreq;
>>>>> uv_async_t as_handle;
>>>>> struct linger spec;
>>>>>
>>>>> if(N_Sockets > 0)
>>>>> N_Sockets--;
>>>>>
>>>>> //
>>>>> // Stop this. TBD: Might need to do this via proxy in the
>>>>> IO_Task() Poll_Loop.
>>>>> //
>>>>> uv_poll_stop(cdesc->poll_handle);
>>>>>
>>>>> uv_async_init(&Connect_Loop, &as_handle, NULL);
>>>>>
>>>>> close_data.handle = (uv_handle_t *) cdesc->conn_handle;
>>>>> close_data.callback = close_callback;
>>>>> //
>>>>> // Call uv_close() in the close_proxy()
>>>>> //
>>>>> wreq.data = (void *) &close_data;
>>>>> uv_queue_work(&Connect_Loop, &wreq, close_proxy, NULL);
>>>>>
>>>>> spec.l_onoff = TRUE;
>>>>> spec.l_linger = 0;
>>>>> setsockopt(cdesc->fd, SOL_SOCKET, SO_LINGER, &spec, sizeof(spec) );
>>>>>
>>>>> uv_async_send(&as_handle);
>>>>> uv_close((uv_handle_t *) &as_handle, NULL);
>>>>>
>>>>> free(cdesc->poll_handle);
>>>>>
>>>>> ENTER_MUTEX(&Service_Q_Mutex);
>>>>> DELETE_CONN(cdesc);
>>>>> cdesc->fd = -1;
>>>>> flush_msg(&cdesc->task_input_q);
>>>>> EXIT_MUTEX(&Service_Q_Mutex);
>>>>>
>>>>> return;
>>>>> }
>>>>>
>>>> --
> You received this message because you are subscribed to the Google Groups
> "libuv" group.
> To unsubscribe from this group and stop receiving emails from it, send an
> email to libuv+unsubscr...@googlegroups.com.
> To view this discussion on the web visit
> https://groups.google.com/d/msgid/libuv/893eb3ed-685e-416b-9d05-3501588095den%40googlegroups.com
> <https://groups.google.com/d/msgid/libuv/893eb3ed-685e-416b-9d05-3501588095den%40googlegroups.com?utm_medium=email&utm_source=footer>
> .
>
--
You received this message because you are subscribed to the Google Groups
"libuv" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to libuv+unsubscr...@googlegroups.com.
To view this discussion on the web visit
https://groups.google.com/d/msgid/libuv/CADnnjUU5fwbCzRO4eLRRjdzD0jtTZUxcQ6B84aQ9P_pt_McW3Q%40mail.gmail.com.
