[
https://issues.apache.org/jira/browse/OPENMEETINGS-2355?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Konstantin Kuzov updated OPENMEETINGS-2355:
-------------------------------------------
Description:
After OPENMEETINGS-2315 and OPENMEETINGS-2316 some stream connections states
not terminating fully and there are growing list of stale streams in
admin->connections (look at stale-processors.png).
It happens randomly, most likely due to unreliable network, slow devices or
browser quirks.
I found the only one reliable way to reproduce this (for me working 100% of
the time) using android's chrome which also unearth reconnection problem:
- enter the room with one another participant
- not closing tab with om, just switch to another tab with random page.
Another participant will see frozen image.
- wait around 10 seconds and switch back, on your end you will see fully
working webcam, but on participant's end the image will still be frozen.
- after another 5 seconds there will be a message on both ends about trying to
reconnect which will result in fast refresh of already seamingly working camera
on android chrome, but on another participant there will be only empty window
with avatar. And it will stay that way forever. At that point android user can
exit the room and his stream will stuck.
During reconnect attempt there are this error:
{noformat}
ERROR 05-14 16:42:46.464 o.a.o.w.c.OmWebSocketPanel:144 [-nio-443-exec-7] -
Error while processing incoming message
java.lang.NullPointerException: null
at
org.apache.openmeetings.core.remote.KStream.startBroadcast(KStream.java:88)
at
org.apache.openmeetings.core.remote.StreamProcessor.startBroadcast(StreamProcessor.java:212)
at
org.apache.openmeetings.core.remote.StreamProcessor.handleBroadcastStarted(StreamProcessor.java:187)
at
org.apache.openmeetings.core.remote.StreamProcessor.onMessage(StreamProcessor.java:110)
at
org.apache.openmeetings.core.remote.KurentoHandler.onMessage(KurentoHandler.java:233)
at
org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$FastClassByCGLIB$$756e116d.invoke(<generated>)
at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at
org.apache.wicket.proxy.LazyInitProxyFactory$AbstractCGLibInterceptor.intercept(LazyInitProxyFactory.java:364)
at
org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$EnhancerByCGLIB$$e2e8f0f.onMessage(<generated>)
at
org.apache.openmeetings.web.common.OmWebSocketPanel$2.onMessage(OmWebSocketPanel.java:125)
at
org.apache.wicket.protocol.ws.api.WebSocketBehavior.onEvent(WebSocketBehavior.java:64)
at
org.apache.wicket.settings.FrameworkSettings.dispatchEvent(FrameworkSettings.java:147)
at
org.apache.wicket.ComponentEventSender.dispatchToComponent(ComponentEventSender.java:292)
at
org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:329)
at
org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:306)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:144)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:123)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:192)
at
org.apache.wicket.MarkupContainer.visitChildren(MarkupContainer.java:988)
at
org.apache.wicket.ComponentEventSender.breadth(ComponentEventSender.java:160)
at
org.apache.wicket.ComponentEventSender.send(ComponentEventSender.java:68)
at org.apache.wicket.Component.send(Component.java:4418)
at
org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler$1.run(WebSocketMessageBroadcastHandler.java:74)
at
org.apache.wicket.protocol.ws.WebSocketSettings$SameThreadExecutor.run(WebSocketSettings.java:393)
at
org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler.respond(WebSocketMessageBroadcastHandler.java:97)
at
org.apache.wicket.request.cycle.RequestCycle$HandlerExecutor.respond(RequestCycle.java:917)
at
org.apache.wicket.request.RequestHandlerExecutor.execute(RequestHandlerExecutor.java:63)
at
org.apache.wicket.request.cycle.RequestCycle.execute(RequestCycle.java:283)
at
org.apache.wicket.request.cycle.RequestCycle.processRequest(RequestCycle.java:254)
at
org.apache.wicket.request.cycle.RequestCycle.processRequestAndDetach(RequestCycle.java:222)
at
org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.broadcastMessage(AbstractWebSocketProcessor.java:272)
at
org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.onMessage(AbstractWebSocketProcessor.java:151)
at
org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:65)
at
org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:60)
at
org.apache.tomcat.websocket.WsFrameBase.sendMessageText(WsFrameBase.java:395)
at
org.apache.tomcat.websocket.server.WsFrameServer.sendMessageText(WsFrameServer.java:119)
at
org.apache.tomcat.websocket.WsFrameBase.processDataText(WsFrameBase.java:495)
at
org.apache.tomcat.websocket.WsFrameBase.processData(WsFrameBase.java:294)
at
org.apache.tomcat.websocket.WsFrameBase.processInputBuffer(WsFrameBase.java:133)
at
org.apache.tomcat.websocket.server.WsFrameServer.onDataAvailable(WsFrameServer.java:82)
at
org.apache.tomcat.websocket.server.WsFrameServer.doOnDataAvailable(WsFrameServer.java:171)
at
org.apache.tomcat.websocket.server.WsFrameServer.notifyDataAvailable(WsFrameServer.java:151)
at
org.apache.tomcat.websocket.server.WsHttpUpgradeHandler.upgradeDispatch(WsHttpUpgradeHandler.java:148)
at
org.apache.coyote.http11.upgrade.UpgradeProcessorInternal.dispatch(UpgradeProcessorInternal.java:54)
at
org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:59)
at
org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
at
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1590)
at
org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.base/java.lang.Thread.run(Thread.java:834)
{noformat}
And right after one above multiple ones like this:
{noformat}
DEBUG 05-14 16:42:46.471 o.a.o.c.r.StreamProcessor:91 [-nio-443-exec-7] -
Incoming message from user with ID '5':
{"type":"kurento","id":"onIceCandidate","candidate":{"candidate":"candidate:505713252
1 udp 2122260223 10.1.1.67 43537 typ host generation 0 ufrag JMPb network-id 2
network-cost
10","sdpMid":"0","sdpMLineIndex":0},"uid":"eec0c0ef-10fe-4070-b40d-95bcec65a09a","luid":"eec0c0ef-10fe-4070-b40d-95bcec65a09a"}
ERROR 05-14 16:42:46.471 o.a.o.w.c.OmWebSocketPanel:144 [-nio-443-exec-7] -
Error while processing incoming message
java.lang.NullPointerException: null
at
org.apache.openmeetings.core.remote.KStream.addCandidate(KStream.java:393)
at
org.apache.openmeetings.core.remote.StreamProcessor.onMessage(StreamProcessor.java:122)
at
org.apache.openmeetings.core.remote.KurentoHandler.onMessage(KurentoHandler.java:233)
at
org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$FastClassByCGLIB$$756e116d.invoke(<generated>)
at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at
org.apache.wicket.proxy.LazyInitProxyFactory$AbstractCGLibInterceptor.intercept(LazyInitProxyFactory.java:364)
at
org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$EnhancerByCGLIB$$e2e8f0f.onMessage(<generated>)
at
org.apache.openmeetings.web.common.OmWebSocketPanel$2.onMessage(OmWebSocketPanel.java:125)
at
org.apache.wicket.protocol.ws.api.WebSocketBehavior.onEvent(WebSocketBehavior.java:64)
at
org.apache.wicket.settings.FrameworkSettings.dispatchEvent(FrameworkSettings.java:147)
at
org.apache.wicket.ComponentEventSender.dispatchToComponent(ComponentEventSender.java:292)
at
org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:329)
at
org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:306)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:144)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:123)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:192)
at
org.apache.wicket.MarkupContainer.visitChildren(MarkupContainer.java:988)
at
org.apache.wicket.ComponentEventSender.breadth(ComponentEventSender.java:160)
at
org.apache.wicket.ComponentEventSender.send(ComponentEventSender.java:68)
at org.apache.wicket.Component.send(Component.java:4418)
at
org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler$1.run(WebSocketMessageBroadcastHandler.java:74)
at
org.apache.wicket.protocol.ws.WebSocketSettings$SameThreadExecutor.run(WebSocketSettings.java:393)
at
org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler.respond(WebSocketMessageBroadcastHandler.java:97)
at
org.apache.wicket.request.cycle.RequestCycle$HandlerExecutor.respond(RequestCycle.java:917)
at
org.apache.wicket.request.RequestHandlerExecutor.execute(RequestHandlerExecutor.java:63)
at
org.apache.wicket.request.cycle.RequestCycle.execute(RequestCycle.java:283)
at
org.apache.wicket.request.cycle.RequestCycle.processRequest(RequestCycle.java:254)
at
org.apache.wicket.request.cycle.RequestCycle.processRequestAndDetach(RequestCycle.java:222)
at
org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.broadcastMessage(AbstractWebSocketProcessor.java:272)
at
org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.onMessage(AbstractWebSocketProcessor.java:151)
at
org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:65)
at
org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:60)
at
org.apache.tomcat.websocket.WsFrameBase.sendMessageText(WsFrameBase.java:395)
at
org.apache.tomcat.websocket.server.WsFrameServer.sendMessageText(WsFrameServer.java:119)
at
org.apache.tomcat.websocket.WsFrameBase.processDataText(WsFrameBase.java:495)
at
org.apache.tomcat.websocket.WsFrameBase.processData(WsFrameBase.java:294)
at
org.apache.tomcat.websocket.WsFrameBase.processInputBuffer(WsFrameBase.java:133)
at
org.apache.tomcat.websocket.server.WsFrameServer.onDataAvailable(WsFrameServer.java:82)
at
org.apache.tomcat.websocket.server.WsFrameServer.doOnDataAvailable(WsFrameServer.java:171)
at
org.apache.tomcat.websocket.server.WsFrameServer.notifyDataAvailable(WsFrameServer.java:151)
at
org.apache.tomcat.websocket.server.WsHttpUpgradeHandler.upgradeDispatch(WsHttpUpgradeHandler.java:148)
at
org.apache.coyote.http11.upgrade.UpgradeProcessorInternal.dispatch(UpgradeProcessorInternal.java:54)
at
org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:59)
at
org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
at
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1590)
at
org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.base/java.lang.Thread.run(Thread.java:834)
{noformat}
was:
After OPENMEETINGS-2315 and OPENMEETINGS-2316 some stream connections states
not terminating fully and there are growing list of stale streams in
admin->connections (look at stale-processors.png).
It happens randomly, most likely due to unreliable network, slow devices or
browser quirks.
I found the only one reliable way to reproduce this (for me working 100% of
the time) using android's chrome which also unearth reconnection problem:
- enter the room with one another participant
- not closing tab with om, just switch to another tab with random page.
Another participant will see frozen image.
- wait around 10 seconds and switch back, on your end you will see fully
working webcam, but on participant's end the image will still be frozen.
- after another 5 seconds there will be a message on both ends about trying to
reconnect which will result in fast refresh already seamingly working camera on
android chrome, but on another participant there will be only empty window with
avatar. And it will stay that way forever. At that point android user can exit
the room and his stream will stuck.
During reconnect attempt there are this error:
{noformat}
ERROR 05-14 16:42:46.464 o.a.o.w.c.OmWebSocketPanel:144 [-nio-443-exec-7] -
Error while processing incoming message
java.lang.NullPointerException: null
at
org.apache.openmeetings.core.remote.KStream.startBroadcast(KStream.java:88)
at
org.apache.openmeetings.core.remote.StreamProcessor.startBroadcast(StreamProcessor.java:212)
at
org.apache.openmeetings.core.remote.StreamProcessor.handleBroadcastStarted(StreamProcessor.java:187)
at
org.apache.openmeetings.core.remote.StreamProcessor.onMessage(StreamProcessor.java:110)
at
org.apache.openmeetings.core.remote.KurentoHandler.onMessage(KurentoHandler.java:233)
at
org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$FastClassByCGLIB$$756e116d.invoke(<generated>)
at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at
org.apache.wicket.proxy.LazyInitProxyFactory$AbstractCGLibInterceptor.intercept(LazyInitProxyFactory.java:364)
at
org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$EnhancerByCGLIB$$e2e8f0f.onMessage(<generated>)
at
org.apache.openmeetings.web.common.OmWebSocketPanel$2.onMessage(OmWebSocketPanel.java:125)
at
org.apache.wicket.protocol.ws.api.WebSocketBehavior.onEvent(WebSocketBehavior.java:64)
at
org.apache.wicket.settings.FrameworkSettings.dispatchEvent(FrameworkSettings.java:147)
at
org.apache.wicket.ComponentEventSender.dispatchToComponent(ComponentEventSender.java:292)
at
org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:329)
at
org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:306)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:144)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:123)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:192)
at
org.apache.wicket.MarkupContainer.visitChildren(MarkupContainer.java:988)
at
org.apache.wicket.ComponentEventSender.breadth(ComponentEventSender.java:160)
at
org.apache.wicket.ComponentEventSender.send(ComponentEventSender.java:68)
at org.apache.wicket.Component.send(Component.java:4418)
at
org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler$1.run(WebSocketMessageBroadcastHandler.java:74)
at
org.apache.wicket.protocol.ws.WebSocketSettings$SameThreadExecutor.run(WebSocketSettings.java:393)
at
org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler.respond(WebSocketMessageBroadcastHandler.java:97)
at
org.apache.wicket.request.cycle.RequestCycle$HandlerExecutor.respond(RequestCycle.java:917)
at
org.apache.wicket.request.RequestHandlerExecutor.execute(RequestHandlerExecutor.java:63)
at
org.apache.wicket.request.cycle.RequestCycle.execute(RequestCycle.java:283)
at
org.apache.wicket.request.cycle.RequestCycle.processRequest(RequestCycle.java:254)
at
org.apache.wicket.request.cycle.RequestCycle.processRequestAndDetach(RequestCycle.java:222)
at
org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.broadcastMessage(AbstractWebSocketProcessor.java:272)
at
org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.onMessage(AbstractWebSocketProcessor.java:151)
at
org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:65)
at
org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:60)
at
org.apache.tomcat.websocket.WsFrameBase.sendMessageText(WsFrameBase.java:395)
at
org.apache.tomcat.websocket.server.WsFrameServer.sendMessageText(WsFrameServer.java:119)
at
org.apache.tomcat.websocket.WsFrameBase.processDataText(WsFrameBase.java:495)
at
org.apache.tomcat.websocket.WsFrameBase.processData(WsFrameBase.java:294)
at
org.apache.tomcat.websocket.WsFrameBase.processInputBuffer(WsFrameBase.java:133)
at
org.apache.tomcat.websocket.server.WsFrameServer.onDataAvailable(WsFrameServer.java:82)
at
org.apache.tomcat.websocket.server.WsFrameServer.doOnDataAvailable(WsFrameServer.java:171)
at
org.apache.tomcat.websocket.server.WsFrameServer.notifyDataAvailable(WsFrameServer.java:151)
at
org.apache.tomcat.websocket.server.WsHttpUpgradeHandler.upgradeDispatch(WsHttpUpgradeHandler.java:148)
at
org.apache.coyote.http11.upgrade.UpgradeProcessorInternal.dispatch(UpgradeProcessorInternal.java:54)
at
org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:59)
at
org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
at
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1590)
at
org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.base/java.lang.Thread.run(Thread.java:834)
{noformat}
And right after one above multiple ones like this:
{noformat}
DEBUG 05-14 16:42:46.471 o.a.o.c.r.StreamProcessor:91 [-nio-443-exec-7] -
Incoming message from user with ID '5':
{"type":"kurento","id":"onIceCandidate","candidate":{"candidate":"candidate:505713252
1 udp 2122260223 10.1.1.67 43537 typ host generation 0 ufrag JMPb network-id 2
network-cost
10","sdpMid":"0","sdpMLineIndex":0},"uid":"eec0c0ef-10fe-4070-b40d-95bcec65a09a","luid":"eec0c0ef-10fe-4070-b40d-95bcec65a09a"}
ERROR 05-14 16:42:46.471 o.a.o.w.c.OmWebSocketPanel:144 [-nio-443-exec-7] -
Error while processing incoming message
java.lang.NullPointerException: null
at
org.apache.openmeetings.core.remote.KStream.addCandidate(KStream.java:393)
at
org.apache.openmeetings.core.remote.StreamProcessor.onMessage(StreamProcessor.java:122)
at
org.apache.openmeetings.core.remote.KurentoHandler.onMessage(KurentoHandler.java:233)
at
org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$FastClassByCGLIB$$756e116d.invoke(<generated>)
at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at
org.apache.wicket.proxy.LazyInitProxyFactory$AbstractCGLibInterceptor.intercept(LazyInitProxyFactory.java:364)
at
org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$EnhancerByCGLIB$$e2e8f0f.onMessage(<generated>)
at
org.apache.openmeetings.web.common.OmWebSocketPanel$2.onMessage(OmWebSocketPanel.java:125)
at
org.apache.wicket.protocol.ws.api.WebSocketBehavior.onEvent(WebSocketBehavior.java:64)
at
org.apache.wicket.settings.FrameworkSettings.dispatchEvent(FrameworkSettings.java:147)
at
org.apache.wicket.ComponentEventSender.dispatchToComponent(ComponentEventSender.java:292)
at
org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:329)
at
org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:306)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:144)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:123)
at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:192)
at
org.apache.wicket.MarkupContainer.visitChildren(MarkupContainer.java:988)
at
org.apache.wicket.ComponentEventSender.breadth(ComponentEventSender.java:160)
at
org.apache.wicket.ComponentEventSender.send(ComponentEventSender.java:68)
at org.apache.wicket.Component.send(Component.java:4418)
at
org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler$1.run(WebSocketMessageBroadcastHandler.java:74)
at
org.apache.wicket.protocol.ws.WebSocketSettings$SameThreadExecutor.run(WebSocketSettings.java:393)
at
org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler.respond(WebSocketMessageBroadcastHandler.java:97)
at
org.apache.wicket.request.cycle.RequestCycle$HandlerExecutor.respond(RequestCycle.java:917)
at
org.apache.wicket.request.RequestHandlerExecutor.execute(RequestHandlerExecutor.java:63)
at
org.apache.wicket.request.cycle.RequestCycle.execute(RequestCycle.java:283)
at
org.apache.wicket.request.cycle.RequestCycle.processRequest(RequestCycle.java:254)
at
org.apache.wicket.request.cycle.RequestCycle.processRequestAndDetach(RequestCycle.java:222)
at
org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.broadcastMessage(AbstractWebSocketProcessor.java:272)
at
org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.onMessage(AbstractWebSocketProcessor.java:151)
at
org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:65)
at
org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:60)
at
org.apache.tomcat.websocket.WsFrameBase.sendMessageText(WsFrameBase.java:395)
at
org.apache.tomcat.websocket.server.WsFrameServer.sendMessageText(WsFrameServer.java:119)
at
org.apache.tomcat.websocket.WsFrameBase.processDataText(WsFrameBase.java:495)
at
org.apache.tomcat.websocket.WsFrameBase.processData(WsFrameBase.java:294)
at
org.apache.tomcat.websocket.WsFrameBase.processInputBuffer(WsFrameBase.java:133)
at
org.apache.tomcat.websocket.server.WsFrameServer.onDataAvailable(WsFrameServer.java:82)
at
org.apache.tomcat.websocket.server.WsFrameServer.doOnDataAvailable(WsFrameServer.java:171)
at
org.apache.tomcat.websocket.server.WsFrameServer.notifyDataAvailable(WsFrameServer.java:151)
at
org.apache.tomcat.websocket.server.WsHttpUpgradeHandler.upgradeDispatch(WsHttpUpgradeHandler.java:148)
at
org.apache.coyote.http11.upgrade.UpgradeProcessorInternal.dispatch(UpgradeProcessorInternal.java:54)
at
org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:59)
at
org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
at
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1590)
at
org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.base/java.lang.Thread.run(Thread.java:834)
{noformat}
> Growing list of stream connection in admin->connections
> -------------------------------------------------------
>
> Key: OPENMEETINGS-2355
> URL: https://issues.apache.org/jira/browse/OPENMEETINGS-2355
> Project: Openmeetings
> Issue Type: Bug
> Components: Room
> Affects Versions: 5.0.0-M5
> Reporter: Konstantin Kuzov
> Assignee: Maxim Solodovnik
> Priority: Major
> Attachments: stale-processors.png
>
>
> After OPENMEETINGS-2315 and OPENMEETINGS-2316 some stream connections states
> not terminating fully and there are growing list of stale streams in
> admin->connections (look at stale-processors.png).
> It happens randomly, most likely due to unreliable network, slow devices or
> browser quirks.
> I found the only one reliable way to reproduce this (for me working 100% of
> the time) using android's chrome which also unearth reconnection problem:
> - enter the room with one another participant
> - not closing tab with om, just switch to another tab with random page.
> Another participant will see frozen image.
> - wait around 10 seconds and switch back, on your end you will see fully
> working webcam, but on participant's end the image will still be frozen.
> - after another 5 seconds there will be a message on both ends about trying
> to reconnect which will result in fast refresh of already seamingly working
> camera on android chrome, but on another participant there will be only empty
> window with avatar. And it will stay that way forever. At that point android
> user can exit the room and his stream will stuck.
> During reconnect attempt there are this error:
> {noformat}
> ERROR 05-14 16:42:46.464 o.a.o.w.c.OmWebSocketPanel:144 [-nio-443-exec-7] -
> Error while processing incoming message
> java.lang.NullPointerException: null
> at
> org.apache.openmeetings.core.remote.KStream.startBroadcast(KStream.java:88)
> at
> org.apache.openmeetings.core.remote.StreamProcessor.startBroadcast(StreamProcessor.java:212)
> at
> org.apache.openmeetings.core.remote.StreamProcessor.handleBroadcastStarted(StreamProcessor.java:187)
> at
> org.apache.openmeetings.core.remote.StreamProcessor.onMessage(StreamProcessor.java:110)
> at
> org.apache.openmeetings.core.remote.KurentoHandler.onMessage(KurentoHandler.java:233)
> at
> org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$FastClassByCGLIB$$756e116d.invoke(<generated>)
> at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
> at
> org.apache.wicket.proxy.LazyInitProxyFactory$AbstractCGLibInterceptor.intercept(LazyInitProxyFactory.java:364)
> at
> org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$EnhancerByCGLIB$$e2e8f0f.onMessage(<generated>)
> at
> org.apache.openmeetings.web.common.OmWebSocketPanel$2.onMessage(OmWebSocketPanel.java:125)
> at
> org.apache.wicket.protocol.ws.api.WebSocketBehavior.onEvent(WebSocketBehavior.java:64)
> at
> org.apache.wicket.settings.FrameworkSettings.dispatchEvent(FrameworkSettings.java:147)
> at
> org.apache.wicket.ComponentEventSender.dispatchToComponent(ComponentEventSender.java:292)
> at
> org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:329)
> at
> org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:306)
> at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:144)
> at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
> at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
> at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:123)
> at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:192)
> at
> org.apache.wicket.MarkupContainer.visitChildren(MarkupContainer.java:988)
> at
> org.apache.wicket.ComponentEventSender.breadth(ComponentEventSender.java:160)
> at
> org.apache.wicket.ComponentEventSender.send(ComponentEventSender.java:68)
> at org.apache.wicket.Component.send(Component.java:4418)
> at
> org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler$1.run(WebSocketMessageBroadcastHandler.java:74)
> at
> org.apache.wicket.protocol.ws.WebSocketSettings$SameThreadExecutor.run(WebSocketSettings.java:393)
> at
> org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler.respond(WebSocketMessageBroadcastHandler.java:97)
> at
> org.apache.wicket.request.cycle.RequestCycle$HandlerExecutor.respond(RequestCycle.java:917)
> at
> org.apache.wicket.request.RequestHandlerExecutor.execute(RequestHandlerExecutor.java:63)
> at
> org.apache.wicket.request.cycle.RequestCycle.execute(RequestCycle.java:283)
> at
> org.apache.wicket.request.cycle.RequestCycle.processRequest(RequestCycle.java:254)
> at
> org.apache.wicket.request.cycle.RequestCycle.processRequestAndDetach(RequestCycle.java:222)
> at
> org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.broadcastMessage(AbstractWebSocketProcessor.java:272)
> at
> org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.onMessage(AbstractWebSocketProcessor.java:151)
> at
> org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:65)
> at
> org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:60)
> at
> org.apache.tomcat.websocket.WsFrameBase.sendMessageText(WsFrameBase.java:395)
> at
> org.apache.tomcat.websocket.server.WsFrameServer.sendMessageText(WsFrameServer.java:119)
> at
> org.apache.tomcat.websocket.WsFrameBase.processDataText(WsFrameBase.java:495)
> at
> org.apache.tomcat.websocket.WsFrameBase.processData(WsFrameBase.java:294)
> at
> org.apache.tomcat.websocket.WsFrameBase.processInputBuffer(WsFrameBase.java:133)
> at
> org.apache.tomcat.websocket.server.WsFrameServer.onDataAvailable(WsFrameServer.java:82)
> at
> org.apache.tomcat.websocket.server.WsFrameServer.doOnDataAvailable(WsFrameServer.java:171)
> at
> org.apache.tomcat.websocket.server.WsFrameServer.notifyDataAvailable(WsFrameServer.java:151)
> at
> org.apache.tomcat.websocket.server.WsHttpUpgradeHandler.upgradeDispatch(WsHttpUpgradeHandler.java:148)
> at
> org.apache.coyote.http11.upgrade.UpgradeProcessorInternal.dispatch(UpgradeProcessorInternal.java:54)
> at
> org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:59)
> at
> org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
> at
> org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1590)
> at
> org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at
> org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
> at java.base/java.lang.Thread.run(Thread.java:834)
> {noformat}
> And right after one above multiple ones like this:
> {noformat}
> DEBUG 05-14 16:42:46.471 o.a.o.c.r.StreamProcessor:91 [-nio-443-exec-7] -
> Incoming message from user with ID '5':
> {"type":"kurento","id":"onIceCandidate","candidate":{"candidate":"candidate:505713252
> 1 udp 2122260223 10.1.1.67 43537 typ host generation 0 ufrag JMPb network-id
> 2 network-cost
> 10","sdpMid":"0","sdpMLineIndex":0},"uid":"eec0c0ef-10fe-4070-b40d-95bcec65a09a","luid":"eec0c0ef-10fe-4070-b40d-95bcec65a09a"}
> ERROR 05-14 16:42:46.471 o.a.o.w.c.OmWebSocketPanel:144 [-nio-443-exec-7] -
> Error while processing incoming message
> java.lang.NullPointerException: null
> at
> org.apache.openmeetings.core.remote.KStream.addCandidate(KStream.java:393)
> at
> org.apache.openmeetings.core.remote.StreamProcessor.onMessage(StreamProcessor.java:122)
> at
> org.apache.openmeetings.core.remote.KurentoHandler.onMessage(KurentoHandler.java:233)
> at
> org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$FastClassByCGLIB$$756e116d.invoke(<generated>)
> at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
> at
> org.apache.wicket.proxy.LazyInitProxyFactory$AbstractCGLibInterceptor.intercept(LazyInitProxyFactory.java:364)
> at
> org.apache.openmeetings.core.remote.Wicket_Proxy_KurentoHandler$$EnhancerByCGLIB$$e2e8f0f.onMessage(<generated>)
> at
> org.apache.openmeetings.web.common.OmWebSocketPanel$2.onMessage(OmWebSocketPanel.java:125)
> at
> org.apache.wicket.protocol.ws.api.WebSocketBehavior.onEvent(WebSocketBehavior.java:64)
> at
> org.apache.wicket.settings.FrameworkSettings.dispatchEvent(FrameworkSettings.java:147)
> at
> org.apache.wicket.ComponentEventSender.dispatchToComponent(ComponentEventSender.java:292)
> at
> org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:329)
> at
> org.apache.wicket.ComponentEventSender$ComponentEventVisitor.component(ComponentEventSender.java:306)
> at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:144)
> at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
> at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:162)
> at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:123)
> at org.apache.wicket.util.visit.Visits.visitChildren(Visits.java:192)
> at
> org.apache.wicket.MarkupContainer.visitChildren(MarkupContainer.java:988)
> at
> org.apache.wicket.ComponentEventSender.breadth(ComponentEventSender.java:160)
> at
> org.apache.wicket.ComponentEventSender.send(ComponentEventSender.java:68)
> at org.apache.wicket.Component.send(Component.java:4418)
> at
> org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler$1.run(WebSocketMessageBroadcastHandler.java:74)
> at
> org.apache.wicket.protocol.ws.WebSocketSettings$SameThreadExecutor.run(WebSocketSettings.java:393)
> at
> org.apache.wicket.protocol.ws.api.WebSocketMessageBroadcastHandler.respond(WebSocketMessageBroadcastHandler.java:97)
> at
> org.apache.wicket.request.cycle.RequestCycle$HandlerExecutor.respond(RequestCycle.java:917)
> at
> org.apache.wicket.request.RequestHandlerExecutor.execute(RequestHandlerExecutor.java:63)
> at
> org.apache.wicket.request.cycle.RequestCycle.execute(RequestCycle.java:283)
> at
> org.apache.wicket.request.cycle.RequestCycle.processRequest(RequestCycle.java:254)
> at
> org.apache.wicket.request.cycle.RequestCycle.processRequestAndDetach(RequestCycle.java:222)
> at
> org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.broadcastMessage(AbstractWebSocketProcessor.java:272)
> at
> org.apache.wicket.protocol.ws.api.AbstractWebSocketProcessor.onMessage(AbstractWebSocketProcessor.java:151)
> at
> org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:65)
> at
> org.apache.wicket.protocol.ws.javax.JavaxWebSocketProcessor$StringMessageHandler.onMessage(JavaxWebSocketProcessor.java:60)
> at
> org.apache.tomcat.websocket.WsFrameBase.sendMessageText(WsFrameBase.java:395)
> at
> org.apache.tomcat.websocket.server.WsFrameServer.sendMessageText(WsFrameServer.java:119)
> at
> org.apache.tomcat.websocket.WsFrameBase.processDataText(WsFrameBase.java:495)
> at
> org.apache.tomcat.websocket.WsFrameBase.processData(WsFrameBase.java:294)
> at
> org.apache.tomcat.websocket.WsFrameBase.processInputBuffer(WsFrameBase.java:133)
> at
> org.apache.tomcat.websocket.server.WsFrameServer.onDataAvailable(WsFrameServer.java:82)
> at
> org.apache.tomcat.websocket.server.WsFrameServer.doOnDataAvailable(WsFrameServer.java:171)
> at
> org.apache.tomcat.websocket.server.WsFrameServer.notifyDataAvailable(WsFrameServer.java:151)
> at
> org.apache.tomcat.websocket.server.WsHttpUpgradeHandler.upgradeDispatch(WsHttpUpgradeHandler.java:148)
> at
> org.apache.coyote.http11.upgrade.UpgradeProcessorInternal.dispatch(UpgradeProcessorInternal.java:54)
> at
> org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:59)
> at
> org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
> at
> org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1590)
> at
> org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at
> org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
> at java.base/java.lang.Thread.run(Thread.java:834)
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
