This is an automated email from the ASF dual-hosted git repository. solomax pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/openmeetings.git
The following commit(s) were added to refs/heads/master by this push: new f48bff6 [OPENMEETINGS-2424] audio/video stream is better cleaned f48bff6 is described below commit f48bff6dca8ad62934462bde08975642373fc569 Author: Maxim Solodovnik <solomax...@gmail.com> AuthorDate: Sun Sep 6 12:08:18 2020 +0700 [OPENMEETINGS-2424] audio/video stream is better cleaned --- .../apache/openmeetings/core/remote/KStream.java | 3 + openmeetings-web/pom.xml | 1 + .../web/room/activities/ActivitiesPanel.java | 8 - .../activities.js => raw-activities.js} | 0 .../org/apache/openmeetings/web/room/raw-room.js | 6 + .../apache/openmeetings/web/room/raw-settings.js | 14 +- .../openmeetings/web/room/raw-video-manager.js | 16 +- .../apache/openmeetings/web/room/raw-video-util.js | 50 +++-- .../org/apache/openmeetings/web/room/raw-video.js | 213 +++++++++++++-------- .../openmeetings/webservice/NetTestWebService.java | 15 +- 10 files changed, 198 insertions(+), 128 deletions(-) diff --git a/openmeetings-core/src/main/java/org/apache/openmeetings/core/remote/KStream.java b/openmeetings-core/src/main/java/org/apache/openmeetings/core/remote/KStream.java index 17fd834..77a5e2d 100644 --- a/openmeetings-core/src/main/java/org/apache/openmeetings/core/remote/KStream.java +++ b/openmeetings-core/src/main/java/org/apache/openmeetings/core/remote/KStream.java @@ -390,6 +390,9 @@ public class KStream extends AbstractStream { public void addCandidate(IceCandidate candidate, String uid) { if (this.uid.equals(uid)) { + if (outgoingMedia == null) { + return; + } outgoingMedia.addIceCandidate(candidate); } else { WebRtcEndpoint endpoint = listeners.get(uid); diff --git a/openmeetings-web/pom.xml b/openmeetings-web/pom.xml index d72d8cf..e192aa6 100644 --- a/openmeetings-web/pom.xml +++ b/openmeetings-web/pom.xml @@ -202,6 +202,7 @@ <jsSourceDir>../java/org/apache/openmeetings/web/room</jsSourceDir> <jsSourceFiles> <jsSourceFile>NoSleep.js</jsSourceFile> + <jsSourceFile>raw-activities.js</jsSourceFile> <jsSourceFile>raw-video.js</jsSourceFile> <jsSourceFile>raw-video-manager.js</jsSourceFile> <jsSourceFile>raw-sharer.js</jsSourceFile> diff --git a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/activities/ActivitiesPanel.java b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/activities/ActivitiesPanel.java index 41c9b41..bdc567f 100644 --- a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/activities/ActivitiesPanel.java +++ b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/activities/ActivitiesPanel.java @@ -40,10 +40,8 @@ import org.apache.wicket.ajax.AbstractDefaultAjaxBehavior; import org.apache.wicket.ajax.AjaxRequestTarget; import org.apache.wicket.core.request.handler.IPartialPageRequestHandler; import org.apache.wicket.markup.head.IHeaderResponse; -import org.apache.wicket.markup.head.JavaScriptHeaderItem; import org.apache.wicket.markup.head.PriorityHeaderItem; import org.apache.wicket.markup.html.panel.Panel; -import org.apache.wicket.request.resource.JavaScriptResourceReference; import org.apache.wicket.spring.injection.annot.SpringBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -260,12 +258,6 @@ public class ActivitiesPanel extends Panel { handler.appendJavaScript(String.format("Activities.remove(%s);", arr)); } - @Override - public void renderHead(IHeaderResponse response) { - super.renderHead(response); - response.render(new PriorityHeaderItem(JavaScriptHeaderItem.forReference(new JavaScriptResourceReference(ActivitiesPanel.class, "activities.js")))); - } - private static CharSequence getClass(Activity a) { StringBuilder cls = new StringBuilder(); switch (a.getType()) { diff --git a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/activities/activities.js b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-activities.js similarity index 100% rename from openmeetings-web/src/main/java/org/apache/openmeetings/web/room/activities/activities.js rename to openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-activities.js diff --git a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-room.js b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-room.js index 6e5297f..4d8a7c4 100644 --- a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-room.js +++ b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-room.js @@ -372,6 +372,9 @@ var Room = (function() { }).appendTo(container); } function _addClient(_clients) { + if (!options) { + return; //too early + } const clients = Array.isArray(_clients) ? _clients : [_clients]; clients.forEach(c => { const self = c.uid === options.uid; @@ -392,6 +395,9 @@ var Room = (function() { __sortUserList(); } function _updateClient(c) { + if (!options) { + return; //too early + } const self = c.uid === options.uid , le = Room.getClient(c.uid) , hasAudio = VideoUtil.hasMic(c) diff --git a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-settings.js b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-settings.js index 4ad895d..72fb92c 100644 --- a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-settings.js +++ b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-settings.js @@ -307,7 +307,7 @@ var VideoSettings = (function() { options , function(error) { if (error) { - if (true === this.cleaned) { + if (true === rtcPeer.cleaned) { return; } return OmUtil.error(error); @@ -321,7 +321,7 @@ var VideoSettings = (function() { } rtcPeer.generateOffer(function(error, _offerSdp) { if (error) { - if (true === this.cleaned) { + if (true === rtcPeer.cleaned) { return; } return OmUtil.error('Error generating the offer'); @@ -496,14 +496,14 @@ var VideoSettings = (function() { options , function(error) { if (error) { - if (true === this.cleaned) { + if (true === rtcPeer.cleaned) { return; } return OmUtil.error(error); } rtcPeer.generateOffer(function(error, offerSdp) { if (error) { - if (true === this.cleaned) { + if (true === rtcPeer.cleaned) { return; } return OmUtil.error('Error generating the offer'); @@ -520,7 +520,7 @@ var VideoSettings = (function() { OmUtil.log('Play SDP answer received from server. Processing ...'); rtcPeer.processAnswer(m.sdpAnswer, function(error) { if (error) { - if (true === this.cleaned) { + if (true === rtcPeer.cleaned) { return; } return OmUtil.error(error); @@ -534,7 +534,7 @@ var VideoSettings = (function() { OmUtil.log('SDP answer received from server. Processing ...'); rtcPeer.processAnswer(m.sdpAnswer, function(error) { if (error) { - if (true === this.cleaned) { + if (true === rtcPeer.cleaned) { return; } return OmUtil.error(error); @@ -544,7 +544,7 @@ var VideoSettings = (function() { case 'iceCandidate': rtcPeer.addIceCandidate(m.candidate, function(error) { if (error) { - if (true === this.cleaned) { + if (true === rtcPeer.cleaned) { return; } return OmUtil.error('Error adding candidate: ' + error); diff --git a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video-manager.js b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video-manager.js index 2e49eed..b456652 100644 --- a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video-manager.js +++ b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video-manager.js @@ -8,12 +8,12 @@ var VideoManager = (function() { , v = w.data() , peer = v && v.getPeer(); - if (peer) { + if (peer && false === peer.cleaned) { peer.processAnswer(m.sdpAnswer, function (error) { + if (true === peer.cleaned) { + return; + } if (error) { - if (true === this.cleaned) { - return; - } return OmUtil.error(error); } const vidEls = w.find('audio, video') @@ -85,12 +85,12 @@ var VideoManager = (function() { , v = w.data() , peer = v && v.getPeer(); - if (peer) { + if (peer && false === peer.cleaned) { peer.addIceCandidate(m.candidate, function (error) { + if (true === this.cleaned) { + return; + } if (error) { - if (true === this.cleaned) { - return; - } OmUtil.error('Error adding candidate: ' + error); return; } diff --git a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video-util.js b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video-util.js index 91a7549..92ff8f9 100644 --- a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video-util.js +++ b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video-util.js @@ -175,34 +175,52 @@ var VideoUtil = (function() { } function _cleanStream(stream) { if (!!stream) { - stream.getTracks().forEach(function(track) { - try { - track.stop(); - } catch(e) { - //no-op - } - }); + stream.getTracks().forEach(track => track.stop()); } } function _cleanPeer(peer) { if (!!peer) { peer.cleaned = true; - const pc = peer.peerConnection; try { - if (!!pc && !!pc.getLocalStreams()) { - pc.getLocalStreams().forEach(function(stream) { - _cleanStream(stream); + const pc = peer.peerConnection; + if (!!pc) { + pc.getSenders().forEach(sender => { + try { + if (sender.track) { + sender.track.stop(); + } + } catch(e) { + OmUtil.log('Failed to clean sender' + e); + } + }); + pc.getReceivers().forEach(receiver => { + try { + if (receiver.track) { + receiver.track.stop(); + } + } catch(e) { + OmUtil.log('Failed to clean receiver' + e); + } }); + pc.onconnectionstatechange = null; + pc.ontrack = null; + pc.onremovetrack = null; + pc.onremovestream = null; + pc.onicecandidate = null; + pc.oniceconnectionstatechange = null; + pc.onsignalingstatechange = null; + pc.onicegatheringstatechange = null; + pc.onnegotiationneeded = null; } - } catch(e) { - OmUtil.log('Failed to clean peer' + e); - } - try { peer.dispose(); + peer.removeAllListeners('icecandidate'); + delete peer.generateOffer; + delete peer.processAnswer; + delete peer.processOffer; + delete peer.addIceCandidate; } catch(e) { //no-op } - peer = null; } } function _isChrome(_b) { diff --git a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video.js b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video.js index f350d5a..5cf51f3 100644 --- a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video.js +++ b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/raw-video.js @@ -1,8 +1,8 @@ /* Licensed under the Apache License, Version 2.0 (the "License") http://www.apache.org/licenses/LICENSE-2.0 */ var Video = (function() { - const self = {} + const self = {}, states = [] , AudioCtx = window.AudioContext || window.webkitAudioContext; - let sd, v, vc, t, footer, size, vol, video, iceServers + let sd, v, vc, t, footer, size, vol, iceServers , lm, level, userSpeaks = false, muteOthers , hasVideo, isSharing, isRecording; @@ -19,7 +19,7 @@ var Video = (function() { OmUtil.sendMessage({type: 'mic', id: 'activity', active: speaks}); } } - function _getScreenStream(msg, callback) { + function _getScreenStream(msg, state, callback) { function __handleScreenError(err) { VideoManager.sendMessage({id: 'errorSharing'}); Sharer.setShareState(SHARE_STOPPED); @@ -50,11 +50,14 @@ var Video = (function() { }); } promise.then(function(stream) { - __createVideo(); - callback(msg, cnts, stream); + if (!state.disposed) { + __createVideo(state); + state.stream = stream; + callback(msg, state, cnts); + } }).catch(__handleScreenError); } - function _getVideoStream(msg, callback) { + function _getVideoStream(msg, state, callback) { VideoSettings.constraints(sd, function(cnts) { if ((VideoUtil.hasCam(sd) && !cnts.video) || (VideoUtil.hasMic(sd) && !cnts.audio)) { VideoManager.sendMessage({ @@ -71,7 +74,7 @@ var Video = (function() { } navigator.mediaDevices.getUserMedia(cnts) .then(function(stream) { - if (msg.instanceUid !== v.data('instance-uid')) { + if (state.disposed || msg.instanceUid !== v.data('instance-uid')) { return; } let _stream = stream; @@ -97,8 +100,10 @@ var Video = (function() { }); } } - __createVideo(data); - callback(msg, cnts, _stream); + state.data = data; + __createVideo(state); + state.stream = _stream; + callback(msg, state, cnts); }) .catch(function(err) { VideoManager.sendMessage({ @@ -116,9 +121,9 @@ var Video = (function() { }); }); } - function __attachListener(rtcPeer) { - if (rtcPeer) { - const pc = rtcPeer.peerConnection; + function __attachListener(state) { + if (!state.disposed && state.data.rtcPeer) { + const pc = state.data.rtcPeer.peerConnection; pc.onconnectionstatechange = function(event) { console.warn(`!!RTCPeerConnection state changed: ${pc.connectionState}, user: ${sd.user.displayName}, uid: ${sd.uid}`); switch(pc.connectionState) { @@ -141,20 +146,20 @@ var Video = (function() { } } } - function __createSendPeer(msg, cnts, stream) { - const options = { - videoStream: stream + function __createSendPeer(msg, state, cnts) { + state.options = { + videoStream: state.stream , mediaConstraints: cnts , onicecandidate: self.onIceCandidate }; if (!isSharing) { - options.localVideo = video[0]; + state.options.localVideo = state.video[0]; } - const data = video.data(); + const data = state.data; data.rtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerSendonly( - VideoUtil.addIceServers(options, msg) + VideoUtil.addIceServers(state.options, msg) , function (error) { - if (true === this.cleaned) { + if (state.disposed || true === data.rtcPeer.cleaned) { return; } if (error) { @@ -164,8 +169,8 @@ var Video = (function() { level = MicLevel(); level.meter(data.analyser, lm, _micActivity, OmUtil.error); } - this.generateOffer(function(error, offerSdp) { - if (true === this.cleaned) { + data.rtcPeer.generateOffer(function(error, offerSdp) { + if (state.disposed || true === data.rtcPeer.cleaned) { return; } if (error) { @@ -185,33 +190,34 @@ var Video = (function() { } }); }); - __attachListener(data.rtcPeer); + data.rtcPeer.cleaned = false; + __attachListener(state); } - function _createSendPeer(msg) { + function _createSendPeer(msg, state) { if (isSharing || isRecording) { - _getScreenStream(msg, __createSendPeer); + _getScreenStream(msg, state, __createSendPeer); } else { - _getVideoStream(msg, __createSendPeer); + _getVideoStream(msg, state, __createSendPeer); } } - function _createResvPeer(msg) { - __createVideo(); + function _createResvPeer(msg, state) { + __createVideo(state); const options = VideoUtil.addIceServers({ - remoteVideo : video[0] + remoteVideo : state.video[0] , onicecandidate : self.onIceCandidate }, msg); - const data = video.data(); + const data = state.data; data.rtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerRecvonly( options , function(error) { - if (true === this.cleaned) { + if (state.disposed || true === data.rtcPeer.cleaned) { return; } if (error) { return OmUtil.error(error); } - this.generateOffer(function(error, offerSdp) { - if (true === this.cleaned) { + data.rtcPeer.generateOffer(function(error, offerSdp) { + if (state.disposed || true === data.rtcPeer.cleaned) { return; } if (error) { @@ -225,7 +231,8 @@ var Video = (function() { }); }); }); - __attachListener(data.rtcPeer); + data.rtcPeer.cleaned = false; + __attachListener(state); } function _handleMicStatus(state) { if (!footer || !footer.is(':visible')) { @@ -403,27 +410,27 @@ var Video = (function() { _init({stream: sd, iceServers: iceServers}); } } - function __createVideo(data) { + function __createVideo(state) { const _id = VideoUtil.getVid(sd.uid); _resizeDlgArea(size.width, size.height); if (hasVideo && !isSharing && !isRecording) { VideoUtil.setPos(v, VideoUtil.getPos(VideoUtil.getRects(VIDWIN_SEL), sd.width, sd.height + 25)); } - video = $(hasVideo ? '<video>' : '<audio>').attr('id', 'vid' + _id) + state.video = $(hasVideo ? '<video>' : '<audio>').attr('id', 'vid' + _id) .attr('playsinline', 'playsinline') .width(vc.width()).height(vc.height()) .prop('autoplay', true).prop('controls', false); - if (data) { - video.data(data); + if (state.data) { + state.video.data(state.data); } if (hasVideo) { vc.removeClass('audio-only').css('background-image', '');; vc.parents('.ui-dialog').removeClass('audio-only'); - video.attr('poster', sd.user.pictureUri); + state.video.attr('poster', sd.user.pictureUri); } else { vc.addClass('audio-only'); } - vc.append(video); + vc.append(state.video); if (VideoUtil.hasMic(sd)) { const volIco = vol.create(self) if (hasVideo) { @@ -439,12 +446,17 @@ var Video = (function() { function _refresh(_msg) { const msg = _msg || {iceServers: iceServers}; _cleanup(); - const hasAudio = VideoUtil.hasMic(sd); + const hasAudio = VideoUtil.hasMic(sd) + , state = { + disposed: false + , data: {} + }; + states.push(state); if (sd.self) { - _createSendPeer(msg); + _createSendPeer(msg, state); _handleMicStatus(hasAudio); } else { - _createResvPeer(msg); + _createResvPeer(msg, state); } } function _setRights() { @@ -456,43 +468,73 @@ var Video = (function() { muteOthers.removeClass('enabled').off(); } } - function _cleanup() { - OmUtil.log('Disposing participant ' + sd.uid); - if (video && video.length > 0) { - const data = video.data(); - if (data.analyser) { - VideoUtil.disconnect(data.analyser); - data.analyser = null; - } - if (data.gainNode) { - VideoUtil.disconnect(data.gainNode); - data.gainNode = null; + function _cleanData(data) { + if (!data) { + return; + } + if (data.analyser) { + VideoUtil.disconnect(data.analyser); + data.analyser = null; + } + if (data.gainNode) { + VideoUtil.disconnect(data.gainNode); + data.gainNode = null; + } + if (data.aSrc) { + VideoUtil.cleanStream(data.aSrc.mediaStream); + VideoUtil.cleanStream(data.aSrc.origStream); + VideoUtil.disconnect(data.aSrc); + data.aSrc = null; + } + if (data.aDest) { + VideoUtil.disconnect(data.aDest); + data.aDest = null; + } + if (data.aCtx) { + if (data.aCtx.destination) { + VideoUtil.disconnect(data.aCtx.destination); } - if (data.aSrc) { - VideoUtil.cleanStream(data.aSrc.mediaStream); - VideoUtil.cleanStream(data.aSrc.origStream); - VideoUtil.disconnect(data.aSrc); - data.aSrc = null; + if ('closed' !== data.aCtx.state) { + try { + data.aCtx.close(); + } catch(e) { + console.error(e); + } } - if (data.aDest) { - VideoUtil.disconnect(data.aDest); - data.aDest = null; + data.aCtx = null; + } + VideoUtil.cleanPeer(data.rtcPeer); + data.rtcPeer = null; + } + function _cleanup(evt) { + OmUtil.log('!!Disposing participant ' + sd.uid); + let state; + while(state = states.pop()) { + state.disposed = true; + if (state.options) { + delete state.options.videoStream; + delete state.options.mediaConstraints; + delete state.options.onicecandidate; + delete state.options.localVideo; + state.options = null; } - if (data.aCtx) { - if (data.aCtx.destination) { - VideoUtil.disconnect(data.aCtx.destination); - } - data.aCtx.close(); - data.aCtx = null; + _cleanData(state.data); + VideoUtil.cleanStream(state.stream); + state.data = null; + state.stream = null; + const video = state.video; + if (video && video.length > 0) { + video.attr('id', 'dummy'); + const vidNode = video[0]; + VideoUtil.cleanStream(vidNode.srcObject); + vidNode.srcObject = null; + vidNode.load(); + vidNode.removeAttribute("src"); + vidNode.removeAttribute("srcObject"); + vidNode.parentNode.removeChild(vidNode); + state.video.data({}); + state.video = null; } - video.attr('id', 'dummy'); - const vidNode = video[0]; - VideoUtil.cleanStream(vidNode.srcObject); - vidNode.srcObject = null; - vidNode.parentNode.removeChild(vidNode); - - VideoUtil.cleanPeer(data.rtcPeer); - video = null; } if (lm && lm.length > 0) { _micActivity(false); @@ -505,14 +547,19 @@ var Video = (function() { } vc.find('audio,video').remove(); vol.destroy(); + if (evt && evt.target) { + $(evt).off(); + } } function _reattachStream() { - if (video && video.length > 0) { - const data = video.data(); - if (data.rtcPeer) { - video[0].srcObject = sd.self ? data.rtcPeer.getLocalStream() : data.rtcPeer.getRemoteStream(); + states.forEach(state => { + if (state.video && state.video.length > 0) { + const data = state.data; + if (data.rtcPeer) { + state.video[0].srcObject = sd.self ? data.rtcPeer.getLocalStream() : data.rtcPeer.getRemoteStream(); + } } - } + }); } self.update = _update; @@ -526,7 +573,9 @@ var Video = (function() { self.init = _init; self.stream = function() { return sd; }; self.setRights = _setRights; - self.getPeer = function() { return video ? video.data().rtcPeer : null; }; + self.getPeer = function() { + return states.length > 0 ? states[0].data.rtcPeer : null; + }; self.onIceCandidate = function(candidate) { const opts = Room.getOptions(); OmUtil.log('Local candidate ' + JSON.stringify(candidate)); @@ -539,7 +588,7 @@ var Video = (function() { }; self.reattachStream = _reattachStream; self.video = function() { - return video; + return states.length > 0 ? states[0].video : null; }; self.handleMicStatus = _handleMicStatus; return self; diff --git a/openmeetings-webservice/src/main/java/org/apache/openmeetings/webservice/NetTestWebService.java b/openmeetings-webservice/src/main/java/org/apache/openmeetings/webservice/NetTestWebService.java index a2de47c..67da389 100644 --- a/openmeetings-webservice/src/main/java/org/apache/openmeetings/webservice/NetTestWebService.java +++ b/openmeetings-webservice/src/main/java/org/apache/openmeetings/webservice/NetTestWebService.java @@ -19,9 +19,9 @@ package org.apache.openmeetings.webservice; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.ThreadLocalRandom; import javax.ws.rs.Consumes; import javax.ws.rs.GET; @@ -32,9 +32,10 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.ResponseBuilder; -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.ThreadLocalRandom; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; @Service("netTestWebService") @Path("/networktest") @@ -58,7 +59,7 @@ public class NetTestWebService { public Response get(@QueryParam("type") String type, @QueryParam("size") int _size) { final int size; TestType testType = getTypeByString(type); - log.debug("Network test:: get"); + log.debug("Network test:: get, {}, {}", testType, _size); // choose data to send switch (testType) {