- Revision
- 271401
- Author
- [email protected]
- Date
- 2021-01-12 06:34:28 -0800 (Tue, 12 Jan 2021)
Log Message
Unmuting a track in case of end of interruption by another web process tab is not working well with capture muting icons
https://bugs.webkit.org/show_bug.cgi?id=220058
Reviewed by Eric Carlson.
Source/WebCore:
Before the patch, we were muting all tracks in case of Audio Session interruption.
This works well in case another application triggers the interruption.
In case the interruption is done by another tab, this does not work well for two reasons:
- The UIProcess is doing the work of muting the tracks when another tab is starting to capture
- Unmuting in case of interruption might unmute several tabs if they are in the same process
As an example, if a tab is capturing, and another tab starts to capture, going back to first tab will automatically
restart the audio capture as the Audio Session will end its interruption. But it will not mute the camera track.
To fix that, the Audio Shared Unit is no longer directly muting or unmuting the audio tracks.
Instead, each page is notified of start/end of AudioSession interruption.
When an interruption starts, all capture tracks are muted.
When an interruption ends, all capture tracks are updated according the page muted state, which is controlled by UIProcess.
This also covers the case of several tabs capturing in the same process, since only one tab will not be set as muted by UIProcess.
In case of several capture MediaStreamTrack in the same document and they are not the active source, we pick the first one as there is
no way to know which one was the last unmuted one.
We should probably add some support to always remember the last living track of a tab in iOS.
Manually tested.
* Modules/mediastream/MediaStreamTrack.cpp:
(WebCore::MediaStreamTrack::create):
(WebCore::MediaStreamTrack::updateCaptureAccordingToMutedState):
(WebCore::MediaStreamTrack::updateToPageMutedState):
* Modules/mediastream/MediaStreamTrack.h:
* dom/Document.cpp:
(WebCore::Document::visibilityStateChanged):
(WebCore::Document::pageMutedStateDidChange):
* page/Page.cpp:
(WebCore::Page::beginAudioCaptureInterruption):
(WebCore::Page::endAudioCaptureInterruption):
* page/Page.h:
* platform/mediastream/mac/BaseAudioSharedUnit.cpp:
(WebCore::BaseAudioSharedUnit::resume):
(WebCore::BaseAudioSharedUnit::suspend):
Source/WebKit:
Add an observer of AudioSession interruptions that forwards the signal to all tabs of the process.
* WebProcess/cocoa/WebProcessCocoa.mm:
(WebKit::WebProcess::platformInitializeWebProcess):
(WebKit::PageAudioSessionInterruptionObserver::PageAudioSessionInterruptionObserver):
(WebKit::PageAudioSessionInterruptionObserver::~PageAudioSessionInterruptionObserver):
(WebKit::PageAudioSessionInterruptionObserver::beginAudioSessionInterruption):
(WebKit::PageAudioSessionInterruptionObserver::endAudioSessionInterruption):
(WebKit::listenToAudioSessionInterruption):
Modified Paths
Diff
Modified: trunk/LayoutTests/fast/mediastream/media-stream-track-interrupted.html (271400 => 271401)
--- trunk/LayoutTests/fast/mediastream/media-stream-track-interrupted.html 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/LayoutTests/fast/mediastream/media-stream-track-interrupted.html 2021-01-12 14:34:28 UTC (rev 271401)
@@ -50,8 +50,11 @@
function testTrack(track, title)
{
- promise_test((test) => {
- return new Promise((resolve, reject) => {
+ promise_test(async (test) => {
+ if (track.muted)
+ await new Promise(resolve => track._onunmute_ = resolve);
+
+ await new Promise((resolve, reject) => {
let isVideo = track.kind == "video";
if (window.internals) {
assert_false(internals.pageMediaState().includes('HasMutedVideoCaptureDevice'));
@@ -67,12 +70,9 @@
track._onunmute_ = (evt) => {
waitForPageStateChange(10, pageState, resolve, reject)
}
-
if (window.internals) {
- assert_true(pageMediaState.includes(isVideo ? 'HasMutedVideoCaptureDevice' : 'HasMutedAudioCaptureDevice'));
- assert_false(pageMediaState.includes(isVideo ? 'HasMutedAudioCaptureDevice' : 'HasMutedVideoCaptureDevice'));
- assert_true(pageMediaState.includes(isVideo ? 'HasActiveAudioCaptureDevice' : 'HasActiveVideoCaptureDevice'));
- assert_false(pageMediaState.includes(isVideo ? 'HasActiveVideoCaptureDevice' : 'HasActiveAudioCaptureDevice'));
+ assert_true(pageMediaState.includes('HasMutedAudioCaptureDevice'), 'audio muted');
+ assert_true(pageMediaState.includes('HasMutedVideoCaptureDevice'), 'video muted');
pageState = internals.pageMediaState();
internals.setMediaStreamSourceInterrupted(track, false)
}
@@ -85,6 +85,8 @@
}
setTimeout(() => reject("Muted state did not change in 1 second"), 1000);
});
+ track._onmute_ = () => { };
+ track._onunmute_ = () => { };
}, title);
}
Modified: trunk/Source/WebCore/ChangeLog (271400 => 271401)
--- trunk/Source/WebCore/ChangeLog 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebCore/ChangeLog 2021-01-12 14:34:28 UTC (rev 271401)
@@ -1,3 +1,47 @@
+2021-01-12 Youenn Fablet <[email protected]>
+
+ Unmuting a track in case of end of interruption by another web process tab is not working well with capture muting icons
+ https://bugs.webkit.org/show_bug.cgi?id=220058
+
+ Reviewed by Eric Carlson.
+
+ Before the patch, we were muting all tracks in case of Audio Session interruption.
+ This works well in case another application triggers the interruption.
+
+ In case the interruption is done by another tab, this does not work well for two reasons:
+ - The UIProcess is doing the work of muting the tracks when another tab is starting to capture
+ - Unmuting in case of interruption might unmute several tabs if they are in the same process
+ As an example, if a tab is capturing, and another tab starts to capture, going back to first tab will automatically
+ restart the audio capture as the Audio Session will end its interruption. But it will not mute the camera track.
+
+ To fix that, the Audio Shared Unit is no longer directly muting or unmuting the audio tracks.
+ Instead, each page is notified of start/end of AudioSession interruption.
+ When an interruption starts, all capture tracks are muted.
+ When an interruption ends, all capture tracks are updated according the page muted state, which is controlled by UIProcess.
+ This also covers the case of several tabs capturing in the same process, since only one tab will not be set as muted by UIProcess.
+
+ In case of several capture MediaStreamTrack in the same document and they are not the active source, we pick the first one as there is
+ no way to know which one was the last unmuted one.
+ We should probably add some support to always remember the last living track of a tab in iOS.
+
+ Manually tested.
+
+ * Modules/mediastream/MediaStreamTrack.cpp:
+ (WebCore::MediaStreamTrack::create):
+ (WebCore::MediaStreamTrack::updateCaptureAccordingToMutedState):
+ (WebCore::MediaStreamTrack::updateToPageMutedState):
+ * Modules/mediastream/MediaStreamTrack.h:
+ * dom/Document.cpp:
+ (WebCore::Document::visibilityStateChanged):
+ (WebCore::Document::pageMutedStateDidChange):
+ * page/Page.cpp:
+ (WebCore::Page::beginAudioCaptureInterruption):
+ (WebCore::Page::endAudioCaptureInterruption):
+ * page/Page.h:
+ * platform/mediastream/mac/BaseAudioSharedUnit.cpp:
+ (WebCore::BaseAudioSharedUnit::resume):
+ (WebCore::BaseAudioSharedUnit::suspend):
+
2021-01-12 Philippe Normand <[email protected]>
[GStreamer] Bump version requirement
Modified: trunk/Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp (271400 => 271401)
--- trunk/Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp 2021-01-12 14:34:28 UTC (rev 271401)
@@ -71,8 +71,10 @@
auto track = adoptRef(*new MediaStreamTrack(context, WTFMove(privateTrack)));
track->suspendIfNeeded();
- if (track->isCaptureTrack())
- track->updateToPageMutedState();
+ if (track->isCaptureTrack()) {
+ if (auto* page = track->document()->page())
+ track->updateToPageMutedState(page->mutedState());
+ }
return track;
}
@@ -469,7 +471,7 @@
}
#endif
-void MediaStreamTrack::updateCaptureAccordingToMutedState(Document& document)
+void MediaStreamTrack::updateCaptureAccordingToMutedState(Document& document, MediaProducer::MutedStateFlags mutedState)
{
#if PLATFORM(IOS_FAMILY)
auto* page = document.page();
@@ -476,40 +478,50 @@
if (!page)
return;
+ bool pageMuted = mutedState & MediaProducer::AudioAndVideoCaptureIsMuted;
+
+ auto updateTracksAccordingPageMutedState = [](const Document& document, CaptureDevice::DeviceType deviceType, bool pageMuted) {
+ // We can only have one source at a time: we can mute all tracks but unmute only one of them.
+ for (auto* captureTrack : allCaptureTracks()) {
+ if (captureTrack->document() != &document || captureTrack->ended() || captureTrack->source().deviceType() != deviceType)
+ continue;
+ captureTrack->m_private->setMuted(pageMuted);
+ // If unmuting, unmute the first source of the document we know.
+ if (!pageMuted)
+ break;
+ }
+ };
auto* activeAudioSource = RealtimeMediaSourceCenter::singleton().audioCaptureFactory().activeSource();
- if (activeAudioSource && isSourceCapturingForTrackInDocument(*activeAudioSource, document)) {
- bool pageMuted = page->mutedState() & MediaProducer::AudioAndVideoCaptureIsMuted;
+ if (activeAudioSource && isSourceCapturingForTrackInDocument(*activeAudioSource, document))
activeAudioSource->setMuted(pageMuted || (document.hidden() && document.settings().interruptAudioOnPageVisibilityChangeEnabled()));
- }
+ else
+ updateTracksAccordingPageMutedState(document, CaptureDevice::DeviceType::Microphone, pageMuted);
auto* activeVideoSource = RealtimeMediaSourceCenter::singleton().videoCaptureFactory().activeSource();
- if (activeVideoSource && isSourceCapturingForTrackInDocument(*activeVideoSource, document)) {
- bool pageMuted = page->mutedState() & MediaProducer::AudioAndVideoCaptureIsMuted;
+ if (activeVideoSource && isSourceCapturingForTrackInDocument(*activeVideoSource, document))
activeVideoSource->setMuted(pageMuted || document.hidden());
- }
+ else
+ updateTracksAccordingPageMutedState(document, CaptureDevice::DeviceType::Camera, pageMuted);
#else
for (auto* captureTrack : allCaptureTracks()) {
if (captureTrack->document() == &document && !captureTrack->ended())
- captureTrack->updateToPageMutedState();
+ captureTrack->updateToPageMutedState(mutedState);
}
#endif
}
-void MediaStreamTrack::updateToPageMutedState()
+void MediaStreamTrack::updateToPageMutedState(MediaProducer::MutedStateFlags mutedState)
{
ASSERT(isCaptureTrack());
- auto* page = document()->page();
- if (!page)
- return;
switch (source().deviceType()) {
case CaptureDevice::DeviceType::Microphone:
case CaptureDevice::DeviceType::Camera:
- m_private->setMuted(page->mutedState() & MediaProducer::AudioAndVideoCaptureIsMuted);
+ m_private->setMuted(mutedState & MediaProducer::AudioAndVideoCaptureIsMuted);
break;
case CaptureDevice::DeviceType::Screen:
case CaptureDevice::DeviceType::Window:
- m_private->setMuted(page->mutedState() & MediaProducer::ScreenCaptureIsMuted);
+ m_private->setMuted(mutedState & MediaProducer::ScreenCaptureIsMuted);
break;
case CaptureDevice::DeviceType::Speaker:
case CaptureDevice::DeviceType::Unknown:
Modified: trunk/Source/WebCore/Modules/mediastream/MediaStreamTrack.h (271400 => 271401)
--- trunk/Source/WebCore/Modules/mediastream/MediaStreamTrack.h 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebCore/Modules/mediastream/MediaStreamTrack.h 2021-01-12 14:34:28 UTC (rev 271401)
@@ -72,7 +72,7 @@
static void endCapture(Document&);
static MediaProducer::MediaStateFlags captureState(Document&);
- static void updateCaptureAccordingToMutedState(Document&);
+ static void updateCaptureAccordingToMutedState(Document&, MediaProducer::MutedStateFlags);
virtual bool isCanvas() const { return false; }
@@ -169,7 +169,7 @@
explicit MediaStreamTrack(MediaStreamTrack&);
void configureTrackRendering();
- void updateToPageMutedState();
+ void updateToPageMutedState(MediaProducer::MutedStateFlags);
// ActiveDOMObject API.
void stop() final { stopTrack(); }
Modified: trunk/Source/WebCore/dom/Document.cpp (271400 => 271401)
--- trunk/Source/WebCore/dom/Document.cpp 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebCore/dom/Document.cpp 2021-01-12 14:34:28 UTC (rev 271401)
@@ -1810,9 +1810,13 @@
client->visibilityStateChanged();
#if ENABLE(MEDIA_STREAM) && PLATFORM(IOS_FAMILY)
+ auto* page = this->page();
+ if (!page)
+ return;
+
if (auto mediaSessionManager = PlatformMediaSessionManager::sharedManagerIfExists()) {
if (!mediaSessionManager->isInterrupted())
- MediaStreamTrack::updateCaptureAccordingToMutedState(*this);
+ MediaStreamTrack::updateCaptureAccordingToMutedState(*this, page->mutedState());
}
#endif
}
@@ -4298,7 +4302,7 @@
audioProducer.pageMutedStateDidChange();
#if ENABLE(MEDIA_STREAM)
- MediaStreamTrack::updateCaptureAccordingToMutedState(*this);
+ MediaStreamTrack::updateCaptureAccordingToMutedState(*this, page()->mutedState());
#endif
}
Modified: trunk/Source/WebCore/page/Page.cpp (271400 => 271401)
--- trunk/Source/WebCore/page/Page.cpp 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebCore/page/Page.cpp 2021-01-12 14:34:28 UTC (rev 271401)
@@ -79,6 +79,7 @@
#include "LowPowerModeNotifier.h"
#include "MediaCanStartListener.h"
#include "MediaRecorderProvider.h"
+#include "MediaStreamTrack.h"
#include "Navigator.h"
#include "PageConfiguration.h"
#include "PageConsoleClient.h"
@@ -2074,6 +2075,24 @@
#endif
}
+void Page::beginAudioCaptureInterruption()
+{
+#if ENABLE(MEDIA_STREAM)
+ forEachDocument([mutedState = mutedState()](auto& document) {
+ MediaStreamTrack::updateCaptureAccordingToMutedState(document, mutedState | MediaProducer::AudioAndVideoCaptureIsMuted);
+ });
+#endif
+}
+
+void Page::endAudioCaptureInterruption()
+{
+#if ENABLE(MEDIA_STREAM)
+ forEachDocument([mutedState = mutedState()](auto& document) {
+ MediaStreamTrack::updateCaptureAccordingToMutedState(document, mutedState);
+ });
+#endif
+}
+
bool Page::mediaPlaybackExists()
{
#if ENABLE(VIDEO)
Modified: trunk/Source/WebCore/page/Page.h (271400 => 271401)
--- trunk/Source/WebCore/page/Page.h 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebCore/page/Page.h 2021-01-12 14:34:28 UTC (rev 271401)
@@ -831,6 +831,9 @@
bool textInteractionEnabled() { return m_textInteractionEnabled; }
void setTextInteractionEnabled(bool value) { m_textInteractionEnabled = value; }
+ WEBCORE_EXPORT void beginAudioCaptureInterruption();
+ WEBCORE_EXPORT void endAudioCaptureInterruption();
+
private:
struct Navigation {
RegistrableDomain domain;
Modified: trunk/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.cpp (271400 => 271401)
--- trunk/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.cpp 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.cpp 2021-01-12 14:34:28 UTC (rev 271401)
@@ -183,13 +183,6 @@
m_needsReconfiguration = false;
reconfigure();
}
-
- ASSERT(!m_producingCount);
-
- forEachClient([](auto& client) {
- client.setMuted(false);
- });
-
return 0;
}
@@ -201,13 +194,6 @@
m_suspended = true;
stopInternal();
-
- forEachClient([](auto& client) {
- client.setMuted(true);
- });
-
- ASSERT(!m_producingCount);
-
return 0;
}
Modified: trunk/Source/WebCore/testing/Internals.cpp (271400 => 271401)
--- trunk/Source/WebCore/testing/Internals.cpp 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebCore/testing/Internals.cpp 2021-01-12 14:34:28 UTC (rev 271401)
@@ -5273,7 +5273,16 @@
void Internals::setMediaStreamSourceInterrupted(MediaStreamTrack& track, bool interrupted)
{
+ auto* document = contextDocument();
+ auto* page = document ? document->page() : nullptr;
+ if (!page)
+ return;
+
track.source().setInterruptedForTesting(interrupted);
+ if (interrupted)
+ page->beginAudioCaptureInterruption();
+ else
+ page->endAudioCaptureInterruption();
}
bool Internals::isMediaStreamSourceInterrupted(MediaStreamTrack& track) const
Modified: trunk/Source/WebKit/ChangeLog (271400 => 271401)
--- trunk/Source/WebKit/ChangeLog 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebKit/ChangeLog 2021-01-12 14:34:28 UTC (rev 271401)
@@ -1,3 +1,20 @@
+2021-01-12 Youenn Fablet <[email protected]>
+
+ Unmuting a track in case of end of interruption by another web process tab is not working well with capture muting icons
+ https://bugs.webkit.org/show_bug.cgi?id=220058
+
+ Reviewed by Eric Carlson.
+
+ Add an observer of AudioSession interruptions that forwards the signal to all tabs of the process.
+
+ * WebProcess/cocoa/WebProcessCocoa.mm:
+ (WebKit::WebProcess::platformInitializeWebProcess):
+ (WebKit::PageAudioSessionInterruptionObserver::PageAudioSessionInterruptionObserver):
+ (WebKit::PageAudioSessionInterruptionObserver::~PageAudioSessionInterruptionObserver):
+ (WebKit::PageAudioSessionInterruptionObserver::beginAudioSessionInterruption):
+ (WebKit::PageAudioSessionInterruptionObserver::endAudioSessionInterruption):
+ (WebKit::listenToAudioSessionInterruption):
+
2021-01-12 Philippe Normand <[email protected]>
[GStreamer] Lazy initialization support
Modified: trunk/Source/WebKit/WebProcess/WebProcess.cpp (271400 => 271401)
--- trunk/Source/WebKit/WebProcess/WebProcess.cpp 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebKit/WebProcess/WebProcess.cpp 2021-01-12 14:34:28 UTC (rev 271401)
@@ -1814,7 +1814,8 @@
void WebProcess::revokeUserMediaDeviceSandboxExtensions(const Vector<String>& extensionIDs)
{
- checkDocumentsCaptureStateConsistency(extensionIDs);
+ if (!MockRealtimeMediaSourceCenter::mockRealtimeMediaSourceCenterEnabled())
+ checkDocumentsCaptureStateConsistency(extensionIDs);
for (const auto& extensionID : extensionIDs) {
auto extension = m_mediaCaptureSandboxExtensions.take(extensionID);
Modified: trunk/Source/WebKit/WebProcess/cocoa/WebProcessCocoa.mm (271400 => 271401)
--- trunk/Source/WebKit/WebProcess/cocoa/WebProcessCocoa.mm 2021-01-12 12:44:06 UTC (rev 271400)
+++ trunk/Source/WebKit/WebProcess/cocoa/WebProcessCocoa.mm 2021-01-12 14:34:28 UTC (rev 271401)
@@ -54,6 +54,7 @@
#import <_javascript_Core/Options.h>
#import <WebCore/AVAssetMIMETypeCache.h>
#import <WebCore/AXObjectCache.h>
+#import <WebCore/AudioSession.h>
#import <WebCore/CPUMonitor.h>
#import <WebCore/DisplayRefreshMonitorManager.h>
#import <WebCore/FontCache.h>
@@ -170,6 +171,10 @@
static const double serviceWorkerCPULimit { 0.5 }; // 50% average CPU usage over 8 minutes.
#endif
+#if PLATFORM(IOS)
+static void listenToAudioSessionInterruption();
+#endif
+
void WebProcess::platformSetCacheModel(CacheModel)
{
}
@@ -406,6 +411,10 @@
#endif
WebCore::IOSurface::setMaximumSize(parameters.maximumIOSurfaceSize);
+
+#if PLATFORM(IOS)
+ listenToAudioSessionInterruption();
+#endif
}
void WebProcess::platformSetWebsiteDataStoreParameters(WebProcessDataStoreParameters&& parameters)
@@ -1177,6 +1186,49 @@
}
}
+#if PLATFORM(IOS)
+class PageAudioSessionInterruptionObserver : public AudioSession::InterruptionObserver {
+public:
+ PageAudioSessionInterruptionObserver();
+ ~PageAudioSessionInterruptionObserver();
+
+private:
+ void beginAudioSessionInterruption() final;
+ void endAudioSessionInterruption(WebCore::AudioSession::MayResume) final;
+};
+
+PageAudioSessionInterruptionObserver::PageAudioSessionInterruptionObserver()
+{
+ AudioSession::sharedSession().addInterruptionObserver(*this);
+}
+
+PageAudioSessionInterruptionObserver::~PageAudioSessionInterruptionObserver()
+{
+ AudioSession::sharedSession().removeInterruptionObserver(*this);
+}
+
+void PageAudioSessionInterruptionObserver::beginAudioSessionInterruption()
+{
+ Page::forEachPage([](auto& page) {
+ page.beginAudioCaptureInterruption();
+ });
+}
+
+void PageAudioSessionInterruptionObserver::endAudioSessionInterruption(AudioSession::MayResume mayResume)
+{
+ if (mayResume == AudioSession::MayResume::No)
+ return;
+ Page::forEachPage([](auto& page) {
+ page.endAudioCaptureInterruption();
+ });
+}
+
+void listenToAudioSessionInterruption()
+{
+ static NeverDestroyed<PageAudioSessionInterruptionObserver> observer;
+}
+#endif
+
} // namespace WebKit
#undef RELEASE_LOG_SESSION_ID
