- Revision
- 206025
- Author
- jer.no...@apple.com
- Date
- 2016-09-16 09:21:58 -0700 (Fri, 16 Sep 2016)
Log Message
[media-source] Only fire 'resize' events when new sized frame is displayed, not parsed.
https://bugs.webkit.org/show_bug.cgi?id=162039
Reviewed by Eric Carlson.
Fixes tests: imported/w3c/web-platform-tests/media-source/mediasource-config-change-mp4-av-framesize.html
imported/w3c/web-platform-tests/media-source/mediasource-config-change-mp4-v-framesize.html
Currently, the SourceBufferPrivateAVFObjC will signal a size change as soon as the sample is
parsed during appendData(). This is incorrect, as the intrinsic size of the video element
should be based on when the sample is displayed, and it causes some W3C test cases to fail.
Set a boundary time observer on the player's synchronizer at each sample's presentation time
where that sample signals a change in intrinsic size. Flush those observers whenever the
samples are flushed un-displayed (typically, during a seek). Because the observer callback
may have already been issued (but not executed) at a flush, use a separate weak pointer
factory, and invalidate all outstanding size change observer weak pointers when flushing.
* Modules/mediasource/SourceBuffer.cpp:
(WebCore::SourceBuffer::reenqueueMediaForTime): When re-enqueing, enqueue the next decodable
sample, even if it doesn't include the current time. This handles cases where the current
time is 0, and the first video sample starts at 0.033.
* platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h:
* platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::MediaPlayerPrivateMediaSourceAVFObjC):
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::~MediaPlayerPrivateMediaSourceAVFObjC):
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::naturalSize):
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::sizeWillChangeAtTime):
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::flushPendingSizeChanges):
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::sizeChanged): Deleted.
* platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h:
* platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm:
(WebCore::SourceBufferPrivateAVFObjC::processCodedFrame):
(WebCore::SourceBufferPrivateAVFObjC::flushAndEnqueueNonDisplayingSamples):
(WebCore::SourceBufferPrivateAVFObjC::enqueueSample):
Modified Paths
Diff
Modified: trunk/Source/WebCore/ChangeLog (206024 => 206025)
--- trunk/Source/WebCore/ChangeLog 2016-09-16 16:21:30 UTC (rev 206024)
+++ trunk/Source/WebCore/ChangeLog 2016-09-16 16:21:58 UTC (rev 206025)
@@ -1,3 +1,40 @@
+2016-09-16 Jer Noble <jer.no...@apple.com>
+
+ [media-source] Only fire 'resize' events when new sized frame is displayed, not parsed.
+ https://bugs.webkit.org/show_bug.cgi?id=162039
+
+ Reviewed by Eric Carlson.
+
+ Fixes tests: imported/w3c/web-platform-tests/media-source/mediasource-config-change-mp4-av-framesize.html
+ imported/w3c/web-platform-tests/media-source/mediasource-config-change-mp4-v-framesize.html
+
+ Currently, the SourceBufferPrivateAVFObjC will signal a size change as soon as the sample is
+ parsed during appendData(). This is incorrect, as the intrinsic size of the video element
+ should be based on when the sample is displayed, and it causes some W3C test cases to fail.
+ Set a boundary time observer on the player's synchronizer at each sample's presentation time
+ where that sample signals a change in intrinsic size. Flush those observers whenever the
+ samples are flushed un-displayed (typically, during a seek). Because the observer callback
+ may have already been issued (but not executed) at a flush, use a separate weak pointer
+ factory, and invalidate all outstanding size change observer weak pointers when flushing.
+
+ * Modules/mediasource/SourceBuffer.cpp:
+ (WebCore::SourceBuffer::reenqueueMediaForTime): When re-enqueing, enqueue the next decodable
+ sample, even if it doesn't include the current time. This handles cases where the current
+ time is 0, and the first video sample starts at 0.033.
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
+ (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::MediaPlayerPrivateMediaSourceAVFObjC):
+ (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::~MediaPlayerPrivateMediaSourceAVFObjC):
+ (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::naturalSize):
+ (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::sizeWillChangeAtTime):
+ (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::flushPendingSizeChanges):
+ (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::sizeChanged): Deleted.
+ * platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h:
+ * platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm:
+ (WebCore::SourceBufferPrivateAVFObjC::processCodedFrame):
+ (WebCore::SourceBufferPrivateAVFObjC::flushAndEnqueueNonDisplayingSamples):
+ (WebCore::SourceBufferPrivateAVFObjC::enqueueSample):
+
2016-09-16 Alex Christensen <achristen...@webkit.org>
Fix Windows clean build after r205929
Modified: trunk/Source/WebCore/Modules/mediasource/SourceBuffer.cpp (206024 => 206025)
--- trunk/Source/WebCore/Modules/mediasource/SourceBuffer.cpp 2016-09-16 16:21:30 UTC (rev 206024)
+++ trunk/Source/WebCore/Modules/mediasource/SourceBuffer.cpp 2016-09-16 16:21:58 UTC (rev 206025)
@@ -1854,7 +1854,7 @@
void SourceBuffer::reenqueueMediaForTime(TrackBuffer& trackBuffer, AtomicString trackID, const MediaTime& time)
{
// Find the sample which contains the current presentation time.
- auto currentSamplePTSIterator = trackBuffer.samples.presentationOrder().findSampleContainingPresentationTime(time);
+ auto currentSamplePTSIterator = trackBuffer.samples.presentationOrder().findSampleOnOrAfterPresentationTime(time);
if (currentSamplePTSIterator == trackBuffer.samples.presentationOrder().end()) {
trackBuffer.decodeQueue.clear();
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h (206024 => 206025)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h 2016-09-16 16:21:30 UTC (rev 206024)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h 2016-09-16 16:21:58 UTC (rev 206025)
@@ -83,7 +83,8 @@
void durationChanged();
void effectiveRateChanged();
- void sizeChanged();
+ void sizeWillChangeAtTime(const MediaTime&, const FloatSize&);
+ void flushPendingSizeChanges();
void characteristicsChanged();
#if PLATFORM(MAC) && ENABLE(VIDEO_PRESENTATION_MODE)
@@ -210,6 +211,7 @@
MediaPlayer* m_player;
WeakPtrFactory<MediaPlayerPrivateMediaSourceAVFObjC> m_weakPtrFactory;
+ WeakPtrFactory<MediaPlayerPrivateMediaSourceAVFObjC> m_sizeChangeObserverWeakPtrFactory;
RefPtr<MediaSourcePrivateAVFObjC> m_mediaSourcePrivate;
RetainPtr<AVAsset> m_asset;
RetainPtr<AVSampleBufferDisplayLayer> m_sampleBufferDisplayLayer;
@@ -218,11 +220,13 @@
RetainPtr<id> m_timeJumpedObserver;
RetainPtr<id> m_durationObserver;
RetainPtr<AVStreamSession> m_streamSession;
+ Deque<RetainPtr<id>> m_sizeChangeObservers;
Timer m_seekTimer;
CDMSessionMediaSourceAVFObjC* m_session;
MediaPlayer::NetworkState m_networkState;
MediaPlayer::ReadyState m_readyState;
MediaTime m_lastSeekTime;
+ FloatSize m_naturalSize;
double m_rate;
bool m_playing;
bool m_seeking;
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm (206024 => 206025)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm 2016-09-16 16:21:30 UTC (rev 206024)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm 2016-09-16 16:21:58 UTC (rev 206025)
@@ -43,6 +43,7 @@
#import <AVFoundation/AVTime.h>
#import <QuartzCore/CALayer.h>
#import <objc_runtime.h>
+#import <wtf/Deque.h>
#import <wtf/MainThread.h>
#import <wtf/NeverDestroyed.h>
@@ -141,6 +142,7 @@
MediaPlayerPrivateMediaSourceAVFObjC::MediaPlayerPrivateMediaSourceAVFObjC(MediaPlayer* player)
: m_player(player)
, m_weakPtrFactory(this)
+ , m_sizeChangeObserverWeakPtrFactory(this)
, m_synchronizer(adoptNS([allocAVSampleBufferRenderSynchronizerInstance() init]))
, m_seekTimer(*this, &MediaPlayerPrivateMediaSourceAVFObjC::seekInternal)
, m_session(nullptr)
@@ -195,6 +197,7 @@
[m_synchronizer removeTimeObserver:m_timeJumpedObserver.get()];
if (m_durationObserver)
[m_synchronizer removeTimeObserver:m_durationObserver.get()];
+ flushPendingSizeChanges();
m_seekTimer.stop();
}
@@ -360,10 +363,7 @@
FloatSize MediaPlayerPrivateMediaSourceAVFObjC::naturalSize() const
{
- if (!m_mediaSourcePrivate)
- return FloatSize();
-
- return m_mediaSourcePrivate->naturalSize();
+ return m_naturalSize;
}
bool MediaPlayerPrivateMediaSourceAVFObjC::hasVideo() const
@@ -682,11 +682,29 @@
m_player->rateChanged();
}
-void MediaPlayerPrivateMediaSourceAVFObjC::sizeChanged()
+void MediaPlayerPrivateMediaSourceAVFObjC::sizeWillChangeAtTime(const MediaTime& time, const FloatSize& size)
{
- m_player->sizeChanged();
+ auto weakThis = m_sizeChangeObserverWeakPtrFactory.createWeakPtr();
+ NSArray* times = @[[NSValue valueWithCMTime:toCMTime(time)]];
+ RetainPtr<id> observer = [m_synchronizer addBoundaryTimeObserverForTimes:times queue:dispatch_get_main_queue() usingBlock:[weakThis, size] {
+ if (!weakThis)
+ return;
+ RetainPtr<id> observer = weakThis->m_sizeChangeObservers.takeFirst();
+ weakThis->m_naturalSize = size;
+ weakThis->m_player->sizeChanged();
+ }];
+ m_sizeChangeObservers.append(WTFMove(observer));
}
+void MediaPlayerPrivateMediaSourceAVFObjC::flushPendingSizeChanges()
+{
+ while (!m_sizeChangeObservers.isEmpty()) {
+ RetainPtr<id> observer = m_sizeChangeObservers.takeFirst();
+ [m_synchronizer removeTimeObserver:observer.get()];
+ }
+ m_sizeChangeObserverWeakPtrFactory.revokeAll();
+}
+
#if ENABLE(ENCRYPTED_MEDIA_V2)
AVStreamSession* MediaPlayerPrivateMediaSourceAVFObjC::streamSession()
{
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h (206024 => 206025)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h 2016-09-16 16:21:30 UTC (rev 206024)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h 2016-09-16 16:21:58 UTC (rev 206025)
@@ -154,6 +154,7 @@
CDMSessionMediaSourceAVFObjC* m_session { nullptr };
FloatSize m_cachedSize;
+ FloatSize m_currentSize;
bool m_parsingSucceeded;
int m_enabledVideoTrackID;
int m_protectedTrackID;
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm (206024 => 206025)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm 2016-09-16 16:21:30 UTC (rev 206024)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm 2016-09-16 16:21:58 UTC (rev 206025)
@@ -551,22 +551,12 @@
bool SourceBufferPrivateAVFObjC::processCodedFrame(int trackID, CMSampleBufferRef sampleBuffer, const String&)
{
- if (trackID == m_enabledVideoTrackID) {
- CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer);
- FloatSize formatSize = FloatSize(CMVideoFormatDescriptionGetPresentationDimensions(formatDescription, true, true));
- if (formatSize != m_cachedSize) {
- LOG(MediaSource, "SourceBufferPrivateAVFObjC::processCodedFrame(%p) - size change detected: {width=%lf, height=%lf}", formatSize.width(), formatSize.height());
- m_cachedSize = formatSize;
- if (m_mediaSource)
- m_mediaSource->player()->sizeChanged();
- }
- } else if (!m_audioRenderers.contains(trackID)) {
+ if (trackID != m_enabledVideoTrackID && !m_audioRenderers.contains(trackID)) {
// FIXME(125161): We don't handle text tracks, and passing this sample up to SourceBuffer
// will just confuse its state. Drop this sample until we can handle text tracks properly.
return false;
}
-
if (m_client) {
Ref<MediaSample> mediaSample = MediaSampleAVFObjC::create(sampleBuffer, trackID);
LOG(MediaSourceSamples, "SourceBufferPrivateAVFObjC::processCodedFrame(%p) - sample(%s)", this, toString(mediaSample.get()).utf8().data());
@@ -957,8 +947,10 @@
[layer enqueueSampleBuffer:sampleBuffer.get()];
}
- if (m_mediaSource)
+ if (m_mediaSource) {
m_mediaSource->player()->setHasAvailableVideoFrame(false);
+ m_mediaSource->player()->flushPendingSizeChanges();
+ }
}
void SourceBufferPrivateAVFObjC::enqueueSample(PassRefPtr<MediaSample> prpMediaSample, AtomicString trackIDString)
@@ -976,6 +968,15 @@
LOG(MediaSourceSamples, "SourceBufferPrivateAVFObjC::enqueueSample(%p) - sample(%s)", this, toString(*mediaSample).utf8().data());
if (trackID == m_enabledVideoTrackID) {
+ CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(platformSample.sample.cmSampleBuffer);
+ FloatSize formatSize = FloatSize(CMVideoFormatDescriptionGetPresentationDimensions(formatDescription, true, true));
+ if (formatSize != m_cachedSize) {
+ LOG(MediaSource, "SourceBufferPrivateAVFObjC::enqueueSample(%p) - size change detected: {width=%lf, height=%lf}", formatSize.width(), formatSize.height());
+ m_cachedSize = formatSize;
+ if (m_mediaSource)
+ m_mediaSource->player()->sizeWillChangeAtTime(mediaSample->presentationTime(), formatSize);
+ }
+
[m_displayLayer enqueueSampleBuffer:platformSample.sample.cmSampleBuffer];
if (m_mediaSource)
m_mediaSource->player()->setHasAvailableVideoFrame(true);
