Diff
Modified: trunk/LayoutTests/ChangeLog (221097 => 221098)
--- trunk/LayoutTests/ChangeLog 2017-08-23 20:14:37 UTC (rev 221097)
+++ trunk/LayoutTests/ChangeLog 2017-08-23 20:16:51 UTC (rev 221098)
@@ -1,3 +1,14 @@
+2017-08-23 Jer Noble <jer.no...@apple.com>
+
+ Track VideoPlaybackQuality metrics when using WebCoreDecompressionSession.
+ https://bugs.webkit.org/show_bug.cgi?id=175835
+ <rdar://problem/34022234>
+
+ Reviewed by Eric Carlson.
+
+ * platform/mac/media/media-source/videoplaybackquality-decompressionsession-expected.txt: Added.
+ * platform/mac/media/media-source/videoplaybackquality-decompressionsession.html: Added.
+
2017-08-23 Matt Lewis <jlew...@apple.com>
Removed flaky timeout expectation for inspector/codemirror/prettyprinting-css.html.
Modified: trunk/LayoutTests/media/media-source/media-source-loader.js (221097 => 221098)
--- trunk/LayoutTests/media/media-source/media-source-loader.js 2017-08-23 20:14:37 UTC (rev 221097)
+++ trunk/LayoutTests/media/media-source/media-source-loader.js 2017-08-23 20:16:51 UTC (rev 221098)
@@ -1,6 +1,7 @@
-function MediaSourceLoader(url)
+function MediaSourceLoader(url, prefix)
{
this._url = url;
+ this._prefix = prefix;
setTimeout(this.loadManifest.bind(this));
this._onload_ = null;
@@ -40,7 +41,8 @@
loadMediaData: function()
{
var request = new XMLHttpRequest();
- request.open('GET', this._manifest.url, true);
+ var url = "" ? this._prefix : '') + this._manifest.url
+ request.open('GET', url, true);
request.responseType = 'arraybuffer';
request._onload_ = this.loadMediaDataSucceeded.bind(this);
request._onerror_ = this.loadMediaDataFailed.bind(this);
Added: trunk/LayoutTests/platform/mac/media/media-source/videoplaybackquality-decompressionsession-expected.txt (0 => 221098)
--- trunk/LayoutTests/platform/mac/media/media-source/videoplaybackquality-decompressionsession-expected.txt (rev 0)
+++ trunk/LayoutTests/platform/mac/media/media-source/videoplaybackquality-decompressionsession-expected.txt 2017-08-23 20:16:51 UTC (rev 221098)
@@ -0,0 +1,23 @@
+RUN(video.src = ""
+EVENT(sourceopen)
+RUN(source.duration = loader.duration())
+RUN(sourceBuffer = source.addSourceBuffer(loader.type()))
+RUN(sourceBuffer.appendBuffer(loader.initSegment()))
+EVENT(update)
+RUN(sourceBuffer.appendBuffer(loader.mediaSegment(0)))
+EVENT(update)
+RUN(canvas = document.createElement("canvas"))
+RUN(gl = canvas.getContext("webgl"))
+RUN(texture = gl.createTexture())
+RUN(gl.bindTexture(gl.TEXTURE_2D, texture))
+RUN(gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR))
+RUN(gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR))
+RUN(gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE))
+RUN(gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE))
+RUN(gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video))
+RUN(video.play())
+Wait for currentTime to advance beyond 0s.
+EVENT(timeupdate)
+EXPECTED (video.getVideoPlaybackQuality().totalVideoFrames > '0') OK
+END OF TEST
+
Added: trunk/LayoutTests/platform/mac/media/media-source/videoplaybackquality-decompressionsession.html (0 => 221098)
--- trunk/LayoutTests/platform/mac/media/media-source/videoplaybackquality-decompressionsession.html (rev 0)
+++ trunk/LayoutTests/platform/mac/media/media-source/videoplaybackquality-decompressionsession.html 2017-08-23 20:16:51 UTC (rev 221098)
@@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <title>audio-session-category-track-change</title>
+ <script src=""
+ <script src=""
+ <script>
+ mediaElement = video = document.createElement('video');
+
+ loader = new MediaSourceLoader('../../../../media/media-source/content/test-fragmented-manifest.json', '../../../../media/media-source/');
+ loader._onerror_ = () => {
+ failTest('Media data loading failed');
+ };
+
+ loader._onload_ = () => {
+ source = new MediaSource();
+ waitForEvent('sourceopen', sourceOpen, false, false, source);
+ waitForEventAndFail('error');
+ run('video.src = ""
+ }
+
+ function sourceOpen() {
+ run('source.duration = loader.duration()');
+ run('sourceBuffer = source.addSourceBuffer(loader.type())');
+ waitForEventOn(sourceBuffer, 'update', sourceInitialized, false, true);
+ run('sourceBuffer.appendBuffer(loader.initSegment())');
+ }
+
+ function sourceInitialized() {
+ waitForEventOn(sourceBuffer, 'update', mediaSegmentAppended, false, true);
+ run('sourceBuffer.appendBuffer(loader.mediaSegment(0))');
+ }
+
+ function mediaSegmentAppended() {
+ run('canvas = document.createElement("canvas")');
+ run('gl = canvas.getContext("webgl")');
+ run('texture = gl.createTexture()');
+ run('gl.bindTexture(gl.TEXTURE_2D, texture)');
+ run('gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)');
+ run('gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)');
+ run('gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)');
+ run('gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)');
+ run('gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video)');
+ run('video.play()');
+ consoleWrite('Wait for currentTime to advance beyond 0s.');
+ video.addEventListener('timeupdate', timeupdate);
+ }
+
+ function timeupdate() {
+ if (video.currentTime == 0)
+ return;
+
+ consoleWrite('EVENT(timeupdate)');
+ testExpected('video.getVideoPlaybackQuality().totalVideoFrames', '0', '>');
+ endTest();
+ }
+
+ </script>
+</head>
+</html>
\ No newline at end of file
Modified: trunk/Source/WebCore/ChangeLog (221097 => 221098)
--- trunk/Source/WebCore/ChangeLog 2017-08-23 20:14:37 UTC (rev 221097)
+++ trunk/Source/WebCore/ChangeLog 2017-08-23 20:16:51 UTC (rev 221098)
@@ -1,3 +1,33 @@
+2017-08-23 Jer Noble <jer.no...@apple.com>
+
+ Track VideoPlaybackQuality metrics when using WebCoreDecompressionSession.
+ https://bugs.webkit.org/show_bug.cgi?id=175835
+ <rdar://problem/34022234>
+
+ Reviewed by Eric Carlson.
+
+ Test: platform/mac/media/media-source/videoplaybackquality-decompressionsession.html
+
+ Track the total number of frames decoded, dropped, & corrupted, as well as the total
+ delay imposed by decoding in the WebCoreDecompressionSession.
+
+ Drive-by fix: implement frame dropping by skipping frames whose presentation times are
+ before the video's current time and which aren't depended upon by other frames.
+
+ * platform/cf/CoreMediaSoftLink.cpp:
+ * platform/cf/CoreMediaSoftLink.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
+ (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::videoPlaybackQualityMetrics):
+ * platform/graphics/cocoa/WebCoreDecompressionSession.h:
+ (WebCore::WebCoreDecompressionSession::totalVideoFrames):
+ (WebCore::WebCoreDecompressionSession::droppedVideoFrames):
+ (WebCore::WebCoreDecompressionSession::corruptedVideoFrames):
+ (WebCore::WebCoreDecompressionSession::totalFrameDelay):
+ * platform/graphics/cocoa/WebCoreDecompressionSession.mm:
+ (WebCore::WebCoreDecompressionSession::shouldDecodeSample):
+ (WebCore::WebCoreDecompressionSession::decodeSample):
+ (WebCore::WebCoreDecompressionSession::handleDecompressionOutput):
+
2017-08-23 Andy Estes <aes...@apple.com>
[Payment Request] Update to "In Development" in features.json
Modified: trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.cpp (221097 => 221098)
--- trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.cpp 2017-08-23 20:14:37 UTC (rev 221097)
+++ trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.cpp 2017-08-23 20:16:51 UTC (rev 221098)
@@ -48,6 +48,7 @@
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeGetSeconds, Float64, (CMTime time), (time))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMake, CMTime, (int64_t value, int32_t timescale), (value, timescale))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMakeWithSeconds, CMTime, (Float64 seconds, int32_t preferredTimeScale), (seconds, preferredTimeScale))
+SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeGetEnd, CMTime, (CMTimeRange range), (range))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeMake, CMTimeRange, (CMTime start, CMTime duration), (start, duration))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeEqual, Boolean, (CMTimeRange range1, CMTimeRange range2), (range1, range2))
@@ -155,7 +156,6 @@
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMaximum, CMTime, (CMTime time1, CMTime time2), (time1, time2))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMinimum, CMTime, (CMTime time1, CMTime time2), (time1, time2))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeContainsTime, Boolean, (CMTimeRange range, CMTime time), (range, time))
-SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMTimeIndefinite, CMTime)
#endif // PLATFORM(IOS)
Modified: trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.h (221097 => 221098)
--- trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.h 2017-08-23 20:14:37 UTC (rev 221097)
+++ trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.h 2017-08-23 20:16:51 UTC (rev 221098)
@@ -59,6 +59,8 @@
#define CMTimeMake softLink_CoreMedia_CMTimeMake
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeMakeWithSeconds, CMTime, (Float64 seconds, int32_t preferredTimeScale), (seconds, preferredTimeScale))
#define CMTimeMakeWithSeconds softLink_CoreMedia_CMTimeMakeWithSeconds
+SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
+#define CMTimeSubtract softLink_CoreMedia_CMTimeSubtract
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeRangeGetEnd, CMTime, (CMTimeRange range), (range))
#define CMTimeRangeGetEnd softLink_CoreMedia_CMTimeRangeGetEnd
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeRangeMake, CMTimeRange, (CMTime start, CMTime duration), (start, duration))
@@ -263,8 +265,6 @@
#define CMTimeMinimum softLink_CoreMedia_CMTimeMinimum
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeRangeContainsTime, Boolean, (CMTimeRange range, CMTime time), (range, time))
#define CMTimeRangeContainsTime softLink_CoreMedia_CMTimeRangeContainsTime
-SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
-#define CMTimeSubtract softLink_CoreMedia_CMTimeSubtract
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMTimeIndefinite, CMTime)
#define kCMTimeIndefinite get_CoreMedia_kCMTimeIndefinite()
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm (221097 => 221098)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm 2017-08-23 20:14:37 UTC (rev 221097)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm 2017-08-23 20:16:51 UTC (rev 221098)
@@ -689,6 +689,14 @@
std::optional<PlatformVideoPlaybackQualityMetrics> MediaPlayerPrivateMediaSourceAVFObjC::videoPlaybackQualityMetrics()
{
+ if (m_decompressionSession) {
+ return PlatformVideoPlaybackQualityMetrics(
+ m_decompressionSession->totalVideoFrames(),
+ m_decompressionSession->droppedVideoFrames(),
+ m_decompressionSession->corruptedVideoFrames(),
+ m_decompressionSession->totalFrameDelay().toDouble()
+ );
+ }
auto metrics = [m_sampleBufferDisplayLayer videoPerformanceMetrics];
if (!metrics)
Modified: trunk/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.h (221097 => 221098)
--- trunk/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.h 2017-08-23 20:14:37 UTC (rev 221097)
+++ trunk/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.h 2017-08-23 20:16:51 UTC (rev 221098)
@@ -70,6 +70,11 @@
RetainPtr<CVPixelBufferRef> imageForTime(const MediaTime&, ImageForTimeFlags = ExactTime);
void flush();
+ unsigned long totalVideoFrames() { return m_totalVideoFrames; }
+ unsigned long droppedVideoFrames() { return m_droppedVideoFrames; }
+ unsigned long corruptedVideoFrames() { return m_corruptedVideoFrames; }
+ MediaTime totalFrameDelay() { return m_totalFrameDelay; }
+
private:
WebCoreDecompressionSession();
@@ -79,6 +84,7 @@
RetainPtr<CVPixelBufferRef> getFirstVideoFrame();
void resetAutomaticDequeueTimer();
void automaticDequeue();
+ bool shouldDecodeSample(CMSampleBufferRef, bool displaying);
static void decompressionOutputCallback(void* decompressionOutputRefCon, void* sourceFrameRefCon, OSStatus, VTDecodeInfoFlags, CVImageBufferRef, CMTime presentationTimeStamp, CMTime presentationDuration);
static CMTime getDecodeTime(CMBufferRef, void* refcon);
@@ -106,6 +112,10 @@
bool m_invalidated { false };
int m_framesBeingDecoded { 0 };
+ unsigned long m_totalVideoFrames { 0 };
+ unsigned long m_droppedVideoFrames { 0 };
+ unsigned long m_corruptedVideoFrames { 0 };
+ MediaTime m_totalFrameDelay;
};
}
Modified: trunk/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm (221097 => 221098)
--- trunk/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm 2017-08-23 20:14:37 UTC (rev 221097)
+++ trunk/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm 2017-08-23 20:16:51 UTC (rev 221098)
@@ -180,6 +180,33 @@
});
}
+bool WebCoreDecompressionSession::shouldDecodeSample(CMSampleBufferRef sample, bool displaying)
+{
+ if (!displaying)
+ return true;
+
+ if (!m_timebase)
+ return true;
+
+ auto currentTime = CMTimebaseGetTime(m_timebase.get());
+ auto presentationTimeStamp = CMSampleBufferGetPresentationTimeStamp(sample);
+ if (CMTimeCompare(presentationTimeStamp, currentTime) >= 0)
+ return true;
+
+ CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sample, false);
+ if (!attachments)
+ return true;
+
+ for (CFIndex index = 0, count = CFArrayGetCount(attachments); index < count; ++index) {
+ CFDictionaryRef attachmentDict = (CFDictionaryRef)CFArrayGetValueAtIndex(attachments, index);
+ CFBooleanRef dependedOn = (CFBooleanRef)CFDictionaryGetValue(attachmentDict, kCMSampleAttachmentKey_IsDependedOnByOthers);
+ if (dependedOn && !CFBooleanGetValue(dependedOn))
+ return false;
+ }
+
+ return true;
+}
+
void WebCoreDecompressionSession::decodeSample(CMSampleBufferRef sample, bool displaying)
{
if (isInvalidated())
@@ -212,6 +239,12 @@
if (!displaying)
flags |= kVTDecodeFrame_DoNotOutputFrame;
+ if (!shouldDecodeSample(sample, displaying)) {
+ ++m_totalVideoFrames;
+ ++m_droppedVideoFrames;
+ return;
+ }
+
VTDecompressionSessionDecodeFrame(m_decompressionSession.get(), sample, flags, reinterpret_cast<void*>(displaying), nullptr);
}
@@ -224,12 +257,15 @@
void WebCoreDecompressionSession::handleDecompressionOutput(bool displaying, OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef rawImageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration)
{
- UNUSED_PARAM(status);
- UNUSED_PARAM(infoFlags);
+ ++m_totalVideoFrames;
+ if (infoFlags & kVTDecodeInfo_FrameDropped)
+ ++m_droppedVideoFrames;
CMVideoFormatDescriptionRef rawImageBufferDescription = nullptr;
- if (noErr != CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, rawImageBuffer, &rawImageBufferDescription))
+ if (status != noErr || noErr != CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, rawImageBuffer, &rawImageBufferDescription)) {
+ ++m_corruptedVideoFrames;
return;
+ }
RetainPtr<CMVideoFormatDescriptionRef> imageBufferDescription = adoptCF(rawImageBufferDescription);
CMSampleTimingInfo imageBufferTiming {
@@ -239,11 +275,17 @@
};
CMSampleBufferRef rawImageSampleBuffer = nullptr;
- if (noErr != CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, rawImageBuffer, imageBufferDescription.get(), &imageBufferTiming, &rawImageSampleBuffer))
+ if (noErr != CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, rawImageBuffer, imageBufferDescription.get(), &imageBufferTiming, &rawImageSampleBuffer)) {
+ ++m_corruptedVideoFrames;
return;
- RefPtr<WebCoreDecompressionSession> protectedThis { this };
- RetainPtr<CMSampleBufferRef> imageSampleBuffer = adoptCF(rawImageSampleBuffer);
- dispatch_async(m_enqueingQueue.get(), [protectedThis, imageSampleBuffer, displaying] {
+ }
+
+ auto currentTime = CMTimebaseGetTime(m_timebase.get());
+ if (m_timebase && CMTimeCompare(presentationTimeStamp, currentTime) < 0)
+ m_totalFrameDelay += toMediaTime(CMTimeSubtract(currentTime, presentationTimeStamp));
+
+ dispatch_async(m_enqueingQueue.get(), [protectedThis = makeRefPtr(this), status, imageSampleBuffer = adoptCF(rawImageSampleBuffer), infoFlags, displaying] {
+ UNUSED_PARAM(infoFlags);
protectedThis->enqueueDecodedSample(imageSampleBuffer.get(), displaying);
});
}
