Diff
Modified: trunk/LayoutTests/ChangeLog (237273 => 237274)
--- trunk/LayoutTests/ChangeLog 2018-10-18 23:01:42 UTC (rev 237273)
+++ trunk/LayoutTests/ChangeLog 2018-10-18 23:02:15 UTC (rev 237274)
@@ -1,5 +1,19 @@
2018-10-18 Jer Noble <jer.no...@apple.com>
+ [MSE] timestampOffset can introduce floating-point rounding errors to incoming samples
+ https://bugs.webkit.org/show_bug.cgi?id=190590
+ <rdar://problem/45275626>
+
+ Reviewed by Eric Carlson.
+
+ * media/media-source/media-source-sequence-timestamps-expected.txt:
+ * media/media-source/media-source-timestampoffset-rounding-error-expected.txt: Added.
+ * media/media-source/media-source-timestampoffset-rounding-error.html: Added.
+ * media/media-source/mock-media-source.js:
+ (makeASample):
+
+2018-10-18 Jer Noble <jer.no...@apple.com>
+
Enable WKPreferences._lowPowerVideoAudioBufferSizeEnabled by default
https://bugs.webkit.org/show_bug.cgi?id=190315
<rdar://problem/45047807>
Modified: trunk/LayoutTests/media/media-source/media-source-sequence-timestamps-expected.txt (237273 => 237274)
--- trunk/LayoutTests/media/media-source/media-source-sequence-timestamps-expected.txt 2018-10-18 23:01:42 UTC (rev 237273)
+++ trunk/LayoutTests/media/media-source/media-source-sequence-timestamps-expected.txt 2018-10-18 23:02:15 UTC (rev 237274)
@@ -7,7 +7,7 @@
RUN(sourceBuffer.appendBuffer(samples))
EVENT(updateend)
EXPECTED (bufferedSamples.length == '6') OK
-{PTS({0/1 = 0.000000}), DTS({0/1 = 0.000000}), duration({1000/1000 = 1.000000}), flags(1), generation(0)}
+{PTS({0/1000 = 0.000000}), DTS({0/1000 = 0.000000}), duration({1000/1000 = 1.000000}), flags(1), generation(0)}
{PTS({1000/1000 = 1.000000}), DTS({1000/1000 = 1.000000}), duration({1000/1000 = 1.000000}), flags(0), generation(0)}
{PTS({2000/1000 = 2.000000}), DTS({2000/1000 = 2.000000}), duration({1000/1000 = 1.000000}), flags(0), generation(0)}
{PTS({3000/1000 = 3.000000}), DTS({3000/1000 = 3.000000}), duration({1000/1000 = 1.000000}), flags(1), generation(0)}
Modified: trunk/LayoutTests/media/media-source/media-source-timeoffset-expected.txt (237273 => 237274)
--- trunk/LayoutTests/media/media-source/media-source-timeoffset-expected.txt 2018-10-18 23:01:42 UTC (rev 237273)
+++ trunk/LayoutTests/media/media-source/media-source-timeoffset-expected.txt 2018-10-18 23:02:15 UTC (rev 237274)
@@ -8,11 +8,11 @@
RUN(sourceBuffer.appendBuffer(samples))
EVENT(updateend)
EXPECTED (bufferedSamples.length == '6') OK
-{PTS({10.000000}), DTS({10.000000}), duration({1000/1000 = 1.000000}), flags(1), generation(0)}
-{PTS({11.000000}), DTS({11.000000}), duration({1000/1000 = 1.000000}), flags(0), generation(0)}
-{PTS({12.000000}), DTS({12.000000}), duration({1000/1000 = 1.000000}), flags(0), generation(0)}
-{PTS({13.000000}), DTS({13.000000}), duration({1000/1000 = 1.000000}), flags(1), generation(0)}
-{PTS({14.000000}), DTS({14.000000}), duration({1000/1000 = 1.000000}), flags(0), generation(0)}
-{PTS({15.000000}), DTS({15.000000}), duration({1000/1000 = 1.000000}), flags(0), generation(0)}
+{PTS({10000/1000 = 10.000000}), DTS({10000/1000 = 10.000000}), duration({1000/1000 = 1.000000}), flags(1), generation(0)}
+{PTS({11000/1000 = 11.000000}), DTS({11000/1000 = 11.000000}), duration({1000/1000 = 1.000000}), flags(0), generation(0)}
+{PTS({12000/1000 = 12.000000}), DTS({12000/1000 = 12.000000}), duration({1000/1000 = 1.000000}), flags(0), generation(0)}
+{PTS({13000/1000 = 13.000000}), DTS({13000/1000 = 13.000000}), duration({1000/1000 = 1.000000}), flags(1), generation(0)}
+{PTS({14000/1000 = 14.000000}), DTS({14000/1000 = 14.000000}), duration({1000/1000 = 1.000000}), flags(0), generation(0)}
+{PTS({15000/1000 = 15.000000}), DTS({15000/1000 = 15.000000}), duration({1000/1000 = 1.000000}), flags(0), generation(0)}
END OF TEST
Added: trunk/LayoutTests/media/media-source/media-source-timestampoffset-rounding-error-expected.txt (0 => 237274)
--- trunk/LayoutTests/media/media-source/media-source-timestampoffset-rounding-error-expected.txt (rev 0)
+++ trunk/LayoutTests/media/media-source/media-source-timestampoffset-rounding-error-expected.txt 2018-10-18 23:02:15 UTC (rev 237274)
@@ -0,0 +1,34 @@
+
+EXPECTED (source.readyState == 'closed') OK
+EVENT(sourceopen)
+RUN(sourceBuffer = source.addSourceBuffer("video/mock; codecs=mock"))
+RUN(sourceBuffer.appendBuffer(makeVideo(0, 6)))
+EVENT(updateend)
+RUN(sourceBuffer.timestampOffset = 1)
+RUN(sourceBuffer.appendBuffer(makeVideo(1, 6)))
+EVENT(updateend)
+RUN(sourceBuffer.timestampOffset = 1.5)
+RUN(sourceBuffer.appendBuffer(makeVideo(2, 5)))
+EVENT(updateend)
+Buffered:
+{PTS({0/6 = 0.000000}), DTS({0/6 = 0.000000}), duration({1/6 = 0.166667}), flags(1), generation(0)}
+{PTS({1/6 = 0.166667}), DTS({1/6 = 0.166667}), duration({1/6 = 0.166667}), flags(0), generation(0)}
+{PTS({2/6 = 0.333333}), DTS({2/6 = 0.333333}), duration({1/6 = 0.166667}), flags(1), generation(0)}
+{PTS({3/6 = 0.500000}), DTS({3/6 = 0.500000}), duration({1/6 = 0.166667}), flags(0), generation(0)}
+{PTS({4/6 = 0.666667}), DTS({4/6 = 0.666667}), duration({1/6 = 0.166667}), flags(1), generation(0)}
+{PTS({5/6 = 0.833333}), DTS({5/6 = 0.833333}), duration({1/6 = 0.166667}), flags(0), generation(0)}
+{PTS({6/6 = 1.000000}), DTS({6/6 = 1.000000}), duration({1/6 = 0.166667}), flags(1), generation(1)}
+{PTS({7/6 = 1.166667}), DTS({7/6 = 1.166667}), duration({1/6 = 0.166667}), flags(0), generation(1)}
+{PTS({8/6 = 1.333333}), DTS({8/6 = 1.333333}), duration({1/6 = 0.166667}), flags(1), generation(1)}
+{PTS({15/10 = 1.500000}), DTS({15/10 = 1.500000}), duration({1/5 = 0.200000}), flags(1), generation(2)}
+{PTS({17/10 = 1.700000}), DTS({17/10 = 1.700000}), duration({1/5 = 0.200000}), flags(0), generation(2)}
+{PTS({19/10 = 1.900000}), DTS({19/10 = 1.900000}), duration({1/5 = 0.200000}), flags(1), generation(2)}
+{PTS({21/10 = 2.100000}), DTS({21/10 = 2.100000}), duration({1/5 = 0.200000}), flags(0), generation(2)}
+{PTS({23/10 = 2.300000}), DTS({23/10 = 2.300000}), duration({1/5 = 0.200000}), flags(1), generation(2)}
+{PTS({25/10 = 2.500000}), DTS({25/10 = 2.500000}), duration({1/5 = 0.200000}), flags(0), generation(2)}
+{PTS({27/10 = 2.700000}), DTS({27/10 = 2.700000}), duration({1/5 = 0.200000}), flags(1), generation(2)}
+{PTS({29/10 = 2.900000}), DTS({29/10 = 2.900000}), duration({1/5 = 0.200000}), flags(0), generation(2)}
+{PTS({31/10 = 3.100000}), DTS({31/10 = 3.100000}), duration({1/5 = 0.200000}), flags(1), generation(2)}
+{PTS({33/10 = 3.300000}), DTS({33/10 = 3.300000}), duration({1/5 = 0.200000}), flags(0), generation(2)}
+END OF TEST
+
Added: trunk/LayoutTests/media/media-source/media-source-timestampoffset-rounding-error.html (0 => 237274)
--- trunk/LayoutTests/media/media-source/media-source-timestampoffset-rounding-error.html (rev 0)
+++ trunk/LayoutTests/media/media-source/media-source-timestampoffset-rounding-error.html 2018-10-18 23:02:15 UTC (rev 237274)
@@ -0,0 +1,71 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <title>media-source-timestampoffset-rounding-error</title>
+ <script src=""
+ <script src=""
+ <script>
+ var source;
+ var sourceBuffer;
+ var initSegment;
+ var bufferedSamples;
+ var enqueuedSamples;
+
+ if (window.internals)
+ internals.initializeMockMediaSource();
+
+ function makeVideo(generation, timeScale) {
+ const init = makeAInit(2, [makeATrack(1, 'mock', TRACK_KIND.VIDEO)]);
+
+ const samples = [];
+ for (let time = 0; time < 2 * timeScale; time++)
+ samples.push(makeASample(time / timeScale, time / timeScale, 1 / timeScale, 1, time % 2 === 0 ? SAMPLE_FLAG.SYNC : SAMPLE_FLAG.NONE, generation, timeScale));
+
+ return concatenateSamples([init].concat(samples));
+ }
+
+ window.addEventListener('load', async () => {
+ findMediaElement();
+ source = new MediaSource();
+ testExpected('source.readyState', 'closed');
+ const sourceOpened = waitFor(source, 'sourceopen');
+
+ const videoSource = document.createElement('source');
+ videoSource.type = 'video/mock; codecs=mock';
+ videoSource.src = ""
+ video.appendChild(videoSource);
+
+ await sourceOpened;
+ run('sourceBuffer = source.addSourceBuffer("video/mock; codecs=mock")');
+
+ run('sourceBuffer.appendBuffer(makeVideo(0, 6))');
+ await waitFor(sourceBuffer, 'updateend');
+
+ bufferedSamples = internals.bufferedSamplesForTrackID(sourceBuffer, 1);
+ enqueuedSamples = internals.enqueuedSamplesForTrackID(sourceBuffer, 1);
+
+ run("sourceBuffer.timestampOffset = 1");
+ run('sourceBuffer.appendBuffer(makeVideo(1, 6))');
+ await waitFor(sourceBuffer, 'updateend');
+
+ bufferedSamples = internals.bufferedSamplesForTrackID(sourceBuffer, 1);
+ enqueuedSamples = internals.enqueuedSamplesForTrackID(sourceBuffer, 1);
+
+ run("sourceBuffer.timestampOffset = 1.5");
+ run('sourceBuffer.appendBuffer(makeVideo(2, 5))');
+ await waitFor(sourceBuffer, 'updateend');
+
+ bufferedSamples = internals.bufferedSamplesForTrackID(sourceBuffer, 1);
+ enqueuedSamples = internals.enqueuedSamplesForTrackID(sourceBuffer, 1);
+
+ consoleWrite("Buffered:");
+ bufferedSamples.forEach(consoleWrite);
+
+ endTest();
+ });
+ </script>
+</head>
+<body>
+ <video controls></video>
+</body>
+</html>
Modified: trunk/LayoutTests/media/media-source/mock-media-source.js (237273 => 237274)
--- trunk/LayoutTests/media/media-source/mock-media-source.js 2018-10-18 23:01:42 UTC (rev 237273)
+++ trunk/LayoutTests/media/media-source/mock-media-source.js 2018-10-18 23:02:15 UTC (rev 237274)
@@ -16,7 +16,9 @@
DELAYED: 1 << 3,
};
-function makeASample(presentationTime, decodeTime, duration, trackID, flags, generation) {
+function makeASample(presentationTime, decodeTime, duration, trackID, flags, generation, timeScale) {
+ if (typeof timeScale === 'undefined')
+ timeScale = 1000
var byteLength = 30;
var buffer = new ArrayBuffer(byteLength);
var array = new Uint8Array(buffer);
@@ -24,8 +26,6 @@
var view = new DataView(buffer);
view.setUint32(4, byteLength, true);
-
- var timeScale = 1000;
view.setInt32(8, timeScale, true);
view.setInt32(12, presentationTime * timeScale, true);
view.setInt32(16, decodeTime * timeScale, true);
Modified: trunk/Source/WebCore/ChangeLog (237273 => 237274)
--- trunk/Source/WebCore/ChangeLog 2018-10-18 23:01:42 UTC (rev 237273)
+++ trunk/Source/WebCore/ChangeLog 2018-10-18 23:02:15 UTC (rev 237274)
@@ -1,3 +1,23 @@
+2018-10-18 Jer Noble <jer.no...@apple.com>
+
+ [MSE] timestampOffset can introduce floating-point rounding errors to incoming samples
+ https://bugs.webkit.org/show_bug.cgi?id=190590
+ <rdar://problem/45275626>
+
+ Reviewed by Eric Carlson.
+
+ Test: media/media-source/media-source-timestampoffset-rounding-error.html
+
+ SourceBuffer.timestampOffset is a double property, which, when added to a MediaTime will
+ result in a double-backed MediaTime as PTS & DTS. This can introduce rounding errors when
+ these samples are appended as overlapping existing samples. Rather than converting a MediaTime
+ to double-backed when adding the timestampOffset, convert the offset to a multiple of the
+ sample's timeBase.
+
+ * Modules/mediasource/SourceBuffer.cpp:
+ (WebCore::SourceBuffer::setTimestampOffset):
+ (WebCore::SourceBuffer::sourceBufferPrivateDidReceiveSample):
+
2018-10-18 Eric Carlson <eric.carl...@apple.com>
[MediaStream] Allow ports to optionally do screen capture in the UI process
Modified: trunk/Source/WebCore/Modules/mediasource/SourceBuffer.cpp (237273 => 237274)
--- trunk/Source/WebCore/Modules/mediasource/SourceBuffer.cpp 2018-10-18 23:01:42 UTC (rev 237273)
+++ trunk/Source/WebCore/Modules/mediasource/SourceBuffer.cpp 2018-10-18 23:02:15 UTC (rev 237274)
@@ -55,6 +55,7 @@
#include <_javascript_Core/JSLock.h>
#include <_javascript_Core/VM.h>
#include <limits>
+#include <wtf/CheckedArithmetic.h>
namespace WebCore {
@@ -67,6 +68,8 @@
MediaTime highestPresentationTimestamp;
MediaTime lastEnqueuedPresentationTime;
MediaTime lastEnqueuedDecodeEndTime;
+ MediaTime roundedTimestampOffset;
+ uint32_t lastFrameTimescale { 0 };
bool needRandomAccessFlag { true };
bool enabled { false };
bool needsReenqueueing { false };
@@ -169,6 +172,11 @@
// 7. Update the attribute to the new value.
m_timestampOffset = newTimestampOffset;
+ for (auto& trackBuffer : m_trackBufferMap.values()) {
+ trackBuffer.lastFrameTimescale = 0;
+ trackBuffer.roundedTimestampOffset = MediaTime::invalidTime();
+ }
+
return { };
}
@@ -1414,13 +1422,21 @@
MediaTime presentationTimestamp;
MediaTime decodeTimestamp;
+ // NOTE: this is out-of-order, but we need the timescale from the
+ // sample's duration for timestamp generation.
+ // 1.2 Let frame duration be a double precision floating point representation of the coded frame's
+ // duration in seconds.
+ MediaTime frameDuration = sample.duration();
+
if (m_shouldGenerateTimestamps) {
// ↳ If generate timestamps flag equals true:
// 1. Let presentation timestamp equal 0.
- presentationTimestamp = MediaTime::zeroTime();
+ // NOTE: Use the duration timscale for the presentation timestamp, as this will eliminate
+ // timescale rounding when generating timestamps.
+ presentationTimestamp = { 0, frameDuration.timeScale() };
// 2. Let decode timestamp equal 0.
- decodeTimestamp = MediaTime::zeroTime();
+ decodeTimestamp = { 0, frameDuration.timeScale() };
} else {
// ↳ Otherwise:
// 1. Let presentation timestamp be a double precision floating point representation of
@@ -1432,15 +1448,16 @@
decodeTimestamp = sample.decodeTime();
}
- // 1.2 Let frame duration be a double precision floating point representation of the coded frame's
- // duration in seconds.
- MediaTime frameDuration = sample.duration();
-
// 1.3 If mode equals "sequence" and group start timestamp is set, then run the following steps:
if (m_mode == AppendMode::Sequence && m_groupStartTimestamp.isValid()) {
// 1.3.1 Set timestampOffset equal to group start timestamp - presentation timestamp.
m_timestampOffset = m_groupStartTimestamp;
+ for (auto& trackBuffer : m_trackBufferMap.values()) {
+ trackBuffer.lastFrameTimescale = 0;
+ trackBuffer.roundedTimestampOffset = MediaTime::invalidTime();
+ }
+
// 1.3.2 Set group end timestamp equal to group start timestamp.
m_groupEndTimestamp = m_groupStartTimestamp;
@@ -1452,15 +1469,7 @@
m_groupStartTimestamp = MediaTime::invalidTime();
}
- // 1.4 If timestampOffset is not 0, then run the following steps:
- if (m_timestampOffset) {
- // 1.4.1 Add timestampOffset to the presentation timestamp.
- presentationTimestamp += m_timestampOffset;
-
- // 1.4.2 Add timestampOffset to the decode timestamp.
- decodeTimestamp += m_timestampOffset;
- }
-
+ // NOTE: this is out-of-order, but we need TrackBuffer to be able to cache the results of timestamp offset rounding
// 1.5 Let track buffer equal the track buffer that the coded frame will be added to.
AtomicString trackID = sample.trackID();
auto it = m_trackBufferMap.find(trackID);
@@ -1472,6 +1481,33 @@
}
TrackBuffer& trackBuffer = it->value;
+ MediaTime microsecond(1, 1000000);
+
+ auto roundTowardsTimeScaleWithRoundingMargin = [] (const MediaTime& time, uint32_t timeScale, const MediaTime& roundingMargin) {
+ while (true) {
+ MediaTime roundedTime = time.toTimeScale(timeScale);
+ if (abs(roundedTime - time) < roundingMargin || timeScale >= MediaTime::MaximumTimeScale)
+ return roundedTime;
+
+ if (!WTF::safeMultiply(timeScale, 2, timeScale) || timeScale > MediaTime::MaximumTimeScale)
+ timeScale = MediaTime::MaximumTimeScale;
+ }
+ };
+
+ // 1.4 If timestampOffset is not 0, then run the following steps:
+ if (m_timestampOffset) {
+ if (!trackBuffer.roundedTimestampOffset.isValid() || presentationTimestamp.timeScale() != trackBuffer.lastFrameTimescale) {
+ trackBuffer.lastFrameTimescale = presentationTimestamp.timeScale();
+ trackBuffer.roundedTimestampOffset = roundTowardsTimeScaleWithRoundingMargin(m_timestampOffset, trackBuffer.lastFrameTimescale, microsecond);
+ }
+
+ // 1.4.1 Add timestampOffset to the presentation timestamp.
+ presentationTimestamp += trackBuffer.roundedTimestampOffset;
+
+ // 1.4.2 Add timestampOffset to the decode timestamp.
+ decodeTimestamp += trackBuffer.roundedTimestampOffset;
+ }
+
// 1.6 ↳ If last decode timestamp for track buffer is set and decode timestamp is less than last
// decode timestamp:
// OR
@@ -1510,11 +1546,13 @@
if (m_mode == AppendMode::Sequence) {
// Use the generated timestamps instead of the sample's timestamps.
sample.setTimestamps(presentationTimestamp, decodeTimestamp);
- } else if (m_timestampOffset) {
+ } else if (trackBuffer.roundedTimestampOffset) {
// Reflect the timestamp offset into the sample.
- sample.offsetTimestampsBy(m_timestampOffset);
+ sample.offsetTimestampsBy(trackBuffer.roundedTimestampOffset);
}
+ LOG(MediaSourceSamples, "SourceBuffer::sourceBufferPrivateDidReceiveSample(%p) - sample(%s)", this, toString(sample).utf8().data());
+
// 1.7 Let frame end timestamp equal the sum of presentation timestamp and frame duration.
MediaTime frameEndTimestamp = presentationTimestamp + frameDuration;
@@ -1560,7 +1598,6 @@
// FIXME: Add support for sample splicing.
SampleMap erasedSamples;
- MediaTime microsecond(1, 1000000);
// 1.14 If last decode timestamp for track buffer is unset and presentation timestamp falls
// falls within the presentation interval of a coded frame in track buffer, then run the
@@ -1724,8 +1761,13 @@
m_groupEndTimestamp = frameEndTimestamp;
// 1.22 If generate timestamps flag equals true, then set timestampOffset equal to frame end timestamp.
- if (m_shouldGenerateTimestamps)
+ if (m_shouldGenerateTimestamps) {
m_timestampOffset = frameEndTimestamp;
+ for (auto& trackBuffer : m_trackBufferMap.values()) {
+ trackBuffer.lastFrameTimescale = 0;
+ trackBuffer.roundedTimestampOffset = MediaTime::invalidTime();
+ }
+ }
// Eliminate small gaps between buffered ranges by coalescing
// disjoint ranges separated by less than a "fudge factor".
