Diff
Modified: trunk/LayoutTests/ChangeLog (237375 => 237376)
--- trunk/LayoutTests/ChangeLog 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/LayoutTests/ChangeLog 2018-10-24 09:05:05 UTC (rev 237376)
@@ -1,3 +1,17 @@
+2018-10-23 Jer Noble <jer.no...@apple.com>
+
+ TextTrack cues should be updated more often than every 250ms.
+ https://bugs.webkit.org/show_bug.cgi?id=190827
+
+ Reviewed by Eric Carlson.
+
+ Add a LayoutTest which plays back 6s of captions, each 50 ms in duration, and uses
+ when the "enter" and "exit" were fired (in media time) to check whether they were missed
+ or not. The test succeeds if fewer than 50 of the 120 cues were missed.
+
+ * media/track/track-cue-missing-expected.txt: Added.
+ * media/track/track-cue-missing.html: Added.
+
2018-10-23 Ryan Haddad <ryanhad...@apple.com>
Unreviewed, rolling out r237261.
Added: trunk/LayoutTests/media/track/track-cue-missing-expected.txt (0 => 237376)
--- trunk/LayoutTests/media/track/track-cue-missing-expected.txt (rev 0)
+++ trunk/LayoutTests/media/track/track-cue-missing-expected.txt 2018-10-24 09:05:05 UTC (rev 237376)
@@ -0,0 +1,10 @@
+
+RUN(video.src = "" "../content/test"))
+RUN(track = video.addTextTrack("subtitles", "English", "en"))
+RUN(track.mode = "showing")
+EVENT(loadeddata)
+Created 121 cues.
+EVENT(ended)
+EXPECTED (missedCueCount < '50') OK
+END OF TEST
+
Added: trunk/LayoutTests/media/track/track-cue-missing.html (0 => 237376)
--- trunk/LayoutTests/media/track/track-cue-missing.html (rev 0)
+++ trunk/LayoutTests/media/track/track-cue-missing.html 2018-10-24 09:05:05 UTC (rev 237376)
@@ -0,0 +1,54 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <script src=""
+ <script src=""
+ <script>
+ window.addEventListener('load', async event => {
+ video = document.querySelector('video');
+
+ run('video.src = "" "../content/test")');
+
+ run('track = video.addTextTrack("subtitles", "English", "en")');
+ run('track.mode = "showing"');
+
+ await waitFor(video, 'loadeddata');
+
+ let cueStartTime = 0;
+ let cueDuration = 0.050;
+ let cueCount = 0;
+ missedCueCount = 0;
+
+ while (cueStartTime < video.duration) {
+ let cue = new VTTCue(cueStartTime, cueStartTime + cueDuration, `Cue ${ ++cueCount }`)
+ cueStartTime += cueDuration;
+
+ cue.addEventListener('enter', event => {
+ cue.enterFireTime = performance.now();
+ });
+ cue.addEventListener('exit', event => {
+ cue.exitFireTime = performance.now();
+ let eventDuration = cue.exitFireTime - cue.enterFireTime;
+ let cueDuration = cue.endTime - cue.startTime;
+
+ if (eventDuration * 2 < cueDuration)
+ ++missedCueCount;
+ });
+
+ track.addCue(cue);
+ }
+
+ consoleWrite(`Created ${ cueCount } cues.`);
+
+ video.play();
+ await waitFor(video, 'ended');
+
+ testExpected('missedCueCount', 50, '<');
+
+ endTest();
+ }, {once: true});
+ </script>
+</head>
+<body>
+ <video controls></video>
+</body>
\ No newline at end of file
Modified: trunk/Source/WebCore/ChangeLog (237375 => 237376)
--- trunk/Source/WebCore/ChangeLog 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/Source/WebCore/ChangeLog 2018-10-24 09:05:05 UTC (rev 237376)
@@ -1,3 +1,38 @@
+2018-10-23 Jer Noble <jer.no...@apple.com>
+
+ TextTrack cues should be updated more often than every 250ms.
+ https://bugs.webkit.org/show_bug.cgi?id=190827
+
+ Reviewed by Eric Carlson.
+
+ Test: media/track/track-cue-timing.html
+
+ TextTracks cues are recalculated on the playback timer, which fires at least every 250ms.
+ In addition to this timer, add a method to MediaPlayer to provide a task which will be
+ performed at a particular media time, and use this new method to request cues be updated
+ at the next interesting media time. The next interesting time would be either when the
+ soonest current cue will end, or when the next non-current cue will start, whichever is
+ earlier.
+
+ (Determining the "next non-current cue" requires new API on PODIntervalTree, as that class
+ does not have iterators per-se.)
+
+ * html/HTMLMediaElement.cpp:
+ (WebCore::compareCueIntervalEndTime):
+ (WebCore::HTMLMediaElement::updateActiveTextTrackCues):
+ * platform/PODIntervalTree.h:
+ * platform/graphics/MediaPlayer.cpp:
+ (WebCore::MediaPlayer::performTaskAtMediaTime):
+ * platform/graphics/MediaPlayer.h:
+ * platform/graphics/MediaPlayerPrivate.h:
+ (WebCore::MediaPlayerPrivateInterface::performTaskAtMediaTime):
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm:
+ (WebCore::MediaPlayerPrivateAVFoundationObjC::performTaskAtMediaTime):
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
+ (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::performTaskAtMediaTime):
+
2018-10-23 Fujii Hironori <hironori.fu...@sony.com>
[Win] Assertion fails while destructing local static AtomicString of FontCache::lastResortFallbackFont
Modified: trunk/Source/WebCore/html/HTMLMediaElement.cpp (237375 => 237376)
--- trunk/Source/WebCore/html/HTMLMediaElement.cpp 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/Source/WebCore/html/HTMLMediaElement.cpp 2018-10-24 09:05:05 UTC (rev 237376)
@@ -1657,6 +1657,11 @@
return one.data()->isOrderedBefore(two.data());
}
+static bool compareCueIntervalEndTime(const CueInterval& one, const CueInterval& two)
+{
+ return one.data()->endMediaTime() > two.data()->endMediaTime();
+}
+
void HTMLMediaElement::updateActiveTextTrackCues(const MediaTime& movieTime)
{
// 4.8.10.8 Playing the media resource
@@ -1676,8 +1681,9 @@
// The user agent must synchronously unset [the text track cue active] flag
// whenever ... the media element's readyState is changed back to HAVE_NOTHING.
+ auto movieTimeInterval = m_cueTree.createInterval(movieTime, movieTime);
if (m_readyState != HAVE_NOTHING && m_player) {
- currentCues = m_cueTree.allOverlaps(m_cueTree.createInterval(movieTime, movieTime));
+ currentCues = m_cueTree.allOverlaps(movieTimeInterval);
if (currentCues.size() > 1)
std::sort(currentCues.begin(), currentCues.end(), &compareCueInterval);
}
@@ -1746,6 +1752,21 @@
activeSetChanged = true;
}
+ MediaTime nextInterestingTime = MediaTime::invalidTime();
+ if (auto nearestEndingCue = std::min_element(currentCues.begin(), currentCues.end(), compareCueIntervalEndTime))
+ nextInterestingTime = nearestEndingCue->data()->endMediaTime();
+
+ std::optional<CueInterval> nextCue = m_cueTree.nextIntervalAfter(movieTimeInterval);
+ if (nextCue)
+ nextInterestingTime = std::min(nextInterestingTime, nextCue->low());
+
+ if (nextInterestingTime.isValid() && m_player) {
+ m_player->performTaskAtMediaTime([weakThis = makeWeakPtr(this), nextInterestingTime] {
+ if (weakThis)
+ weakThis->updateActiveTextTrackCues(weakThis->currentMediaTime());
+ }, nextInterestingTime);
+ }
+
if (!activeSetChanged)
return;
Modified: trunk/Source/WebCore/platform/PODIntervalTree.h (237375 => 237376)
--- trunk/Source/WebCore/platform/PODIntervalTree.h 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/Source/WebCore/platform/PODIntervalTree.h 2018-10-24 09:05:05 UTC (rev 237376)
@@ -30,6 +30,7 @@
#include "PODRedBlackTree.h"
#include <wtf/Assertions.h>
#include <wtf/Noncopyable.h>
+#include <wtf/Optional.h>
#include <wtf/Vector.h>
#include <wtf/text/ValueToString.h>
@@ -115,6 +116,15 @@
return IntervalType(low, high, data);
}
+ std::optional<IntervalType> nextIntervalAfter(const IntervalType& interval)
+ {
+ auto next = smallestNodeGreaterThanFrom(interval, this->root());
+ if (!next)
+ return std::nullopt;
+
+ return next->data();
+ }
+
bool checkInvariants() const override
{
if (!PODRedBlackTree<IntervalType>::checkInvariants())
@@ -165,6 +175,20 @@
searchForOverlapsFrom<AdapterType>(node->right(), adapter);
}
+ IntervalNode* smallestNodeGreaterThanFrom(const IntervalType& interval, IntervalNode* node) const
+ {
+ if (!node)
+ return nullptr;
+
+ if (!(interval.high() < node->data().low()))
+ return smallestNodeGreaterThanFrom(interval, node->right());
+
+ if (auto left = smallestNodeGreaterThanFrom(interval, node->right()))
+ return left;
+
+ return node;
+}
+
bool updateNode(IntervalNode* node) override
{
// Would use const T&, but need to reassign this reference in this
Modified: trunk/Source/WebCore/platform/graphics/MediaPlayer.cpp (237375 => 237376)
--- trunk/Source/WebCore/platform/graphics/MediaPlayer.cpp 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/Source/WebCore/platform/graphics/MediaPlayer.cpp 2018-10-24 09:05:05 UTC (rev 237376)
@@ -1537,6 +1537,11 @@
#endif
+bool MediaPlayer::performTaskAtMediaTime(WTF::Function<void()>&& task, MediaTime time)
+{
+ return m_private->performTaskAtMediaTime(WTFMove(task), time);
+}
+
#if !RELEASE_LOG_DISABLED
const Logger& MediaPlayer::mediaPlayerLogger()
{
Modified: trunk/Source/WebCore/platform/graphics/MediaPlayer.h (237375 => 237376)
--- trunk/Source/WebCore/platform/graphics/MediaPlayer.h 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/Source/WebCore/platform/graphics/MediaPlayer.h 2018-10-24 09:05:05 UTC (rev 237376)
@@ -571,6 +571,8 @@
WEBCORE_EXPORT AVPlayer *objCAVFoundationAVPlayer() const;
#endif
+ bool performTaskAtMediaTime(WTF::Function<void()>&&, MediaTime);
+
private:
MediaPlayer(MediaPlayerClient&);
Modified: trunk/Source/WebCore/platform/graphics/MediaPlayerPrivate.h (237375 => 237376)
--- trunk/Source/WebCore/platform/graphics/MediaPlayerPrivate.h 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/Source/WebCore/platform/graphics/MediaPlayerPrivate.h 2018-10-24 09:05:05 UTC (rev 237376)
@@ -281,6 +281,8 @@
#if USE(AVFOUNDATION)
virtual AVPlayer *objCAVFoundationAVPlayer() const { return nullptr; }
#endif
+
+ virtual bool performTaskAtMediaTime(WTF::Function<void()>&&, MediaTime) { return false; }
};
}
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h (237375 => 237376)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h 2018-10-24 09:05:05 UTC (rev 237376)
@@ -330,6 +330,8 @@
AVPlayer *objCAVFoundationAVPlayer() const final { return m_avPlayer.get(); }
+ bool performTaskAtMediaTime(WTF::Function<void()>&&, MediaTime) final;
+
RetainPtr<AVURLAsset> m_avAsset;
RetainPtr<AVPlayer> m_avPlayer;
RetainPtr<AVPlayerItem> m_avPlayerItem;
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm (237375 => 237376)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm 2018-10-24 09:05:05 UTC (rev 237376)
@@ -3288,6 +3288,19 @@
#endif
}
+bool MediaPlayerPrivateAVFoundationObjC::performTaskAtMediaTime(WTF::Function<void()>&& task, MediaTime time)
+{
+ if (!m_avPlayer)
+ return false;
+
+ __block WTF::Function<void()> taskIn = WTFMove(task);
+
+ [m_avPlayer addBoundaryTimeObserverForTimes:@[[NSValue valueWithCMTime:toCMTime(time)]] queue:dispatch_get_main_queue() usingBlock:^{
+ taskIn();
+ }];
+ return true;
+}
+
NSArray* assetMetadataKeyNames()
{
static NSArray* keys = [[NSArray alloc] initWithObjects:
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h (237375 => 237376)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h 2018-10-24 09:05:05 UTC (rev 237376)
@@ -234,6 +234,8 @@
bool wirelessVideoPlaybackDisabled() const override { return false; }
#endif
+ bool performTaskAtMediaTime(WTF::Function<void()>&&, MediaTime) final;
+
void ensureLayer();
void destroyLayer();
void ensureDecompressionSession();
@@ -272,6 +274,7 @@
ALLOW_NEW_API_WITHOUT_GUARDS_END
RetainPtr<id> m_timeJumpedObserver;
RetainPtr<id> m_durationObserver;
+ RetainPtr<id> m_performTaskObserver;
RetainPtr<AVStreamSession> m_streamSession;
RetainPtr<CVPixelBufferRef> m_lastPixelBuffer;
RetainPtr<CGImageRef> m_lastImage;
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm (237375 => 237376)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm 2018-10-24 04:58:27 UTC (rev 237375)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm 2018-10-24 09:05:05 UTC (rev 237376)
@@ -1110,6 +1110,16 @@
}
#endif
+bool MediaPlayerPrivateMediaSourceAVFObjC::performTaskAtMediaTime(WTF::Function<void()>&& task, MediaTime time)
+{
+ __block WTF::Function<void()> taskIn = WTFMove(task);
+
+ [m_synchronizer addBoundaryTimeObserverForTimes:@[[NSValue valueWithCMTime:toCMTime(time)]] queue:dispatch_get_main_queue() usingBlock:^{
+ taskIn();
+ }];
+ return true;
}
+}
+
#endif
