Diff
Modified: trunk/LayoutTests/ChangeLog (285983 => 285984)
--- trunk/LayoutTests/ChangeLog 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/LayoutTests/ChangeLog 2021-11-18 07:46:05 UTC (rev 285984)
@@ -1,3 +1,14 @@
+2021-11-17 Youenn Fablet <you...@apple.com>
+
+ Add support for more rvfc metadata
+ https://bugs.webkit.org/show_bug.cgi?id=233185
+
+ Reviewed by Eric Carlson.
+
+ * platform/ios-wk2/TestExpectations:
+ * platform/mac-wk1/TestExpectations:
+ * platform/mac-wk2/TestExpectations:
+
2021-11-17 Ricky Mondello <rmonde...@apple.com>
When an "autofilled and viewable" field becomes empty, turn "autofilled and viewable" off
Modified: trunk/LayoutTests/imported/w3c/ChangeLog (285983 => 285984)
--- trunk/LayoutTests/imported/w3c/ChangeLog 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/LayoutTests/imported/w3c/ChangeLog 2021-11-18 07:46:05 UTC (rev 285984)
@@ -1,3 +1,12 @@
+2021-11-17 Youenn Fablet <you...@apple.com>
+
+ Add support for more rvfc metadata
+ https://bugs.webkit.org/show_bug.cgi?id=233185
+
+ Reviewed by Eric Carlson.
+
+ * web-platform-tests/video-rvfc/request-video-frame-callback-webrtc.https-expected.txt:
+
2021-11-17 Chris Dumez <cdu...@apple.com>
Web Locks API does get enabled in Service Workers when running layout tests
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-webrtc.https-expected.txt (285983 => 285984)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-webrtc.https-expected.txt 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-webrtc.https-expected.txt 2021-11-18 07:46:05 UTC (rev 285984)
@@ -1,4 +1,4 @@
-FAIL Test video.requestVideoFrameCallback() parameters for WebRTC applications. assert_greater_than: expected a number but got a "undefined"
+PASS Test video.requestVideoFrameCallback() parameters for WebRTC applications.
Modified: trunk/Source/WebCore/ChangeLog (285983 => 285984)
--- trunk/Source/WebCore/ChangeLog 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/ChangeLog 2021-11-18 07:46:05 UTC (rev 285984)
@@ -1,3 +1,55 @@
+2021-11-17 Youenn Fablet <you...@apple.com>
+
+ Add support for more rvfc metadata
+ https://bugs.webkit.org/show_bug.cgi?id=233185
+
+ Reviewed by Eric Carlson.
+
+ Introduce VideoSampleMetadata that allows a RealtimeMediaSource to provide metadata for each video frame.
+ Fill VideoSampleMetadata for each source (capture and WebRTC)
+ Make use of VideoSampleMetadata in MediaPlayerPrivateMediaStreamAVFObjC to fill VideoFrameMetadata.
+ Enhance MediaPlayerPrivateAVFoundationObjC to fill in the newly exposed metadata in VideoFrameMetadata.
+ Update HTMLVideoElement to process the timestamp values as needed.
+
+ Covered by existing tests.
+
+ * Headers.cmake:
+ * Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp:
+ * WebCore.xcodeproj/project.pbxproj:
+ * html/HTMLVideoElement.cpp:
+ * html/VideoFrameMetadata.idl:
+ * platform/VideoSampleMetadata.h: Added.
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
+ * platform/mediarecorder/MediaRecorderPrivateAVFImpl.cpp:
+ * platform/mediarecorder/MediaRecorderPrivateAVFImpl.h:
+ * platform/mediarecorder/MediaRecorderPrivateMock.cpp:
+ * platform/mediarecorder/MediaRecorderPrivateMock.h:
+ * platform/mediastream/RealtimeIncomingVideoSource.cpp:
+ * platform/mediastream/RealtimeIncomingVideoSource.h:
+ * platform/mediastream/RealtimeMediaSource.cpp:
+ * platform/mediastream/RealtimeMediaSource.h:
+ * platform/mediastream/RealtimeOutgoingVideoSource.h:
+ * platform/mediastream/RealtimeVideoCaptureSource.cpp:
+ * platform/mediastream/RealtimeVideoCaptureSource.h:
+ * platform/mediastream/RealtimeVideoSource.cpp:
+ * platform/mediastream/RealtimeVideoSource.h:
+ * platform/mediastream/cocoa/DisplayCaptureSourceCocoa.cpp:
+ * platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp:
+ * platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp:
+ * platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.cpp:
+ * platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.h:
+ * platform/mediastream/libwebrtc/gstreamer/RealtimeIncomingVideoSourceLibWebRTC.cpp:
+ * platform/mediastream/libwebrtc/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.cpp:
+ * platform/mediastream/libwebrtc/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.h:
+ * platform/mediastream/mac/AVVideoCaptureSource.mm:
+ * platform/mediastream/mac/MockRealtimeVideoSourceMac.mm:
+ * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:
+ * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp:
+ * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h:
+ * testing/Internals.cpp:
+ * testing/Internals.h:
+
2021-11-17 Ricky Mondello <rmonde...@apple.com>
When an "autofilled and viewable" field becomes empty, turn "autofilled and viewable" off
Modified: trunk/Source/WebCore/Headers.cmake (285983 => 285984)
--- trunk/Source/WebCore/Headers.cmake 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/Headers.cmake 2021-11-18 07:46:05 UTC (rev 285984)
@@ -1190,6 +1190,7 @@
platform/UserInterfaceLayoutDirection.h
platform/ValidationBubble.h
platform/VideoFrameMetadata.h
+ platform/VideoSampleMetadata.h
platform/WebGLStateTracker.h
platform/Widget.h
platform/WindowsKeyboardCodes.h
Modified: trunk/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp (285983 => 285984)
--- trunk/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -196,7 +196,9 @@
if (!sample)
return;
- videoSampleAvailable(*sample);
+ VideoSampleMetadata metadata;
+ metadata.captureTime = MonotonicTime::now().secondsSinceEpoch();
+ videoSampleAvailable(*sample, metadata);
}
RefPtr<MediaStreamTrack> CanvasCaptureMediaStreamTrack::clone()
Modified: trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj (285983 => 285984)
--- trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2021-11-18 07:46:05 UTC (rev 285984)
@@ -1118,6 +1118,7 @@
4157EBFB1E3AB67F00AC9FE9 /* MockLibWebRTCPeerConnection.h in Headers */ = {isa = PBXBuildFile; fileRef = 4157EBF81E3AB06800AC9FE9 /* MockLibWebRTCPeerConnection.h */; };
415864A023BF7CBF00A0A61E /* RealtimeVideoUtilities.h in Headers */ = {isa = PBXBuildFile; fileRef = 41D1938F2152C561006F14CA /* RealtimeVideoUtilities.h */; settings = {ATTRIBUTES = (Private, ); }; };
415CDAF51E6B8F8B004F11EE /* CanvasCaptureMediaStreamTrack.h in Headers */ = {isa = PBXBuildFile; fileRef = 41C7E1061E6A54360027B4DE /* CanvasCaptureMediaStreamTrack.h */; };
+ 416049332743B0E800A86FA0 /* VideoSampleMetadata.h in Headers */ = {isa = PBXBuildFile; fileRef = 416049312743B0E700A86FA0 /* VideoSampleMetadata.h */; settings = {ATTRIBUTES = (Private, ); }; };
41614A791DA64241004AD06F /* HTTPHeaderValues.h in Headers */ = {isa = PBXBuildFile; fileRef = 41614A771DA64236004AD06F /* HTTPHeaderValues.h */; settings = {ATTRIBUTES = (Private, ); }; };
4161E2D51FE48DC500EC2E96 /* FetchLoader.h in Headers */ = {isa = PBXBuildFile; fileRef = 4147E2B51C89912600A7E715 /* FetchLoader.h */; settings = {ATTRIBUTES = (Private, ); }; };
4162A451101145AE00DFF3ED /* DedicatedWorkerGlobalScope.h in Headers */ = {isa = PBXBuildFile; fileRef = 4162A44E101145AE00DFF3ED /* DedicatedWorkerGlobalScope.h */; };
@@ -8340,6 +8341,7 @@
415E1BB5215010810022DA96 /* RTCRtpContributingSource.idl */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = RTCRtpContributingSource.idl; sourceTree = "<group>"; };
415E1BB62150152A0022DA96 /* RTCRtpSynchronizationSource.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RTCRtpSynchronizationSource.h; sourceTree = "<group>"; };
415E1BB7215015300022DA96 /* RTCRtpSynchronizationSource.idl */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = RTCRtpSynchronizationSource.idl; sourceTree = "<group>"; };
+ 416049312743B0E700A86FA0 /* VideoSampleMetadata.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = VideoSampleMetadata.h; sourceTree = "<group>"; };
41614A761DA64236004AD06F /* HTTPHeaderValues.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = HTTPHeaderValues.cpp; sourceTree = "<group>"; };
41614A771DA64236004AD06F /* HTTPHeaderValues.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HTTPHeaderValues.h; sourceTree = "<group>"; };
4162A44D101145AE00DFF3ED /* DedicatedWorkerGlobalScope.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = DedicatedWorkerGlobalScope.cpp; sourceTree = "<group>"; };
@@ -29158,6 +29160,7 @@
71C916071D1483A300ACA47D /* UserInterfaceLayoutDirection.h */,
83C45B8D1DC2B67C008871BA /* ValidationBubble.h */,
41DEEFB42719BA1900CB8D74 /* VideoFrameMetadata.h */,
+ 416049312743B0E700A86FA0 /* VideoSampleMetadata.h */,
515F79511CFCA3C700CCED93 /* WebCoreCrossThreadCopier.cpp */,
515F79521CFCA3C700CCED93 /* WebCoreCrossThreadCopier.h */,
839A2F2B1E204A6D0039057E /* WebGLStateTracker.cpp */,
@@ -36733,6 +36736,7 @@
CDE83DB2183C44060031EAA3 /* VideoPlaybackQuality.h in Headers */,
075033A8252BD36800F70CE3 /* VideoPlaybackQualityMetrics.h in Headers */,
0757B13E214AE79900794B0D /* VideoPreset.h in Headers */,
+ 416049332743B0E800A86FA0 /* VideoSampleMetadata.h in Headers */,
CDC939A81E9BDFB100BB768D /* VideoToolboxSoftLink.h in Headers */,
BE88E0DF1715D2A200658D98 /* VideoTrack.h in Headers */,
CD1F9B1827023A2A00617EB6 /* VideoTrackClient.h in Headers */,
Modified: trunk/Source/WebCore/html/HTMLVideoElement.cpp (285983 => 285984)
--- trunk/Source/WebCore/html/HTMLVideoElement.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/html/HTMLVideoElement.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -41,6 +41,7 @@
#include "ImageBuffer.h"
#include "Logging.h"
#include "Page.h"
+#include "Performance.h"
#include "PictureInPictureSupport.h"
#include "RenderImage.h"
#include "RenderVideo.h"
@@ -609,6 +610,16 @@
m_videoFrameRequests.remove(index);
}
+static void processVideoFrameMetadataTimestamps(VideoFrameMetadata& metadata, Performance& performance)
+{
+ metadata.presentationTime = performance.relativeTimeFromTimeOriginInReducedResolution(MonotonicTime::fromRawSeconds(metadata.presentationTime));
+ metadata.expectedDisplayTime = performance.relativeTimeFromTimeOriginInReducedResolution(MonotonicTime::fromRawSeconds(metadata.expectedDisplayTime));
+ if (metadata.captureTime)
+ metadata.captureTime = performance.relativeTimeFromTimeOriginInReducedResolution(MonotonicTime::fromRawSeconds(*metadata.captureTime));
+ if (metadata.receiveTime)
+ metadata.receiveTime = performance.relativeTimeFromTimeOriginInReducedResolution(MonotonicTime::fromRawSeconds(*metadata.receiveTime));
+}
+
void HTMLVideoElement::serviceRequestVideoFrameCallbacks(ReducedResolutionSeconds now)
{
if (!player())
@@ -615,9 +626,11 @@
return;
auto videoFrameMetadata = player()->videoFrameMetadata();
- if (!videoFrameMetadata)
+ if (!videoFrameMetadata || !document().domWindow())
return;
+ processVideoFrameMetadataTimestamps(*videoFrameMetadata, document().domWindow()->performance());
+
Ref protectedThis { *this };
// We store the size before calling callbacks as we do not want to call newly added callbacks.
Modified: trunk/Source/WebCore/html/VideoFrameMetadata.idl (285983 => 285984)
--- trunk/Source/WebCore/html/VideoFrameMetadata.idl 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/html/VideoFrameMetadata.idl 2021-11-18 07:46:05 UTC (rev 285984)
@@ -29,9 +29,8 @@
Conditional=VIDEO,
JSGenerateToJSObject,
] dictionary VideoFrameMetadata {
- // FIXME: expose presentationTime and expectedDisplayTime.
- // required DOMHighResTimeStamp presentationTime;
- // required DOMHighResTimeStamp expectedDisplayTime;
+ required DOMHighResTimeStamp presentationTime;
+ required DOMHighResTimeStamp expectedDisplayTime;
required unsigned long width;
required unsigned long height;
Added: trunk/Source/WebCore/platform/VideoSampleMetadata.h (0 => 285984)
--- trunk/Source/WebCore/platform/VideoSampleMetadata.h (rev 0)
+++ trunk/Source/WebCore/platform/VideoSampleMetadata.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <optional>
+#include <wtf/Seconds.h>
+
+namespace WebCore {
+
+struct VideoSampleMetadata {
+ std::optional<double> processingDuration;
+
+ std::optional<Seconds> captureTime;
+ std::optional<Seconds> receiveTime;
+ std::optional<unsigned> rtpTimestamp;
+
+ template<class Encoder> void encode(Encoder&) const;
+ template<class Decoder> static std::optional<VideoSampleMetadata> decode(Decoder&);
+};
+
+template<class Encoder>
+inline void VideoSampleMetadata::encode(Encoder& encoder) const
+{
+ encoder << processingDuration << captureTime << receiveTime << rtpTimestamp;
+}
+
+template<class Decoder>
+inline std::optional<VideoSampleMetadata> VideoSampleMetadata::decode(Decoder& decoder)
+{
+ std::optional<std::optional<double>> processingDuration;
+ decoder >> processingDuration;
+ if (!processingDuration)
+ return std::nullopt;
+
+ std::optional<std::optional<Seconds>> captureTime;
+ decoder >> captureTime;
+ if (!captureTime)
+ return std::nullopt;
+
+ std::optional<std::optional<Seconds>> receiveTime;
+ decoder >> receiveTime;
+ if (!receiveTime)
+ return std::nullopt;
+
+ std::optional<std::optional<unsigned>> rtpTimestamp;
+ decoder >> rtpTimestamp;
+ if (!rtpTimestamp)
+ return std::nullopt;
+
+ return VideoSampleMetadata { *processingDuration, *captureTime, *receiveTime, *rtpTimestamp };
+}
+
+}
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h (285983 => 285984)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -136,7 +136,7 @@
void flushRenderers();
- void processNewVideoSample(MediaSample&, bool hasChangedOrientation);
+ void processNewVideoSample(MediaSample&, bool hasChangedOrientation, VideoSampleMetadata, Seconds);
void enqueueVideoSample(MediaSample&);
void requestNotificationWhenReadyForVideoData();
@@ -208,7 +208,7 @@
void readyStateChanged(MediaStreamTrackPrivate&) override;
// RealtimeMediaSouce::VideoSampleObserver
- void videoSampleAvailable(MediaSample&) final;
+ void videoSampleAvailable(MediaSample&, VideoSampleMetadata) final;
RetainPtr<PlatformLayer> createVideoFullscreenLayer() override;
void setVideoFullscreenLayer(PlatformLayer*, Function<void()>&& completionHandler) override;
@@ -283,6 +283,8 @@
uint64_t m_sampleCount { 0 };
uint64_t m_lastVideoFrameMetadataSampleCount { 0 };
+ Seconds m_presentationTime { 0 };
+ VideoSampleMetadata m_sampleMetadata;
};
}
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm (285983 => 285984)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm 2021-11-18 07:46:05 UTC (rev 285984)
@@ -244,9 +244,10 @@
return videoTransform;
}
-void MediaPlayerPrivateMediaStreamAVFObjC::videoSampleAvailable(MediaSample& sample)
+void MediaPlayerPrivateMediaStreamAVFObjC::videoSampleAvailable(MediaSample& sample, VideoSampleMetadata metadata)
{
- processNewVideoSample(sample, sample.videoRotation() != m_videoRotation || sample.videoMirrored() != m_videoMirrored);
+ auto presentationTime = MonotonicTime::now().secondsSinceEpoch();
+ processNewVideoSample(sample, sample.videoRotation() != m_videoRotation || sample.videoMirrored() != m_videoMirrored, metadata, presentationTime);
enqueueVideoSample(sample);
}
@@ -276,7 +277,7 @@
m_sampleBufferDisplayLayer->enqueueSample(sample);
}
-void MediaPlayerPrivateMediaStreamAVFObjC::processNewVideoSample(MediaSample& sample, bool hasChangedOrientation)
+void MediaPlayerPrivateMediaStreamAVFObjC::processNewVideoSample(MediaSample& sample, bool hasChangedOrientation, VideoSampleMetadata metadata, Seconds presentationTime)
{
if (!isMainThread()) {
{
@@ -283,7 +284,7 @@
Locker locker { m_currentVideoSampleLock };
m_currentVideoSample = &sample;
}
- callOnMainThread([weakThis = WeakPtr { *this }, hasChangedOrientation]() mutable {
+ callOnMainThread([weakThis = WeakPtr { *this }, hasChangedOrientation, metadata, presentationTime]() mutable {
if (!weakThis)
return;
@@ -298,7 +299,7 @@
if (!sample)
return;
- weakThis->processNewVideoSample(*sample, hasChangedOrientation);
+ weakThis->processNewVideoSample(*sample, hasChangedOrientation, metadata, presentationTime);
});
return;
}
@@ -316,6 +317,8 @@
updateReadyState();
}
+ m_presentationTime = presentationTime;
+ m_sampleMetadata = metadata;
++m_sampleCount;
if (m_displayMode != LivePreview && !m_waitingForFirstImage)
@@ -1106,9 +1109,15 @@
metadata.width = m_intrinsicSize.width();
metadata.height = m_intrinsicSize.height();
metadata.presentedFrames = m_sampleCount;
+ metadata.presentationTime = m_presentationTime.seconds();
+ metadata.expectedDisplayTime = m_presentationTime.seconds();
+ metadata.processingDuration = m_sampleMetadata.processingDuration;
+ if (m_sampleMetadata.captureTime)
+ metadata.captureTime = m_sampleMetadata.captureTime->seconds();
+ if (m_sampleMetadata.receiveTime)
+ metadata.receiveTime = m_sampleMetadata.receiveTime->seconds();
+ metadata.rtpTimestamp = m_sampleMetadata.rtpTimestamp;
- // FIXME: It would be good to expose more video frame metadata.
-
return metadata;
}
Modified: trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -83,7 +83,7 @@
callback(String(m_writer->mimeType()), m_writer->audioBitRate(), m_writer->videoBitRate());
}
-void MediaRecorderPrivateAVFImpl::videoSampleAvailable(MediaSample& sampleBuffer)
+void MediaRecorderPrivateAVFImpl::videoSampleAvailable(MediaSample& sampleBuffer, VideoSampleMetadata)
{
if (shouldMuteVideo()) {
if (!m_blackFrame) {
Modified: trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.h (285983 => 285984)
--- trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -49,7 +49,7 @@
explicit MediaRecorderPrivateAVFImpl(Ref<MediaRecorderPrivateWriter>&&);
// MediaRecorderPrivate
- void videoSampleAvailable(MediaSample&) final;
+ void videoSampleAvailable(MediaSample&, VideoSampleMetadata) final;
void fetchData(FetchDataCallback&&) final;
void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t) final;
void startRecording(StartRecordingCallback&&) final;
Modified: trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -67,7 +67,7 @@
completionHandler();
}
-void MediaRecorderPrivateMock::videoSampleAvailable(MediaSample&)
+void MediaRecorderPrivateMock::videoSampleAvailable(MediaSample&, VideoSampleMetadata)
{
Locker locker { m_bufferLock };
m_buffer.append("Video Track ID: ");
Modified: trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.h (285983 => 285984)
--- trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -42,7 +42,7 @@
private:
// MediaRecorderPrivate
- void videoSampleAvailable(MediaSample&) final;
+ void videoSampleAvailable(MediaSample&, VideoSampleMetadata) final;
void fetchData(FetchDataCallback&&) final;
void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t) final;
void stopRecording(CompletionHandler<void()>&&) final;
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeIncomingVideoSource.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/RealtimeIncomingVideoSource.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeIncomingVideoSource.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -104,6 +104,20 @@
m_currentSettings = std::nullopt;
}
+VideoSampleMetadata RealtimeIncomingVideoSource::metadataFromVideoFrame(const webrtc::VideoFrame& frame)
+{
+ VideoSampleMetadata metadata;
+ if (frame.ntp_time_ms() > 0)
+ metadata.captureTime = Seconds::fromMilliseconds(frame.ntp_time_ms());
+ if (isInBounds<uint32_t>(frame.timestamp()))
+ metadata.rtpTimestamp = frame.timestamp();
+ auto lastPacketTimestamp = std::max_element(frame.packet_infos().cbegin(), frame.packet_infos().cend(), [](const auto& a, const auto& b) {
+ return a.receive_time() < b.receive_time();
+ });
+ metadata.receiveTime = Seconds::fromMicroseconds(lastPacketTimestamp->receive_time().us());
+ return metadata;
+}
+
} // namespace WebCore
#endif // USE(LIBWEBRTC)
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeIncomingVideoSource.h (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/RealtimeIncomingVideoSource.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeIncomingVideoSource.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -62,7 +62,9 @@
#if !RELEASE_LOG_DISABLED
const char* logClassName() const final { return "RealtimeIncomingVideoSource"; }
#endif
-
+
+ static VideoSampleMetadata metadataFromVideoFrame(const webrtc::VideoFrame&);
+
private:
// RealtimeMediaSource API
void startProducingData() final;
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -191,7 +191,7 @@
});
}
-void RealtimeMediaSource::videoSampleAvailable(MediaSample& mediaSample)
+void RealtimeMediaSource::videoSampleAvailable(MediaSample& mediaSample, VideoSampleMetadata metadata)
{
#if !RELEASE_LOG_DISABLED
++m_frameCount;
@@ -211,7 +211,7 @@
Locker locker { m_videoSampleObserversLock };
for (auto* observer : m_videoSampleObservers)
- observer->videoSampleAvailable(mediaSample);
+ observer->videoSampleAvailable(mediaSample, metadata);
}
void RealtimeMediaSource::audioSamplesAvailable(const MediaTime& time, const PlatformAudioData& audioData, const AudioStreamDescription& description, size_t numberOfFrames)
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.h (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -42,6 +42,7 @@
#include "PlatformLayer.h"
#include "RealtimeMediaSourceCapabilities.h"
#include "RealtimeMediaSourceFactory.h"
+#include "VideoSampleMetadata.h"
#include <wtf/CompletionHandler.h>
#include <wtf/Lock.h>
#include <wtf/LoggerHelper.h>
@@ -102,7 +103,7 @@
virtual ~VideoSampleObserver() = default;
// May be called on a background thread.
- virtual void videoSampleAvailable(MediaSample&) = 0;
+ virtual void videoSampleAvailable(MediaSample&, VideoSampleMetadata) = 0;
};
virtual ~RealtimeMediaSource() = default;
@@ -244,7 +245,7 @@
void initializeSampleRate(int sampleRate) { m_sampleRate = sampleRate; }
void initializeEchoCancellation(bool echoCancellation) { m_echoCancellation = echoCancellation; }
- void videoSampleAvailable(MediaSample&);
+ void videoSampleAvailable(MediaSample&, VideoSampleMetadata);
void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t);
void forEachObserver(const Function<void(Observer&)>&);
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.h (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -135,7 +135,7 @@
void trackEnded(MediaStreamTrackPrivate&) final { }
// RealtimeMediaSource::VideoSampleObserver API
- void videoSampleAvailable(MediaSample&) override { }
+ void videoSampleAvailable(MediaSample&, VideoSampleMetadata) override { }
Ref<MediaStreamTrackPrivate> m_videoSource;
Timer m_blackFrameTimer;
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -386,7 +386,7 @@
setFrameRate(match->requestedFrameRate);
}
-void RealtimeVideoCaptureSource::dispatchMediaSampleToObservers(MediaSample& sample)
+void RealtimeVideoCaptureSource::dispatchMediaSampleToObservers(MediaSample& sample, WebCore::VideoSampleMetadata metadata)
{
MediaTime sampleTime = sample.presentationTime();
@@ -400,7 +400,7 @@
if (interval > 1)
m_observedFrameRate = (m_observedFrameTimeStamps.size() / interval);
- videoSampleAvailable(sample);
+ videoSampleAvailable(sample, metadata);
}
void RealtimeVideoCaptureSource::clientUpdatedSizeAndFrameRate(std::optional<int> width, std::optional<int> height, std::optional<double> frameRate)
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.h (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -72,7 +72,7 @@
void updateCapabilities(RealtimeMediaSourceCapabilities&);
- void dispatchMediaSampleToObservers(MediaSample&);
+ void dispatchMediaSampleToObservers(MediaSample&, WebCore::VideoSampleMetadata);
const Vector<IntSize>& standardVideoSizes();
private:
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -179,7 +179,7 @@
}
#endif
-void RealtimeVideoSource::videoSampleAvailable(MediaSample& sample)
+void RealtimeVideoSource::videoSampleAvailable(MediaSample& sample, VideoSampleMetadata metadata)
{
if (m_frameDecimation > 1 && ++m_frameDecimationCounter % m_frameDecimation)
return;
@@ -193,13 +193,13 @@
auto size = this->size();
if (!size.isEmpty() && size != expandedIntSize(sample.presentationSize())) {
if (auto mediaSample = adaptVideoSample(sample)) {
- RealtimeMediaSource::videoSampleAvailable(*mediaSample);
+ RealtimeMediaSource::videoSampleAvailable(*mediaSample, metadata);
return;
}
}
#endif
- RealtimeMediaSource::videoSampleAvailable(sample);
+ RealtimeMediaSource::videoSampleAvailable(sample, metadata);
}
Ref<RealtimeMediaSource> RealtimeVideoSource::clone()
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.h (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -72,7 +72,7 @@
bool preventSourceFromStopping() final;
// RealtimeMediaSource::VideoSampleObserver
- void videoSampleAvailable(MediaSample&) final;
+ void videoSampleAvailable(MediaSample&, VideoSampleMetadata) final;
#if PLATFORM(COCOA)
RefPtr<MediaSample> adaptVideoSample(MediaSample&);
Modified: trunk/Source/WebCore/platform/mediastream/cocoa/DisplayCaptureSourceCocoa.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/cocoa/DisplayCaptureSourceCocoa.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/cocoa/DisplayCaptureSourceCocoa.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -220,7 +220,8 @@
if (!m_imageTransferSession)
m_imageTransferSession = ImageTransferSessionVT::create(preferedPixelBufferFormat());
- auto sampleTime = MediaTime::createWithDouble((elapsedTime() + 100_ms).seconds());
+ auto elapsedTime = this->elapsedTime();
+ auto sampleTime = MediaTime::createWithDouble((elapsedTime + 100_ms).seconds());
auto frame = m_capturer->generateFrame();
auto imageSize = WTF::switchOn(frame,
@@ -282,7 +283,9 @@
return;
}
- videoSampleAvailable(*sample.get());
+ VideoSampleMetadata metadata;
+ metadata.captureTime = MonotonicTime::now().secondsSinceEpoch();
+ videoSampleAvailable(*sample.get(), metadata);
}
void DisplayCaptureSourceCocoa::setLogger(const Logger& logger, const void* identifier)
Modified: trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -262,7 +262,7 @@
}
}
- void videoSampleAvailable(MediaSample& sample) final
+ void videoSampleAvailable(MediaSample& sample, VideoSampleMetadata) final
{
if (!m_parent)
return;
Modified: trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -195,7 +195,7 @@
if (!isProducingData() || muted())
return;
- dispatchMediaSampleToObservers(WTFMove(sample));
+ dispatchMediaSampleToObservers(WTFMove(sample), { });
}
GstFlowReturn GStreamerVideoCaptureSource::newSampleCallback(GstElement* sink, GStreamerVideoCaptureSource* source)
Modified: trunk/Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -88,9 +88,9 @@
m_source->requestToEnd(callingObserver);
}
-void MockDisplayCaptureSourceGStreamer::videoSampleAvailable(MediaSample& sample)
+void MockDisplayCaptureSourceGStreamer::videoSampleAvailable(MediaSample& sample, VideoSampleMetadata metadata)
{
- RealtimeMediaSource::videoSampleAvailable(sample);
+ RealtimeMediaSource::videoSampleAvailable(sample, metadata);
}
const RealtimeMediaSourceCapabilities& MockDisplayCaptureSourceGStreamer::capabilities()
@@ -154,7 +154,7 @@
auto sample = MediaSampleGStreamer::createImageSample(WTFMove(*pixelBuffer), size(), frameRate(), sampleRotation());
sample->offsetTimestampsBy(MediaTime::createWithDouble((elapsedTime() + 100_ms).seconds()));
- dispatchMediaSampleToObservers(sample.get());
+ dispatchMediaSampleToObservers(sample.get(), { });
}
} // namespace WebCore
Modified: trunk/Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.h (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -48,7 +48,7 @@
protected:
// RealtimeMediaSource::VideoSampleObserver
- void videoSampleAvailable(MediaSample&) final;
+ void videoSampleAvailable(MediaSample&, VideoSampleMetadata) final;
private:
MockDisplayCaptureSourceGStreamer(Ref<MockRealtimeVideoSourceGStreamer>&&, CaptureDevice::DeviceType);
Modified: trunk/Source/WebCore/platform/mediastream/libwebrtc/gstreamer/RealtimeIncomingVideoSourceLibWebRTC.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/libwebrtc/gstreamer/RealtimeIncomingVideoSourceLibWebRTC.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/libwebrtc/gstreamer/RealtimeIncomingVideoSourceLibWebRTC.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -61,7 +61,7 @@
callOnMainThread([protectedThis = Ref { *this }, frame] {
auto gstSample = GStreamerSampleFromLibWebRTCVideoFrame(frame);
auto sample = MediaSampleGStreamer::create(WTFMove(gstSample), { }, { });
- protectedThis->videoSampleAvailable(sample.get());
+ protectedThis->videoSampleAvailable(sample.get(), { });
});
}
Modified: trunk/Source/WebCore/platform/mediastream/libwebrtc/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/libwebrtc/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/libwebrtc/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -50,7 +50,7 @@
{
}
-void RealtimeOutgoingVideoSourceLibWebRTC::videoSampleAvailable(MediaSample& sample)
+void RealtimeOutgoingVideoSourceLibWebRTC::videoSampleAvailable(MediaSample& sample, VideoSampleMetadata)
{
switch (sample.videoRotation()) {
case MediaSample::VideoRotation::None:
Modified: trunk/Source/WebCore/platform/mediastream/libwebrtc/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.h (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/libwebrtc/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/libwebrtc/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -42,7 +42,7 @@
rtc::scoped_refptr<webrtc::VideoFrameBuffer> createBlackFrame(size_t, size_t) final;
// RealtimeMediaSource::VideoSampleObserver API
- void videoSampleAvailable(MediaSample&) final;
+ void videoSampleAvailable(MediaSample&, VideoSampleMetadata) final;
};
} // namespace WebCore
Modified: trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm 2021-11-18 07:46:05 UTC (rev 285984)
@@ -570,7 +570,9 @@
auto sample = MediaSampleAVFObjC::create(sampleBuffer, m_sampleRotation, [captureConnection isVideoMirrored]);
m_buffer = &sample.get();
setIntrinsicSize(expandedIntSize(sample->presentationSize()));
- dispatchMediaSampleToObservers(WTFMove(sample));
+ VideoSampleMetadata metadata;
+ metadata.captureTime = MonotonicTime::now().secondsSinceEpoch();
+ dispatchMediaSampleToObservers(WTFMove(sample), metadata);
}
void AVVideoCaptureSource::captureSessionIsRunningDidChange(bool state)
Modified: trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm 2021-11-18 07:46:05 UTC (rev 285984)
@@ -99,8 +99,11 @@
if (!sampleBuffer)
return;
- m_workQueue->dispatch([this, protectedThis = Ref { *this }, sampleBuffer = WTFMove(sampleBuffer)]() mutable {
- dispatchMediaSampleToObservers(*sampleBuffer);
+ auto captureTime = MonotonicTime::now().secondsSinceEpoch();
+ m_workQueue->dispatch([this, protectedThis = Ref { *this }, sampleBuffer = WTFMove(sampleBuffer), captureTime]() mutable {
+ VideoSampleMetadata metadata;
+ metadata.captureTime = captureTime;
+ dispatchMediaSampleToObservers(*sampleBuffer, metadata);
});
}
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm 2021-11-18 07:46:05 UTC (rev 285984)
@@ -196,7 +196,8 @@
auto sample = adoptCF(sampleBuffer);
setIntrinsicSize(IntSize(width, height));
- videoSampleAvailable(MediaSampleAVFObjC::create(sample.get(), rotation));
+
+ videoSampleAvailable(MediaSampleAVFObjC::create(sample.get(), rotation), metadataFromVideoFrame(frame));
}
} // namespace WebCore
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -61,7 +61,7 @@
{
}
-void RealtimeOutgoingVideoSourceCocoa::videoSampleAvailable(MediaSample& sample)
+void RealtimeOutgoingVideoSourceCocoa::videoSampleAvailable(MediaSample& sample, VideoSampleMetadata)
{
#if !RELEASE_LOG_DISABLED
if (!(++m_numberOfFrames % 60))
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h (285983 => 285984)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -45,7 +45,7 @@
rtc::scoped_refptr<webrtc::VideoFrameBuffer> createBlackFrame(size_t width, size_t height) final;
// RealtimeMediaSource::VideoSampleObserver API
- void videoSampleAvailable(MediaSample&) final;
+ void videoSampleAvailable(MediaSample&, VideoSampleMetadata) final;
RetainPtr<CVPixelBufferRef> convertToYUV(CVPixelBufferRef);
RetainPtr<CVPixelBufferRef> rotatePixelBuffer(CVPixelBufferRef, webrtc::VideoRotation);
Modified: trunk/Source/WebCore/testing/Internals.cpp (285983 => 285984)
--- trunk/Source/WebCore/testing/Internals.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/testing/Internals.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -5477,7 +5477,7 @@
m_nextTrackFramePromise = makeUnique<TrackFramePromise>(WTFMove(promise));
}
-void Internals::videoSampleAvailable(MediaSample& sample)
+void Internals::videoSampleAvailable(MediaSample& sample, VideoSampleMetadata)
{
callOnMainThread([this, weakThis = WeakPtr { *this }, sample = Ref { sample }] {
if (!weakThis)
Modified: trunk/Source/WebCore/testing/Internals.h (285983 => 285984)
--- trunk/Source/WebCore/testing/Internals.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebCore/testing/Internals.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -1219,7 +1219,7 @@
#if ENABLE(MEDIA_STREAM)
// RealtimeMediaSource::Observer API
- void videoSampleAvailable(MediaSample&) final;
+ void videoSampleAvailable(MediaSample&, VideoSampleMetadata) final;
// RealtimeMediaSource::AudioSampleObserver API
void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t) final { m_trackAudioSampleCount++; }
Modified: trunk/Source/WebKit/ChangeLog (285983 => 285984)
--- trunk/Source/WebKit/ChangeLog 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebKit/ChangeLog 2021-11-18 07:46:05 UTC (rev 285984)
@@ -1,3 +1,22 @@
+2021-11-17 Youenn Fablet <you...@apple.com>
+
+ Add support for more rvfc metadata
+ https://bugs.webkit.org/show_bug.cgi?id=233185
+
+ Reviewed by Eric Carlson.
+
+ Make sure to receive frame metadats, send it through IPC to WebProcess and pipe it to media source observers.
+
+ * UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp:
+ * WebProcess/GPU/webrtc/MediaRecorderPrivate.cpp:
+ * WebProcess/GPU/webrtc/MediaRecorderPrivate.h:
+ * WebProcess/cocoa/RemoteCaptureSampleManager.cpp:
+ * WebProcess/cocoa/RemoteCaptureSampleManager.h:
+ * WebProcess/cocoa/RemoteCaptureSampleManager.messages.in:
+ * WebProcess/cocoa/RemoteRealtimeDisplaySource.h:
+ * WebProcess/cocoa/RemoteRealtimeVideoSource.cpp:
+ * WebProcess/cocoa/RemoteRealtimeVideoSource.h:
+
2021-11-17 Myles C. Maxfield <mmaxfi...@apple.com>
[WebGPU] Add IPC message handlers for WebGPU
Modified: trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp (285983 => 285984)
--- trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -179,7 +179,7 @@
m_captureSemaphore->signal();
}
- void videoSampleAvailable(MediaSample& sample) final
+ void videoSampleAvailable(MediaSample& sample, VideoSampleMetadata metadata) final
{
std::unique_ptr<RemoteVideoSample> remoteSample;
if (m_shouldApplyRotation && sample.videoRotation() != MediaSample::VideoRotation::None) {
@@ -192,7 +192,7 @@
if (m_webProcessIdentityToken)
remoteSample->setOwnershipIdentity(*m_webProcessIdentityToken);
#endif
- m_connection->send(Messages::RemoteCaptureSampleManager::VideoSampleAvailable(m_id, WTFMove(*remoteSample)), 0);
+ m_connection->send(Messages::RemoteCaptureSampleManager::VideoSampleAvailable(m_id, WTFMove(*remoteSample), metadata), 0);
}
}
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.cpp (285983 => 285984)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -89,7 +89,7 @@
m_connection->send(Messages::RemoteMediaRecorderManager::ReleaseRecorder { m_identifier }, 0);
}
-void MediaRecorderPrivate::videoSampleAvailable(MediaSample& sample)
+void MediaRecorderPrivate::videoSampleAvailable(MediaSample& sample, VideoSampleMetadata)
{
std::unique_ptr<RemoteVideoSample> remoteSample;
if (shouldMuteVideo()) {
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.h (285983 => 285984)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -56,7 +56,7 @@
private:
// WebCore::MediaRecorderPrivate
- void videoSampleAvailable(WebCore::MediaSample&) final;
+ void videoSampleAvailable(WebCore::MediaSample&, WebCore::VideoSampleMetadata) final;
void fetchData(CompletionHandler<void(RefPtr<WebCore::SharedBuffer>&&, const String& mimeType, double)>&&) final;
void stopRecording(CompletionHandler<void()>&&) final;
void startRecording(StartRecordingCallback&&) final;
Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.cpp (285983 => 285984)
--- trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -150,7 +150,7 @@
iterator->value->setStorage(ipcHandle.handle, description, numberOfFrames, WTFMove(semaphore), mediaTime, frameChunkSize);
}
-void RemoteCaptureSampleManager::videoSampleAvailable(RealtimeMediaSourceIdentifier identifier, RemoteVideoSample&& sample)
+void RemoteCaptureSampleManager::videoSampleAvailable(RealtimeMediaSourceIdentifier identifier, RemoteVideoSample&& sample, VideoSampleMetadata metadata)
{
ASSERT(!WTF::isMainRunLoop());
@@ -159,7 +159,7 @@
RELEASE_LOG_ERROR(WebRTC, "Unable to find source %llu for remoteVideoSampleAvailable", identifier.toUInt64());
return;
}
- iterator->value->videoSampleAvailable(WTFMove(sample));
+ iterator->value->videoSampleAvailable(WTFMove(sample), metadata);
}
RemoteCaptureSampleManager::RemoteAudio::RemoteAudio(Ref<RemoteRealtimeAudioSource>&& source)
@@ -232,7 +232,7 @@
{
}
-void RemoteCaptureSampleManager::RemoteVideo::videoSampleAvailable(RemoteVideoSample&& remoteSample)
+void RemoteCaptureSampleManager::RemoteVideo::videoSampleAvailable(RemoteVideoSample&& remoteSample, VideoSampleMetadata metadata)
{
if (!m_imageTransferSession || m_imageTransferSession->pixelFormat() != remoteSample.videoFormat())
m_imageTransferSession = ImageTransferSessionVT::create(remoteSample.videoFormat());
@@ -248,9 +248,9 @@
return;
}
switchOn(m_source, [&](Ref<RemoteRealtimeVideoSource>& source) {
- source->videoSampleAvailable(*sampleRef, remoteSample.size());
+ source->videoSampleAvailable(*sampleRef, remoteSample.size(), metadata);
}, [&](Ref<RemoteRealtimeDisplaySource>& source) {
- source->remoteVideoSampleAvailable(*sampleRef);
+ source->remoteVideoSampleAvailable(*sampleRef, metadata);
});
}
Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.h (285983 => 285984)
--- trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -70,7 +70,7 @@
// Messages
void audioStorageChanged(WebCore::RealtimeMediaSourceIdentifier, const SharedMemory::IPCHandle&, const WebCore::CAAudioStreamDescription&, uint64_t numberOfFrames, IPC::Semaphore&&, const MediaTime&, size_t frameSampleSize);
void audioSamplesAvailable(WebCore::RealtimeMediaSourceIdentifier, MediaTime, uint64_t numberOfFrames);
- void videoSampleAvailable(WebCore::RealtimeMediaSourceIdentifier, WebCore::RemoteVideoSample&&);
+ void videoSampleAvailable(WebCore::RealtimeMediaSourceIdentifier, WebCore::RemoteVideoSample&&, WebCore::VideoSampleMetadata);
void setConnection(IPC::Connection*);
@@ -105,7 +105,7 @@
using Source = std::variant<Ref<RemoteRealtimeVideoSource>, Ref<RemoteRealtimeDisplaySource>>;
explicit RemoteVideo(Source&&);
- void videoSampleAvailable(WebCore::RemoteVideoSample&&);
+ void videoSampleAvailable(WebCore::RemoteVideoSample&&, WebCore::VideoSampleMetadata);
private:
Source m_source;
Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.messages.in (285983 => 285984)
--- trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.messages.in 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.messages.in 2021-11-18 07:46:05 UTC (rev 285984)
@@ -25,7 +25,7 @@
messages -> RemoteCaptureSampleManager NotRefCounted {
AudioStorageChanged(WebCore::RealtimeMediaSourceIdentifier id, WebKit::SharedMemory::IPCHandle storageHandle, WebCore::CAAudioStreamDescription description, uint64_t numberOfFrames, IPC::Semaphore captureSemaphore, MediaTime mediaTime, size_t frameChunkSize);
- VideoSampleAvailable(WebCore::RealtimeMediaSourceIdentifier id, WebCore::RemoteVideoSample sample)
+ VideoSampleAvailable(WebCore::RealtimeMediaSourceIdentifier id, WebCore::RemoteVideoSample sample, struct WebCore::VideoSampleMetadata metadata)
}
#endif
Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeDisplaySource.h (285983 => 285984)
--- trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeDisplaySource.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeDisplaySource.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -61,7 +61,7 @@
void captureStopped();
void captureFailed() final;
- void remoteVideoSampleAvailable(WebCore::MediaSample& sample) { videoSampleAvailable(sample); }
+ void remoteVideoSampleAvailable(WebCore::MediaSample& sample, WebCore::VideoSampleMetadata metadata) { videoSampleAvailable(sample, metadata); }
void sourceMutedChanged(bool value) { notifyMutedChange(value); }
private:
Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.cpp (285983 => 285984)
--- trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.cpp 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.cpp 2021-11-18 07:46:05 UTC (rev 285984)
@@ -112,7 +112,7 @@
notifySettingsDidChangeObservers(changed);
}
-void RemoteRealtimeVideoSource::videoSampleAvailable(MediaSample& sample, IntSize sampleSize)
+void RemoteRealtimeVideoSource::videoSampleAvailable(MediaSample& sample, IntSize sampleSize, VideoSampleMetadata metadata)
{
ASSERT(type() == Type::Video);
@@ -131,7 +131,7 @@
notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height });
});
}
- dispatchMediaSampleToObservers(sample);
+ dispatchMediaSampleToObservers(sample, metadata);
}
bool RemoteRealtimeVideoSource::setShouldApplyRotation(bool shouldApplyRotation)
Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.h (285983 => 285984)
--- trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.h 2021-11-18 07:18:04 UTC (rev 285983)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.h 2021-11-18 07:46:05 UTC (rev 285984)
@@ -66,7 +66,7 @@
void captureStopped();
void captureFailed() final;
- void videoSampleAvailable(WebCore::MediaSample&, WebCore::IntSize);
+ void videoSampleAvailable(WebCore::MediaSample&, WebCore::IntSize, WebCore::VideoSampleMetadata);
void sourceMutedChanged(bool value) { notifyMutedChange(value); }
private:
