- Revision
- 290355
- Author
- you...@apple.com
- Date
- 2022-02-23 00:16:29 -0800 (Wed, 23 Feb 2022)
Log Message
Optimize RemoteVideoFrame handling in WebProcess WebRTC pipeline
https://bugs.webkit.org/show_bug.cgi?id=236970
Reviewed by Eric Carlson.
Source/WebCore:
In case of remote video frame, directly get the remote video frame as a MediaSample in receive side.
On send side, wrap remote video frames as provider-based WebRTC video frames.
LibWebRTCCodecs will optimize the handling of remote frames for hardware encoders and no change will
happen for software encoders.
Covered by existing tests.
* platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:
* platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp:
Source/WebKit:
Fix bug in LibWebRTCCodecsProxy that would make use of shared video frame instead of remote video frame when a shared video frame is available.
Use VideoFrame* as webrtc frame provider for encoding and decoding.
* GPUProcess/webrtc/LibWebRTCCodecsProxy.mm:
* WebProcess/GPU/media/RemoteVideoFrameProxy.h:
* WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp:
Modified Paths
Diff
Modified: trunk/Source/WebCore/ChangeLog (290354 => 290355)
--- trunk/Source/WebCore/ChangeLog 2022-02-23 08:05:44 UTC (rev 290354)
+++ trunk/Source/WebCore/ChangeLog 2022-02-23 08:16:29 UTC (rev 290355)
@@ -1,3 +1,19 @@
+2022-02-23 Youenn Fablet <you...@apple.com>
+
+ Optimize RemoteVideoFrame handling in WebProcess WebRTC pipeline
+ https://bugs.webkit.org/show_bug.cgi?id=236970
+
+ Reviewed by Eric Carlson.
+
+ In case of remote video frame, directly get the remote video frame as a MediaSample in receive side.
+ On send side, wrap remote video frames as provider-based WebRTC video frames.
+ LibWebRTCCodecs will optimize the handling of remote frames for hardware encoders and no change will
+ happen for software encoders.
+ Covered by existing tests.
+
+ * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:
+ * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp:
+
2022-02-23 Lauro Moura <lmo...@igalia.com>
Unreviewed, non-unified build fix after 247623@main
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm (290354 => 290355)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm 2022-02-23 08:05:44 UTC (rev 290354)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm 2022-02-23 08:16:29 UTC (rev 290355)
@@ -33,6 +33,7 @@
#import "CVUtilities.h"
#import "Logging.h"
#import "MediaSampleAVFObjC.h"
+#import "VideoFrame.h"
#import <wtf/cf/TypeCastsCF.h>
ALLOW_UNUSED_PARAMETERS_BEGIN
@@ -127,7 +128,11 @@
return createMediaSampleFromCVPixelBuffer(m_blackFrame.get(), rotation, frame.timestamp_us());
}
- // FIXME: Detect the case of frame having a FrameBufferProvider.
+ if (auto* provider = videoFrameBufferProvider(frame)) {
+ // The only supported provider is VideoFrame.
+ return static_cast<VideoFrame*>(provider);
+ }
+
// In case of in memory samples, we have non interleaved YUV data while CVPixelBuffers prefer interleaved YUV data.
// Maybe we should introduce a MediaSample that would represent non interleaved YUV data as an optimization.
auto pixelBuffer = adoptCF(webrtc::createPixelBufferFromFrame(frame, [this](size_t width, size_t height, webrtc::BufferType bufferType) -> CVPixelBufferRef {
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp (290354 => 290355)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp 2022-02-23 08:05:44 UTC (rev 290354)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp 2022-02-23 08:16:29 UTC (rev 290355)
@@ -33,6 +33,7 @@
#include "Logging.h"
#include "RealtimeIncomingVideoSourceCocoa.h"
#include "RealtimeVideoUtilities.h"
+#include "VideoFrame.h"
ALLOW_UNUSED_PARAMETERS_BEGIN
@@ -84,7 +85,17 @@
break;
}
- // FIXME: Optimize the case of PlatformSample::RemoteVideoFrameProxyType.
+ bool shouldApplyRotation = m_shouldApplyRotation && m_currentRotation != webrtc::kVideoRotation_0;
+ if (!shouldApplyRotation && is<VideoFrame>(sample) && downcast<VideoFrame>(sample).isRemoteProxy()) {
+ Ref videoFrame { downcast<VideoFrame>(sample) };
+ auto size = sample.presentationSize();
+ sendFrame(webrtc::toWebRTCVideoFrameBuffer(&videoFrame.leakRef(),
+ [](auto* pointer) { return static_cast<VideoFrame*>(pointer)->pixelBuffer(); },
+ [](auto* pointer) { static_cast<VideoFrame*>(pointer)->deref(); },
+ static_cast<int>(size.width()), static_cast<int>(size.height())));
+ return;
+ }
+
auto pixelBuffer = sample.pixelBuffer();
auto pixelFormatType = CVPixelBufferGetPixelFormatType(pixelBuffer);
@@ -92,7 +103,7 @@
if (pixelFormatType != preferedPixelBufferFormat())
convertedBuffer = convertToYUV(pixelBuffer);
- if (m_shouldApplyRotation && m_currentRotation != webrtc::kVideoRotation_0)
+ if (shouldApplyRotation)
convertedBuffer = rotatePixelBuffer(convertedBuffer.get(), m_currentRotation);
sendFrame(webrtc::pixelBufferToFrame(convertedBuffer.get()));
Modified: trunk/Source/WebKit/ChangeLog (290354 => 290355)
--- trunk/Source/WebKit/ChangeLog 2022-02-23 08:05:44 UTC (rev 290354)
+++ trunk/Source/WebKit/ChangeLog 2022-02-23 08:16:29 UTC (rev 290355)
@@ -1,3 +1,17 @@
+2022-02-23 Youenn Fablet <you...@apple.com>
+
+ Optimize RemoteVideoFrame handling in WebProcess WebRTC pipeline
+ https://bugs.webkit.org/show_bug.cgi?id=236970
+
+ Reviewed by Eric Carlson.
+
+ Fix bug in LibWebRTCCodecsProxy that would make use of shared video frame instead of remote video frame when a shared video frame is available.
+ Use VideoFrame* as webrtc frame provider for encoding and decoding.
+
+ * GPUProcess/webrtc/LibWebRTCCodecsProxy.mm:
+ * WebProcess/GPU/media/RemoteVideoFrameProxy.h:
+ * WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp:
+
2022-02-22 Chris Dumez <cdu...@apple.com>
http/wpt/push-api/onpush-disabled.html fails
Modified: trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm (290354 => 290355)
--- trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm 2022-02-23 08:05:44 UTC (rev 290354)
+++ trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm 2022-02-23 08:16:29 UTC (rev 290355)
@@ -249,14 +249,16 @@
return;
#if !PLATFORM(MACCATALYST)
- if (sample.surface()) {
- if (auto buffer = WebCore::createCVPixelBuffer(sample.surface()))
- pixelBuffer = WTFMove(*buffer);
- } else if (encoder->frameReader)
- pixelBuffer = encoder->frameReader->read();
+ if (!pixelBuffer) {
+ if (sample.surface()) {
+ if (auto buffer = WebCore::createCVPixelBuffer(sample.surface()))
+ pixelBuffer = WTFMove(*buffer);
+ } else if (encoder->frameReader)
+ pixelBuffer = encoder->frameReader->read();
- if (!pixelBuffer)
- return;
+ if (!pixelBuffer)
+ return;
+ }
webrtc::encodeLocalEncoderFrame(encoder->webrtcEncoder, pixelBuffer.get(), sample.time().toTimeScale(1000000).timeValue(), timeStamp, toWebRTCVideoRotation(sample.rotation()), shouldEncodeAsKeyFrame);
#endif
Modified: trunk/Source/WebKit/WebProcess/GPU/media/RemoteVideoFrameProxy.h (290354 => 290355)
--- trunk/Source/WebKit/WebProcess/GPU/media/RemoteVideoFrameProxy.h 2022-02-23 08:05:44 UTC (rev 290354)
+++ trunk/Source/WebKit/WebProcess/GPU/media/RemoteVideoFrameProxy.h 2022-02-23 08:16:29 UTC (rev 290355)
@@ -81,7 +81,7 @@
RemoteVideoFrameWriteReference write() const;
RemoteVideoFrameReadReference read() const;
- WebCore::IntSize size() const;
+ WebCore::IntSize size() const { return m_size; }
// WebCore::VideoFrame overrides.
WebCore::FloatSize presentationSize() const final { return m_size; }
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp (290354 => 290355)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp 2022-02-23 08:05:44 UTC (rev 290354)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp 2022-02-23 08:16:29 UTC (rev 290355)
@@ -343,9 +343,10 @@
return;
if (remoteVideoFrame) {
- webrtc::videoDecoderTaskComplete(decoder->decodedImageCallback, timeStamp, remoteSample.time().toDouble(), remoteVideoFrame.leakRef(),
- [](auto* pointer) { return static_cast<RemoteVideoFrameProxy*>(pointer)->pixelBuffer(); },
- [](auto* pointer) { static_cast<RemoteVideoFrameProxy*>(pointer)->deref(); },
+ Ref videoFrame { static_cast<VideoFrame&>(*remoteVideoFrame) };
+ webrtc::videoDecoderTaskComplete(decoder->decodedImageCallback, timeStamp, remoteSample.time().toDouble(), &videoFrame.leakRef(),
+ [](auto* pointer) { return static_cast<VideoFrame*>(pointer)->pixelBuffer(); },
+ [](auto* pointer) { static_cast<VideoFrame*>(pointer)->deref(); },
remoteSample.size().width(), remoteSample.size().height());
return;
}
@@ -457,10 +458,10 @@
return WEBRTC_VIDEO_CODEC_ERROR;
std::optional<RemoteVideoFrameReadReference> remoteVideoFrameReadReference;
- if (auto provider = webrtc::videoFrameBufferProvider(frame)) {
- auto& mediaSample = *static_cast<MediaSample*>(provider);
- if (is<RemoteVideoFrameProxy>(mediaSample))
- remoteVideoFrameReadReference = downcast<RemoteVideoFrameProxy>(mediaSample).read();
+ if (auto* provider = webrtc::videoFrameBufferProvider(frame)) {
+ auto* videoFrame = static_cast<VideoFrame*>(provider);
+ if (is<RemoteVideoFrameProxy>(videoFrame))
+ remoteVideoFrameReadReference = downcast<RemoteVideoFrameProxy>(videoFrame)->read();
}
RetainPtr<CVPixelBufferRef> buffer;
