Modified: branches/safari-611-branch/Source/ThirdParty/libwebrtc/ChangeLog (278898 => 278899)
--- branches/safari-611-branch/Source/ThirdParty/libwebrtc/ChangeLog 2021-06-15 21:02:52 UTC (rev 278898)
+++ branches/safari-611-branch/Source/ThirdParty/libwebrtc/ChangeLog 2021-06-15 21:49:19 UTC (rev 278899)
@@ -1,3 +1,68 @@
+2021-06-15 Alan Coon <alanc...@apple.com>
+
+ Cherry-pick r278272. rdar://problem/79355341
+
+ Use tighter bitrate allocation rules for WebRTC H264 software encoder
+ https://bugs.webkit.org/show_bug.cgi?id=226319
+ <rdar://73150695>
+
+ Reviewed by Eric Carlson.
+
+ Software H264 encoder is sometimes overshooting target bitrate in which case WebRTC backend will start dropping frames.
+ The encoder might then think it is on target and will not try to increase compression.
+ This makes it possible to be locked in a very low frame rate but high quality image situation.
+ It is often better to preserve frame rate and lower quality, the application could always lower frame rate if desired.
+
+ To do so, we detect whether the encoder is using software code path or not.
+ If so, we compute the actual frame rate and compare it with the expected frame rate.
+ If the actual frame rate is twice smaller or even below, we enter in a low frame rate mode.
+ Otherwise, we are in a regular frame rate mode where we apply the normal bitrate rules.
+ In the low frame rate mode, we divide the target bitrate by 3 so as to quickly recover frame rate.
+
+ This works well in situations where motion increases from time to time.
+ It is still not perfect for instance in case the video is muted and gets unmuted or when the scene is completely still and suddenly large motion happens.
+ In those cases, frame rate is recovered after a minute or so according my testing.
+
+ * Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm:
+ (-[RTCVideoEncoderH264 initWithCodecInfo:]):
+ (-[RTCVideoEncoderH264 encode:codecSpecificInfo:frameTypes:]):
+ (-[RTCVideoEncoderH264 resetCompressionSessionWithPixelFormat:]):
+ (-[RTCVideoEncoderH264 setEncoderBitrateBps:frameRate:]):
+ (-[RTCVideoEncoderH264 updateBitRateAccordingActualFrameRate]):
+
+
+ git-svn-id: https://svn.webkit.org/repository/webkit/trunk@278272 268f45cc-cd09-0410-ab3c-d52691b4dbfc
+
+ 2021-05-31 Youenn Fablet <you...@apple.com>
+
+ Use tighter bitrate allocation rules for WebRTC H264 software encoder
+ https://bugs.webkit.org/show_bug.cgi?id=226319
+ <rdar://73150695>
+
+ Reviewed by Eric Carlson.
+
+ Software H264 encoder is sometimes overshooting target bitrate in which case WebRTC backend will start dropping frames.
+ The encoder might then think it is on target and will not try to increase compression.
+ This makes it possible to be locked in a very low frame rate but high quality image situation.
+ It is often better to preserve frame rate and lower quality, the application could always lower frame rate if desired.
+
+ To do so, we detect whether the encoder is using software code path or not.
+ If so, we compute the actual frame rate and compare it with the expected frame rate.
+ If the actual frame rate is twice smaller or even below, we enter in a low frame rate mode.
+ Otherwise, we are in a regular frame rate mode where we apply the normal bitrate rules.
+ In the low frame rate mode, we divide the target bitrate by 3 so as to quickly recover frame rate.
+
+ This works well in situations where motion increases from time to time.
+ It is still not perfect for instance in case the video is muted and gets unmuted or when the scene is completely still and suddenly large motion happens.
+ In those cases, frame rate is recovered after a minute or so according my testing.
+
+ * Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm:
+ (-[RTCVideoEncoderH264 initWithCodecInfo:]):
+ (-[RTCVideoEncoderH264 encode:codecSpecificInfo:frameTypes:]):
+ (-[RTCVideoEncoderH264 resetCompressionSessionWithPixelFormat:]):
+ (-[RTCVideoEncoderH264 setEncoderBitrateBps:frameRate:]):
+ (-[RTCVideoEncoderH264 updateBitRateAccordingActualFrameRate]):
+
2021-05-20 Alan Coon <alanc...@apple.com>
Cherry-pick r277332. rdar://problem/78264344
Modified: branches/safari-611-branch/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm (278898 => 278899)
--- branches/safari-611-branch/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm 2021-06-15 21:02:52 UTC (rev 278898)
+++ branches/safari-611-branch/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm 2021-06-15 21:49:19 UTC (rev 278899)
@@ -35,6 +35,7 @@
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/buffer.h"
#include "rtc_base/logging.h"
+#include "rtc_base/system/arch.h"
#include "rtc_base/time_utils.h"
#include "sdk/objc/components/video_codec/nalu_rewriter.h"
#include "third_party/libyuv/include/libyuv/convert_from.h"
@@ -60,7 +61,7 @@
timestamp:(uint32_t)timestamp
rotation:(RTCVideoRotation)rotation
isKeyFrameRequired:(bool)isKeyFrameRequired;
-
+- (void)updateBitRateAccordingActualFrameRate;
@end
namespace { // anonymous namespace
@@ -76,6 +77,9 @@
const OSType kNV12PixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
+const int kBelowFrameRateThreshold = 2;
+const int kBelowFrameRateBitRateDecrease = 3;
+
// Struct that we pass to the encoder per frame to encode. We receive it again
// in the encoder callback.
struct RTCFrameEncodeParams {
@@ -348,6 +352,10 @@
bool _disableEncoding;
bool _isKeyFrameRequired;
bool _isH264LowLatencyEncoderEnabled;
+ bool _isUsingSoftwareEncoder;
+ bool _isBelowExpectedFrameRate;
+ uint32_t _frameCount;
+ int64_t _lastFrameRateEstimationTime;
}
// .5 is set as a mininum to prevent overcompensating for large temporary
@@ -376,7 +384,10 @@
}
_isKeyFrameRequired = false;
_isH264LowLatencyEncoderEnabled = true;
-
+ _isUsingSoftwareEncoder = false;
+ _isBelowExpectedFrameRate = false;
+ _frameCount = 0;
+ _lastFrameRateEstimationTime = 0;
return self;
}
@@ -441,6 +452,9 @@
if (_disableEncoding) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
+ if (_isUsingSoftwareEncoder) {
+ [self updateBitRateAccordingActualFrameRate];
+ }
CVPixelBufferRef pixelBuffer = nullptr;
if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
@@ -750,6 +764,12 @@
nullptr,
&_vcpCompressionSession);
}
+#elif defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) && !(ENABLE_VCP_FOR_H264_BASELINE) && defined(WEBRTC_ARCH_X86_FAMILY)
+ CFBooleanRef hwaccl_enabled = nullptr;
+ if (status == noErr) {
+ auto result = VTSessionCopyProperty(_vtCompressionSession, kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, nullptr, &hwaccl_enabled);
+ _isUsingSoftwareEncoder = result == noErr ? !CFBooleanGetValue(hwaccl_enabled) : _width <= 480 && _height <= 480;
+ }
#else
// Provided encoder should be good enough.
#endif // defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) && ENABLE_VCP_ENCODER
@@ -833,7 +853,8 @@
- (void)setEncoderBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate {
if ([self hasCompressionSession]) {
- SetVTSessionProperty(_vtCompressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps);
+ auto actualTarget = _isBelowExpectedFrameRate ? bitrateBps / kBelowFrameRateBitRateDecrease : bitrateBps;
+ SetVTSessionProperty(_vtCompressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_AverageBitRate, actualTarget);
// With zero |_maxAllowedFrameRate|, we fall back to automatic frame rate detection.
if (_maxAllowedFrameRate > 0) {
@@ -843,7 +864,7 @@
// TODO(tkchin): Add a helper method to set array value.
int64_t dataLimitBytesPerSecondValue =
- static_cast<int64_t>(bitrateBps * kLimitToAverageBitRateFactor / 8);
+ static_cast<int64_t>(_isBelowExpectedFrameRate ? actualTarget / 8 : actualTarget * kLimitToAverageBitRateFactor / 8);
CFNumberRef bytesPerSecond =
CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue);
int64_t _oneSecondValue_ = 1;
@@ -869,6 +890,34 @@
}
}
+- (void)updateBitRateAccordingActualFrameRate {
+ _frameCount++;
+ auto currentTime = rtc::TimeMillis();
+ if (!_lastFrameRateEstimationTime) {
+ _lastFrameRateEstimationTime = currentTime;
+ return;
+ }
+
+ auto timeDifference = currentTime - _lastFrameRateEstimationTime;
+
+ // Let's not check too often.
+ if (timeDifference < 1000) {
+ return;
+ }
+
+ auto frameRate = _frameCount * 1000 / timeDifference;
+ _lastFrameRateEstimationTime = currentTime;
+ _frameCount = 0;
+
+ bool isBelow = frameRate * kBelowFrameRateThreshold < _encoderFrameRate;
+ if (isBelow == _isBelowExpectedFrameRate) {
+ return;
+ }
+
+ _isBelowExpectedFrameRate = isBelow;
+ [self setEncoderBitrateBps:_encoderBitrateBps frameRate:_encoderFrameRate];
+}
+
- (void)frameWasEncoded:(OSStatus)status
flags:(VTEncodeInfoFlags)infoFlags
sampleBuffer:(CMSampleBufferRef)sampleBuffer
