[webkit-changes] [289237] trunk/Source

Mon, 07 Feb 2022 12:19:35 -0800

Title: [289237] trunk/Source
Revision
289237
Author
wenson_hs...@apple.com
Date
2022-02-07 12:18:46 -0800 (Mon, 07 Feb 2022)

Log Message

Add PageClient plumbing for video frame extraction
https://bugs.webkit.org/show_bug.cgi?id=236100
rdar://86237608

Reviewed by Eric Carlson.

Source/WebCore:

Add a method to grab the AVPlayerViewController from a VideoFullscreenInterfaceAVKit. See WebKit/ChangeLog for
more details.

* platform/ios/VideoFullscreenInterfaceAVKit.h:
* platform/ios/VideoFullscreenInterfaceAVKit.mm:
(-[WebAVPlayerViewController avPlayerViewController]):
(VideoFullscreenInterfaceAVKit::avPlayerViewController const):

Source/WebKit:

Finish off support for fullscreen video extraction by adding a new iOS PageClient hook, and calling into it
after grabbing a video frame from the GPU process when the video extraction timer fires.

* UIProcess/Cocoa/VideoFullscreenManagerProxy.h:
* UIProcess/Cocoa/VideoFullscreenManagerProxy.mm:
(WebKit::VideoFullscreenManagerProxy::findInterface const):
(WebKit::VideoFullscreenManagerProxy::playerViewController const):
(WebKit::VideoFullscreenManagerProxy::findInterface): Deleted.

Mark this method as `const`, so that we can invoke it from `VideoFullscreenManagerProxy::playerViewController`.

* UIProcess/Cocoa/WebPageProxyCocoa.mm:
(WebKit::WebPageProxy::updateFullscreenVideoExtraction):
(WebKit::WebPageProxy::fullscreenVideoExtractionTimerFired):

Request video frame data from the GPU process and hand it over to the page client along with the
AVPlayerViewController corresponding to the current fullscreen video.

* UIProcess/PageClient.h:
(WebKit::PageClient::isFullscreenVideoExtractionEnabled const):
* UIProcess/ios/PageClientImplIOS.h:
* UIProcess/ios/PageClientImplIOS.mm:
(WebKit::PageClientImpl::beginFullscreenVideoExtraction):
(WebKit::PageClientImpl::cancelFullscreenVideoExtraction):
(WebKit::PageClientImpl::isFullscreenVideoExtractionEnabled const):
* UIProcess/ios/WKContentViewInteraction.h:
* UIProcess/ios/WKContentViewInteraction.mm:

Add a new WebKitAdditions extension point that contains the implementations of the three methods below; when
compiling without WebKitAdditions, we simply stub out these methods.

(-[WKContentView beginFullscreenVideoExtraction:playerViewController:]):
(-[WKContentView cancelFullscreenVideoExtraction:]):
(-[WKContentView isFullscreenVideoExtractionEnabled]):

Modified Paths

Diff

Modified: trunk/Source/WebCore/ChangeLog (289236 => 289237)


--- trunk/Source/WebCore/ChangeLog	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebCore/ChangeLog	2022-02-07 20:18:46 UTC (rev 289237)
@@ -1,3 +1,19 @@
+2022-02-07  Wenson Hsieh  <wenson_hs...@apple.com>
+
+        Add PageClient plumbing for video frame extraction
+        https://bugs.webkit.org/show_bug.cgi?id=236100
+        rdar://86237608
+
+        Reviewed by Eric Carlson.
+
+        Add a method to grab the AVPlayerViewController from a VideoFullscreenInterfaceAVKit. See WebKit/ChangeLog for
+        more details.
+
+        * platform/ios/VideoFullscreenInterfaceAVKit.h:
+        * platform/ios/VideoFullscreenInterfaceAVKit.mm:
+        (-[WebAVPlayerViewController avPlayerViewController]):
+        (VideoFullscreenInterfaceAVKit::avPlayerViewController const):
+
 2022-02-07  Chris Dumez  <cdu...@apple.com>
 
         WorkerGlobalScope.importScripts() should protect blob urls that were passed in until the imports are done

Modified: trunk/Source/WebCore/platform/ios/VideoFullscreenInterfaceAVKit.h (289236 => 289237)


--- trunk/Source/WebCore/platform/ios/VideoFullscreenInterfaceAVKit.h	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebCore/platform/ios/VideoFullscreenInterfaceAVKit.h	2022-02-07 20:18:46 UTC (rev 289237)
@@ -43,6 +43,7 @@
 #include <wtf/ThreadSafeRefCounted.h>
 #include <wtf/WeakPtr.h>
 
+OBJC_CLASS AVPlayerViewController;
 OBJC_CLASS UIViewController;
 OBJC_CLASS UIWindow;
 OBJC_CLASS UIView;
@@ -162,6 +163,7 @@
     WEBCORE_EXPORT bool pictureInPictureWasStartedWhenEnteringBackground() const;
 
     std::optional<MediaPlayerIdentifier> playerIdentifier() const { return m_playerIdentifier; }
+    WEBCORE_EXPORT AVPlayerViewController *avPlayerViewController() const;
 
 protected:
     WEBCORE_EXPORT VideoFullscreenInterfaceAVKit(PlaybackSessionInterfaceAVKit&);

Modified: trunk/Source/WebCore/platform/ios/VideoFullscreenInterfaceAVKit.mm (289236 => 289237)


--- trunk/Source/WebCore/platform/ios/VideoFullscreenInterfaceAVKit.mm	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebCore/platform/ios/VideoFullscreenInterfaceAVKit.mm	2022-02-07 20:18:46 UTC (rev 289237)
@@ -795,7 +795,7 @@
     _avPlayerViewController.get().playerController = playerController;
 }
 
-- (AVPlayerViewController *) avPlayerViewController
+- (AVPlayerViewController *)avPlayerViewController
 {
     return _avPlayerViewController.get();
 }
@@ -834,6 +834,11 @@
     return m_playbackSessionInterface->playerController();
 }
 
+AVPlayerViewController *VideoFullscreenInterfaceAVKit::avPlayerViewController() const
+{
+    return [m_playerViewController avPlayerViewController];
+}
+
 void VideoFullscreenInterfaceAVKit::setVideoFullscreenModel(VideoFullscreenModel* model)
 {
     if (m_videoFullscreenModel)

Modified: trunk/Source/WebKit/ChangeLog (289236 => 289237)


--- trunk/Source/WebKit/ChangeLog	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebKit/ChangeLog	2022-02-07 20:18:46 UTC (rev 289237)
@@ -1,3 +1,46 @@
+2022-02-07  Wenson Hsieh  <wenson_hs...@apple.com>
+
+        Add PageClient plumbing for video frame extraction
+        https://bugs.webkit.org/show_bug.cgi?id=236100
+        rdar://86237608
+
+        Reviewed by Eric Carlson.
+
+        Finish off support for fullscreen video extraction by adding a new iOS PageClient hook, and calling into it
+        after grabbing a video frame from the GPU process when the video extraction timer fires.
+
+        * UIProcess/Cocoa/VideoFullscreenManagerProxy.h:
+        * UIProcess/Cocoa/VideoFullscreenManagerProxy.mm:
+        (WebKit::VideoFullscreenManagerProxy::findInterface const):
+        (WebKit::VideoFullscreenManagerProxy::playerViewController const):
+        (WebKit::VideoFullscreenManagerProxy::findInterface): Deleted.
+
+        Mark this method as `const`, so that we can invoke it from `VideoFullscreenManagerProxy::playerViewController`.
+
+        * UIProcess/Cocoa/WebPageProxyCocoa.mm:
+        (WebKit::WebPageProxy::updateFullscreenVideoExtraction):
+        (WebKit::WebPageProxy::fullscreenVideoExtractionTimerFired):
+
+        Request video frame data from the GPU process and hand it over to the page client along with the
+        AVPlayerViewController corresponding to the current fullscreen video.
+
+        * UIProcess/PageClient.h:
+        (WebKit::PageClient::isFullscreenVideoExtractionEnabled const):
+        * UIProcess/ios/PageClientImplIOS.h:
+        * UIProcess/ios/PageClientImplIOS.mm:
+        (WebKit::PageClientImpl::beginFullscreenVideoExtraction):
+        (WebKit::PageClientImpl::cancelFullscreenVideoExtraction):
+        (WebKit::PageClientImpl::isFullscreenVideoExtractionEnabled const):
+        * UIProcess/ios/WKContentViewInteraction.h:
+        * UIProcess/ios/WKContentViewInteraction.mm:
+
+        Add a new WebKitAdditions extension point that contains the implementations of the three methods below; when
+        compiling without WebKitAdditions, we simply stub out these methods.
+
+        (-[WKContentView beginFullscreenVideoExtraction:playerViewController:]):
+        (-[WKContentView cancelFullscreenVideoExtraction:]):
+        (-[WKContentView isFullscreenVideoExtractionEnabled]):
+
 2022-02-07  Antoine Quint  <grao...@webkit.org>
 
         [model] improve sizing on iOS

Modified: trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.h (289236 => 289237)


--- trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.h	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.h	2022-02-07 20:18:46 UTC (rev 289237)
@@ -154,6 +154,10 @@
 
     void requestBitmapImageForCurrentTime(PlaybackSessionContextIdentifier, CompletionHandler<void(const ShareableBitmap::Handle&)>&&);
 
+#if PLATFORM(IOS_FAMILY)
+    AVPlayerViewController *playerViewController(PlaybackSessionContextIdentifier) const;
+#endif
+
 private:
     friend class VideoFullscreenModelContext;
 
@@ -165,7 +169,7 @@
     ModelInterfaceTuple& ensureModelAndInterface(PlaybackSessionContextIdentifier);
     VideoFullscreenModelContext& ensureModel(PlaybackSessionContextIdentifier);
     PlatformVideoFullscreenInterface& ensureInterface(PlaybackSessionContextIdentifier);
-    PlatformVideoFullscreenInterface* findInterface(PlaybackSessionContextIdentifier);
+    PlatformVideoFullscreenInterface* findInterface(PlaybackSessionContextIdentifier) const;
     void ensureClientForContext(PlaybackSessionContextIdentifier);
     void addClientForContext(PlaybackSessionContextIdentifier);
     void removeClientForContext(PlaybackSessionContextIdentifier);

Modified: trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.mm (289236 => 289237)


--- trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.mm	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.mm	2022-02-07 20:18:46 UTC (rev 289237)
@@ -447,7 +447,7 @@
     return *std::get<1>(ensureModelAndInterface(contextId));
 }
 
-PlatformVideoFullscreenInterface* VideoFullscreenManagerProxy::findInterface(PlaybackSessionContextIdentifier contextId)
+PlatformVideoFullscreenInterface* VideoFullscreenManagerProxy::findInterface(PlaybackSessionContextIdentifier contextId) const
 {
     auto it = m_contextMap.find(contextId);
     if (it == m_contextMap.end())
@@ -907,6 +907,16 @@
 
 #endif
 
+#if PLATFORM(IOS_FAMILY)
+
+AVPlayerViewController *VideoFullscreenManagerProxy::playerViewController(PlaybackSessionContextIdentifier identifier) const
+{
+    auto* interface = findInterface(identifier);
+    return interface ? interface->avPlayerViewController() : nil;
+}
+
+#endif // PLATFORM(IOS_FAMILY)
+
 } // namespace WebKit
 
 #undef MESSAGE_CHECK

Modified: trunk/Source/WebKit/UIProcess/Cocoa/WebPageProxyCocoa.mm (289236 => 289237)


--- trunk/Source/WebKit/UIProcess/Cocoa/WebPageProxyCocoa.mm	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebKit/UIProcess/Cocoa/WebPageProxyCocoa.mm	2022-02-07 20:18:46 UTC (rev 289237)
@@ -477,6 +477,9 @@
 
 void WebPageProxy::updateFullscreenVideoExtraction()
 {
+    if (!pageClient().isFullscreenVideoExtractionEnabled())
+        return;
+
     if (m_currentFullscreenVideoSessionIdentifier && m_playbackSessionManager && m_playbackSessionManager->isPaused(*m_currentFullscreenVideoSessionIdentifier)) {
         m_fullscreenVideoExtractionTimer.startOneShot(250_ms);
         return;
@@ -483,14 +486,36 @@
     }
 
     m_fullscreenVideoExtractionTimer.stop();
+
+    if (!m_currentFullscreenVideoSessionIdentifier)
+        return;
+
+#if PLATFORM(IOS_FAMILY)
+    if (RetainPtr controller = m_videoFullscreenManager->playerViewController(*m_currentFullscreenVideoSessionIdentifier))
+        pageClient().cancelFullscreenVideoExtraction(controller.get());
+#endif
 }
 
 void WebPageProxy::fullscreenVideoExtractionTimerFired()
 {
-    if (!m_currentFullscreenVideoSessionIdentifier)
+    if (!m_currentFullscreenVideoSessionIdentifier || !m_videoFullscreenManager)
         return;
 
-    // FIXME: Add logic to perform video frame extraction.
+    auto identifier = *m_currentFullscreenVideoSessionIdentifier;
+    m_videoFullscreenManager->requestBitmapImageForCurrentTime(identifier, [identifier, weakThis = WeakPtr { *this }](auto& imageHandle) {
+        RefPtr protectedThis = weakThis.get();
+        if (!protectedThis || protectedThis->m_currentFullscreenVideoSessionIdentifier != identifier)
+            return;
+
+        auto fullscreenManager = protectedThis->m_videoFullscreenManager;
+        if (!fullscreenManager)
+            return;
+
+#if PLATFORM(IOS_FAMILY)
+        if (RetainPtr controller = fullscreenManager->playerViewController(identifier))
+            protectedThis->pageClient().beginFullscreenVideoExtraction(imageHandle, controller.get());
+#endif
+    });
 }
 
 #endif // ENABLE(VIDEO_PRESENTATION_MODE)

Modified: trunk/Source/WebKit/UIProcess/PageClient.h (289236 => 289237)


--- trunk/Source/WebKit/UIProcess/PageClient.h	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebKit/UIProcess/PageClient.h	2022-02-07 20:18:46 UTC (rev 289237)
@@ -67,6 +67,7 @@
 #include <WebCore/TextRecognitionResult.h>
 #endif
 
+OBJC_CLASS AVPlayerViewController;
 OBJC_CLASS CALayer;
 OBJC_CLASS NSFileWrapper;
 OBJC_CLASS NSMenu;
@@ -500,7 +501,11 @@
 
     virtual WebCore::Color contentViewBackgroundColor() = 0;
     virtual String sceneID() = 0;
+
+    virtual void beginFullscreenVideoExtraction(const ShareableBitmap::Handle&, AVPlayerViewController *) = 0;
+    virtual void cancelFullscreenVideoExtraction(AVPlayerViewController *) = 0;
 #endif
+    virtual bool isFullscreenVideoExtractionEnabled() const { return false; }
 
     // Auxiliary Client Creation
 #if ENABLE(FULLSCREEN_API)

Modified: trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.h (289236 => 289237)


--- trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.h	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.h	2022-02-07 20:18:46 UTC (rev 289237)
@@ -303,6 +303,10 @@
     WebCore::Color contentViewBackgroundColor() final;
     String sceneID() final;
 
+    void beginFullscreenVideoExtraction(const ShareableBitmap::Handle&, AVPlayerViewController *) final;
+    void cancelFullscreenVideoExtraction(AVPlayerViewController *) final;
+    bool isFullscreenVideoExtractionEnabled() const final;
+
     WeakObjCPtr<WKContentView> m_contentView;
     RetainPtr<WKEditorUndoTarget> m_undoTarget;
 };

Modified: trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.mm (289236 => 289237)


--- trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.mm	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.mm	2022-02-07 20:18:46 UTC (rev 289237)
@@ -1031,6 +1031,21 @@
     return [m_contentView window].windowScene._sceneIdentifier;
 }
 
+void PageClientImpl::beginFullscreenVideoExtraction(const ShareableBitmap::Handle& imageHandle, AVPlayerViewController *playerViewController)
+{
+    [m_contentView beginFullscreenVideoExtraction:imageHandle playerViewController:playerViewController];
+}
+
+void PageClientImpl::cancelFullscreenVideoExtraction(AVPlayerViewController *controller)
+{
+    [m_contentView cancelFullscreenVideoExtraction:controller];
+}
+
+bool PageClientImpl::isFullscreenVideoExtractionEnabled() const
+{
+    return [m_contentView isFullscreenVideoExtractionEnabled];
+}
+
 } // namespace WebKit
 
 #endif // PLATFORM(IOS_FAMILY)

Modified: trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.h (289236 => 289237)


--- trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.h	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.h	2022-02-07 20:18:46 UTC (rev 289237)
@@ -110,6 +110,7 @@
 class WebPageProxy;
 }
 
+@class AVPlayerViewController;
 @class QLPreviewController;
 @class WebEvent;
 @class WebTextIndicatorLayer;
@@ -528,6 +529,7 @@
     BOOL _hasVisualSearchResults;
 #endif // USE(QUICK_LOOK)
 #endif // ENABLE(IMAGE_ANALYSIS)
+    uint32_t _fullscreenVideoExtractionRequestIdentifier;
 }
 
 @end
@@ -783,6 +785,10 @@
 - (void)requestRectForFoundTextRange:(UITextRange *)range completionHandler:(void (^)(CGRect))completionHandler;
 #endif
 
+- (void)beginFullscreenVideoExtraction:(const WebKit::ShareableBitmap::Handle&)imageHandle playerViewController:(AVPlayerViewController *)playerViewController;
+- (void)cancelFullscreenVideoExtraction:(AVPlayerViewController *)controller;
+@property (nonatomic, readonly) BOOL isFullscreenVideoExtractionEnabled;
+
 @end
 
 @interface WKContentView (WKTesting)

Modified: trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm (289236 => 289237)


--- trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm	2022-02-07 19:57:43 UTC (rev 289236)
+++ trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm	2022-02-07 20:18:46 UTC (rev 289237)
@@ -135,6 +135,7 @@
 #import <WebCore/WritingDirection.h>
 #import <WebKit/WebSelectionRect.h> // FIXME: WebKit should not include WebKitLegacy headers!
 #import <pal/spi/cg/CoreGraphicsSPI.h>
+#import <pal/spi/cocoa/AVKitSPI.h>
 #import <pal/spi/cocoa/DataDetectorsCoreSPI.h>
 #import <pal/spi/cocoa/LaunchServicesSPI.h>
 #import <pal/spi/cocoa/NSAttributedStringSPI.h>
@@ -10628,6 +10629,25 @@
 
 #endif // ENABLE(IMAGE_ANALYSIS)
 
+#if USE(APPLE_INTERNAL_SDK)
+#import <WebKitAdditions/WKContentViewInteractionAdditionsAfter.mm>
+#else
+
+- (void)beginFullscreenVideoExtraction:(const WebKit::ShareableBitmap::Handle&)imageData playerViewController:(AVPlayerViewController *)controller
+{
+}
+
+- (void)cancelFullscreenVideoExtraction:(AVPlayerViewController *)controller
+{
+}
+
+- (BOOL)isFullscreenVideoExtractionEnabled
+{
+    return NO;
+}
+
+#endif
+
 @end
 
 @implementation WKContentView (WKTesting)

_______________________________________________
webkit-changes mailing list
webkit-changes@lists.webkit.org
https://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to