- Revision
- 289221
- Author
- wenson_hs...@apple.com
- Date
- 2022-02-07 08:01:55 -0800 (Mon, 07 Feb 2022)
Log Message
Add a helper method on VideoFullscreenManagerProxy to request video frame data
https://bugs.webkit.org/show_bug.cgi?id=236094
Reviewed by Darin Adler.
Add a helper method on VideoFullscreenManagerProxy to asynchronously grab an image bitmap for the fullscreen
video corresponding to the given PlaybackSessionContextIdentifier, by sending IPC to the GPU process. This
mechanism will be used in a subsequent patch to support "fullscreen video extraction", which involves capturing
image data for paused, fullscreen videos.
See below for more details.
* GPUProcess/GPUProcess.cpp:
(WebKit::GPUProcess::requestBitmapImageForCurrentTime):
* GPUProcess/GPUProcess.h:
* GPUProcess/GPUProcess.messages.in:
* GPUProcess/media/RemoteMediaPlayerManagerProxy.cpp:
(WebKit::RemoteMediaPlayerManagerProxy::bitmapImageForCurrentTime):
* GPUProcess/media/RemoteMediaPlayerManagerProxy.h:
* UIProcess/Cocoa/VideoFullscreenManagerProxy.h:
* UIProcess/Cocoa/VideoFullscreenManagerProxy.mm:
(WebKit::VideoFullscreenManagerProxy::requestBitmapImageForCurrentTime):
Map the given PlaybackSessionContextIdentifier to a MediaPlayerIdentifier, and then send the (web process ID,
media player ID) identifier pair to the GPU process to request frame data for the given media player.
* UIProcess/GPU/GPUProcessProxy.cpp:
(WebKit::GPUProcessProxy::requestBitmapImageForCurrentTime):
* UIProcess/GPU/GPUProcessProxy.h:
Modified Paths
Diff
Modified: trunk/Source/WebKit/ChangeLog (289220 => 289221)
--- trunk/Source/WebKit/ChangeLog 2022-02-07 15:40:19 UTC (rev 289220)
+++ trunk/Source/WebKit/ChangeLog 2022-02-07 16:01:55 UTC (rev 289221)
@@ -1,3 +1,35 @@
+2022-02-07 Wenson Hsieh <wenson_hs...@apple.com>
+
+ Add a helper method on VideoFullscreenManagerProxy to request video frame data
+ https://bugs.webkit.org/show_bug.cgi?id=236094
+
+ Reviewed by Darin Adler.
+
+ Add a helper method on VideoFullscreenManagerProxy to asynchronously grab an image bitmap for the fullscreen
+ video corresponding to the given PlaybackSessionContextIdentifier, by sending IPC to the GPU process. This
+ mechanism will be used in a subsequent patch to support "fullscreen video extraction", which involves capturing
+ image data for paused, fullscreen videos.
+
+ See below for more details.
+
+ * GPUProcess/GPUProcess.cpp:
+ (WebKit::GPUProcess::requestBitmapImageForCurrentTime):
+ * GPUProcess/GPUProcess.h:
+ * GPUProcess/GPUProcess.messages.in:
+ * GPUProcess/media/RemoteMediaPlayerManagerProxy.cpp:
+ (WebKit::RemoteMediaPlayerManagerProxy::bitmapImageForCurrentTime):
+ * GPUProcess/media/RemoteMediaPlayerManagerProxy.h:
+ * UIProcess/Cocoa/VideoFullscreenManagerProxy.h:
+ * UIProcess/Cocoa/VideoFullscreenManagerProxy.mm:
+ (WebKit::VideoFullscreenManagerProxy::requestBitmapImageForCurrentTime):
+
+ Map the given PlaybackSessionContextIdentifier to a MediaPlayerIdentifier, and then send the (web process ID,
+ media player ID) identifier pair to the GPU process to request frame data for the given media player.
+
+ * UIProcess/GPU/GPUProcessProxy.cpp:
+ (WebKit::GPUProcessProxy::requestBitmapImageForCurrentTime):
+ * UIProcess/GPU/GPUProcessProxy.h:
+
2022-02-07 Carlos Garcia Campos <cgar...@igalia.com>
[WebDriver] Add support for shadow roots
Modified: trunk/Source/WebKit/GPUProcess/GPUProcess.cpp (289220 => 289221)
--- trunk/Source/WebKit/GPUProcess/GPUProcess.cpp 2022-02-07 15:40:19 UTC (rev 289220)
+++ trunk/Source/WebKit/GPUProcess/GPUProcess.cpp 2022-02-07 16:01:55 UTC (rev 289221)
@@ -515,6 +515,17 @@
}
#endif
+void GPUProcess::requestBitmapImageForCurrentTime(WebCore::ProcessIdentifier processIdentifier, WebCore::MediaPlayerIdentifier playerIdentifier, CompletionHandler<void(const ShareableBitmap::Handle&)>&& completion)
+{
+ auto iterator = m_webProcessConnections.find(processIdentifier);
+ if (iterator == m_webProcessConnections.end()) {
+ completion({ });
+ return;
+ }
+
+ completion(iterator->value->remoteMediaPlayerManagerProxy().bitmapImageForCurrentTime(playerIdentifier));
+}
+
} // namespace WebKit
#endif // ENABLE(GPU_PROCESS)
Modified: trunk/Source/WebKit/GPUProcess/GPUProcess.h (289220 => 289221)
--- trunk/Source/WebKit/GPUProcess/GPUProcess.h 2022-02-07 15:40:19 UTC (rev 289220)
+++ trunk/Source/WebKit/GPUProcess/GPUProcess.h 2022-02-07 16:01:55 UTC (rev 289221)
@@ -29,8 +29,10 @@
#include "AuxiliaryProcess.h"
#include "SandboxExtension.h"
+#include "ShareableBitmap.h"
#include "WebPageProxyIdentifier.h"
#include <WebCore/LibWebRTCEnumTraits.h>
+#include <WebCore/MediaPlayerIdentifier.h>
#include <WebCore/Timer.h>
#include <pal/SessionID.h>
#include <wtf/Function.h>
@@ -115,6 +117,8 @@
void processIsStartingToCaptureAudio(GPUConnectionToWebProcess&);
#endif
+ void requestBitmapImageForCurrentTime(WebCore::ProcessIdentifier, WebCore::MediaPlayerIdentifier, CompletionHandler<void(const ShareableBitmap::Handle&)>&&);
+
private:
void lowMemoryHandler(Critical, Synchronous);
Modified: trunk/Source/WebKit/GPUProcess/GPUProcess.messages.in (289220 => 289221)
--- trunk/Source/WebKit/GPUProcess/GPUProcess.messages.in 2022-02-07 15:40:19 UTC (rev 289220)
+++ trunk/Source/WebKit/GPUProcess/GPUProcess.messages.in 2022-02-07 16:01:55 UTC (rev 289221)
@@ -72,6 +72,8 @@
SetMediaSourceInlinePaintingEnabled(bool enabled);
#endif
+ RequestBitmapImageForCurrentTime(WebCore::ProcessIdentifier processIdentifier, WebCore::MediaPlayerIdentifier playerIdentifier) -> (WebKit::ShareableBitmap::Handle handle) Async
+
#if ENABLE(CFPREFS_DIRECT_MODE)
NotifyPreferencesChanged(String domain, String key, std::optional<String> encodedValue)
#endif
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerManagerProxy.cpp (289220 => 289221)
--- trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerManagerProxy.cpp 2022-02-07 15:40:19 UTC (rev 289220)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerManagerProxy.cpp 2022-02-07 16:01:55 UTC (rev 289221)
@@ -200,6 +200,32 @@
}
#endif
+ShareableBitmap::Handle RemoteMediaPlayerManagerProxy::bitmapImageForCurrentTime(WebCore::MediaPlayerIdentifier identifier)
+{
+ auto player = mediaPlayer(identifier);
+ if (!player)
+ return { };
+
+ auto image = player->nativeImageForCurrentTime();
+ if (!image)
+ return { };
+
+ auto imageSize = image->size();
+ auto bitmap = ShareableBitmap::createShareable(imageSize, { player->colorSpace() });
+ if (!bitmap)
+ return { };
+
+ auto context = bitmap->createGraphicsContext();
+ if (!context)
+ return { };
+
+ context->drawNativeImage(*image, imageSize, FloatRect { { }, imageSize }, FloatRect { { }, imageSize });
+
+ ShareableBitmap::Handle bitmapHandle;
+ bitmap->createHandle(bitmapHandle);
+ return bitmapHandle;
+}
+
} // namespace WebKit
#endif
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerManagerProxy.h (289220 => 289221)
--- trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerManagerProxy.h 2022-02-07 15:40:19 UTC (rev 289220)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerManagerProxy.h 2022-02-07 16:01:55 UTC (rev 289221)
@@ -31,6 +31,7 @@
#include "GPUConnectionToWebProcess.h"
#include "MessageReceiver.h"
#include "SandboxExtension.h"
+#include "ShareableBitmap.h"
#include "TrackPrivateRemoteIdentifier.h"
#include <WebCore/MediaPlayer.h>
#include <WebCore/MediaPlayerIdentifier.h>
@@ -67,6 +68,8 @@
RefPtr<WebCore::MediaPlayer> mediaPlayer(const WebCore::MediaPlayerIdentifier&);
bool allowsExitUnderMemoryPressure() const;
+ ShareableBitmap::Handle bitmapImageForCurrentTime(WebCore::MediaPlayerIdentifier);
+
private:
// IPC::MessageReceiver
void didReceiveMessage(IPC::Connection&, IPC::Decoder&) final;
Modified: trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.h (289220 => 289221)
--- trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.h 2022-02-07 15:40:19 UTC (rev 289220)
+++ trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.h 2022-02-07 16:01:55 UTC (rev 289221)
@@ -30,6 +30,7 @@
#include "LayerHostingContext.h"
#include "MessageReceiver.h"
#include "PlaybackSessionContextIdentifier.h"
+#include "ShareableBitmap.h"
#include <WebCore/AudioSession.h>
#include <WebCore/GraphicsLayer.h>
#include <WebCore/MediaPlayerIdentifier.h>
@@ -151,6 +152,8 @@
void forEachSession(Function<void(VideoFullscreenModelContext&, PlatformVideoFullscreenInterface&)>&&);
+ void requestBitmapImageForCurrentTime(PlaybackSessionContextIdentifier, CompletionHandler<void(const ShareableBitmap::Handle&)>&&);
+
private:
friend class VideoFullscreenModelContext;
Modified: trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.mm (289220 => 289221)
--- trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.mm 2022-02-07 15:40:19 UTC (rev 289220)
+++ trunk/Source/WebKit/UIProcess/Cocoa/VideoFullscreenManagerProxy.mm 2022-02-07 16:01:55 UTC (rev 289221)
@@ -30,6 +30,7 @@
#import "APIUIClient.h"
#import "DrawingAreaProxy.h"
+#import "GPUProcessProxy.h"
#import "PlaybackSessionManagerProxy.h"
#import "VideoFullscreenManagerMessages.h"
#import "VideoFullscreenManagerProxyMessages.h"
@@ -511,6 +512,29 @@
}
}
+void VideoFullscreenManagerProxy::requestBitmapImageForCurrentTime(PlaybackSessionContextIdentifier identifier, CompletionHandler<void(const ShareableBitmap::Handle&)>&& completionHandler)
+{
+ auto* gpuProcess = GPUProcessProxy::singletonIfCreated();
+ if (!gpuProcess) {
+ completionHandler({ });
+ return;
+ }
+
+ auto* interface = findInterface(identifier);
+ if (!interface) {
+ completionHandler({ });
+ return;
+ }
+
+ auto playerIdentifier = valueOrDefault(interface->playerIdentifier());
+ if (!playerIdentifier) {
+ completionHandler({ });
+ return;
+ }
+
+ gpuProcess->requestBitmapImageForCurrentTime(m_page->process().coreProcessIdentifier(), playerIdentifier, WTFMove(completionHandler));
+}
+
void VideoFullscreenManagerProxy::addVideoInPictureInPictureDidChangeObserver(const VideoInPictureInPictureDidChangeObserver& observer)
{
ASSERT(!m_pipChangeObservers.contains(observer));
Modified: trunk/Source/WebKit/UIProcess/GPU/GPUProcessProxy.cpp (289220 => 289221)
--- trunk/Source/WebKit/UIProcess/GPU/GPUProcessProxy.cpp 2022-02-07 15:40:19 UTC (rev 289220)
+++ trunk/Source/WebKit/UIProcess/GPU/GPUProcessProxy.cpp 2022-02-07 16:01:55 UTC (rev 289221)
@@ -648,6 +648,11 @@
}
#endif
+void GPUProcessProxy::requestBitmapImageForCurrentTime(ProcessIdentifier processIdentifier, MediaPlayerIdentifier playerIdentifier, CompletionHandler<void(const ShareableBitmap::Handle&)>&& completion)
+{
+ sendWithAsyncReply(Messages::GPUProcess::RequestBitmapImageForCurrentTime(processIdentifier, playerIdentifier), WTFMove(completion));
+}
+
} // namespace WebKit
#undef MESSAGE_CHECK
Modified: trunk/Source/WebKit/UIProcess/GPU/GPUProcessProxy.h (289220 => 289221)
--- trunk/Source/WebKit/UIProcess/GPU/GPUProcessProxy.h 2022-02-07 15:40:19 UTC (rev 289220)
+++ trunk/Source/WebKit/UIProcess/GPU/GPUProcessProxy.h 2022-02-07 16:01:55 UTC (rev 289221)
@@ -33,8 +33,10 @@
#include "ProcessTerminationReason.h"
#include "ProcessThrottler.h"
#include "ProcessThrottlerClient.h"
+#include "ShareableBitmap.h"
#include "WebPageProxyIdentifier.h"
#include "WebProcessProxyMessagesReplies.h"
+#include <WebCore/MediaPlayerIdentifier.h>
#include <WebCore/PageIdentifier.h>
#include <memory>
#include <pal/SessionID.h>
@@ -99,6 +101,8 @@
void terminateForTesting();
void webProcessConnectionCountForTesting(CompletionHandler<void(uint64_t)>&&);
+ void requestBitmapImageForCurrentTime(WebCore::ProcessIdentifier, WebCore::MediaPlayerIdentifier, CompletionHandler<void(const ShareableBitmap::Handle&)>&&);
+
private:
explicit GPUProcessProxy();
