Diff
Modified: trunk/Source/WebCore/ChangeLog (289762 => 289763)
--- trunk/Source/WebCore/ChangeLog 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebCore/ChangeLog 2022-02-14 22:15:35 UTC (rev 289763)
@@ -1,3 +1,24 @@
+2022-02-14 Wenson Hsieh <wenson_hs...@apple.com>
+
+ [Live Text] Plumb platform image analysis objects to the web process
+ https://bugs.webkit.org/show_bug.cgi?id=236535
+ rdar://88845367
+
+ Reviewed by Devin Rousso.
+
+ Add a `platformData` member to TextRecognitionResult, which (when `ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)` is
+ enabled) contains a pointer to VKCImageAnalysis. See WebKit/ChangeLog for more details.
+
+ * SourcesCocoa.txt:
+ * WebCore.xcodeproj/project.pbxproj:
+ * platform/TextRecognitionResult.h:
+
+ Also add new WebKitAdditions extension points around TextRecognitionResult.
+
+ (WebCore::TextRecognitionResult::encode const):
+ (WebCore::TextRecognitionResult::decode):
+ * platform/cocoa/TextRecognitionResultCocoa.mm:
+
2022-02-14 Devin Rousso <drou...@apple.com>
REGRESSION(r286560): MediaElement's getStartDate returning an incorrect time
Modified: trunk/Source/WebCore/PAL/ChangeLog (289762 => 289763)
--- trunk/Source/WebCore/PAL/ChangeLog 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebCore/PAL/ChangeLog 2022-02-14 22:15:35 UTC (rev 289763)
@@ -1,3 +1,23 @@
+2022-02-14 Wenson Hsieh <wenson_hs...@apple.com>
+
+ [Live Text] Plumb platform image analysis objects to the web process
+ https://bugs.webkit.org/show_bug.cgi?id=236535
+ rdar://88845367
+
+ Reviewed by Devin Rousso.
+
+ Add soft-linking support for VKCImageAnalysis. See WebCore and WebKit ChangeLogs for more detail.
+
+ * pal/cocoa/AVFoundationSoftLink.h:
+
+ Avoid a build error after shifting around Cocoa unified sources, by removing an unused definition of
+ AVOutputContextOutputDevicesDidChangeNotification. This definition causes the unified build to break downstream,
+ when other media-related source files end up importing AVKit headers that try to reference this symbol
+ (AVOutputContextOutputDevicesDidChangeNotification) but end up expanding the WebKit soft-linking macro instead.
+
+ * pal/cocoa/VisionKitCoreSoftLink.h:
+ * pal/cocoa/VisionKitCoreSoftLink.mm:
+
2022-02-12 Jer Noble <jer.no...@apple.com>
Add settings to restrict media containers and codecs when in Captive Portal mode
Modified: trunk/Source/WebCore/PAL/pal/cocoa/AVFoundationSoftLink.h (289762 => 289763)
--- trunk/Source/WebCore/PAL/pal/cocoa/AVFoundationSoftLink.h 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebCore/PAL/pal/cocoa/AVFoundationSoftLink.h 2022-02-14 22:15:35 UTC (rev 289763)
@@ -343,7 +343,6 @@
#if HAVE(AVROUTEPICKERVIEW)
SOFT_LINK_CONSTANT_FOR_HEADER(PAL, AVFoundation, AVOutputContextOutputDevicesDidChangeNotification, NSNotificationName)
-#define AVOutputContextOutputDevicesDidChangeNotification PAL::get_AVFoundation_AVOutputContextOutputDevicesDidChangeNotification()
#endif // HAVE(AVROUTEPICKERVIEW)
#if PLATFORM(COCOA)
Modified: trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.h (289762 => 289763)
--- trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.h 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.h 2022-02-14 22:15:35 UTC (rev 289763)
@@ -35,6 +35,7 @@
SOFT_LINK_CLASS_FOR_HEADER(PAL, VKImageAnalyzerRequest)
SOFT_LINK_CLASS_FOR_HEADER(PAL, VKCImageAnalyzer)
SOFT_LINK_CLASS_FOR_HEADER(PAL, VKCImageAnalyzerRequest)
+SOFT_LINK_CLASS_FOR_HEADER(PAL, VKCImageAnalysis)
#if USE(APPLE_INTERNAL_SDK)
#import <WebKitAdditions/VisionKitCoreSoftLinkAdditions.h>
Modified: trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.mm (289762 => 289763)
--- trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.mm 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.mm 2022-02-14 22:15:35 UTC (rev 289763)
@@ -35,6 +35,7 @@
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKImageAnalyzerRequest, PAL_EXPORT, true)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKCImageAnalyzer, PAL_EXPORT, true)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKCImageAnalyzerRequest, PAL_EXPORT, true)
+SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKCImageAnalysis, PAL_EXPORT, true)
#if USE(APPLE_INTERNAL_SDK)
#import <WebKitAdditions/VisionKitCoreSoftLinkAdditions.mm>
Modified: trunk/Source/WebCore/SourcesCocoa.txt (289762 => 289763)
--- trunk/Source/WebCore/SourcesCocoa.txt 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebCore/SourcesCocoa.txt 2022-02-14 22:15:35 UTC (rev 289763)
@@ -275,6 +275,7 @@
platform/cocoa/SharedBufferCocoa.mm
platform/cocoa/SystemVersion.mm
platform/cocoa/TelephoneNumberDetectorCocoa.cpp
+platform/cocoa/TextRecognitionResultCocoa.mm
platform/cocoa/ThemeCocoa.mm
platform/cocoa/UserAgentCocoa.mm
platform/cocoa/VideoFullscreenModelVideoElement.mm
Modified: trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj (289762 => 289763)
--- trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2022-02-14 22:15:35 UTC (rev 289763)
@@ -17869,6 +17869,7 @@
F4D9817D2195FBF6008230FC /* ChangeListTypeCommand.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ChangeListTypeCommand.h; sourceTree = "<group>"; };
F4D9817E2195FBF6008230FC /* ChangeListTypeCommand.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = ChangeListTypeCommand.cpp; sourceTree = "<group>"; };
F4DD866225CC60840059BFDA /* imageOverlay.css */ = {isa = PBXFileReference; lastKnownFileType = text.css; path = imageOverlay.css; sourceTree = "<group>"; };
+ F4DE26CC27B743AE0071FD6E /* TextRecognitionResultCocoa.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = TextRecognitionResultCocoa.mm; sourceTree = "<group>"; };
F4E1965A21F2395000285078 /* JSUndoItemCustom.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = JSUndoItemCustom.cpp; sourceTree = "<group>"; };
F4E1965F21F26E4E00285078 /* UndoItem.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = UndoItem.cpp; sourceTree = "<group>"; };
F4E1966121F27D3C00285078 /* CustomUndoStep.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CustomUndoStep.h; sourceTree = "<group>"; };
@@ -26882,6 +26883,7 @@
5D5975B119635F1100D00878 /* SystemVersion.h */,
5D5975B219635F1100D00878 /* SystemVersion.mm */,
7CC564B918BAC720001B9652 /* TelephoneNumberDetectorCocoa.cpp */,
+ F4DE26CC27B743AE0071FD6E /* TextRecognitionResultCocoa.mm */,
310D71941B335C9D009C7B73 /* ThemeCocoa.h */,
310D71931B335C9D009C7B73 /* ThemeCocoa.mm */,
A3AF9D81203252EE006CAD06 /* UserAgentCocoa.mm */,
Modified: trunk/Source/WebCore/platform/TextRecognitionResult.h (289762 => 289763)
--- trunk/Source/WebCore/platform/TextRecognitionResult.h 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebCore/platform/TextRecognitionResult.h 2022-02-14 22:15:35 UTC (rev 289763)
@@ -27,6 +27,8 @@
#if ENABLE(IMAGE_ANALYSIS)
+OBJC_CLASS VKCImageAnalysis;
+
#if ENABLE(DATA_DETECTION)
OBJC_CLASS DDScannerResult;
#endif
@@ -37,6 +39,8 @@
namespace WebCore {
+struct CharacterRange;
+
struct TextRecognitionWordData {
TextRecognitionWordData(const String& theText, FloatQuad&& quad, bool leadingWhitespace)
: text(theText)
@@ -183,6 +187,10 @@
Vector<TextRecognitionBlockData> blocks;
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ RetainPtr<VKCImageAnalysis> platformData;
+#endif
+
bool isEmpty() const
{
if (!lines.isEmpty())
@@ -210,6 +218,9 @@
encoder << dataDetectors;
#endif
encoder << blocks;
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ encoder << platformData;
+#endif
}
template<class Decoder> std::optional<TextRecognitionResult> TextRecognitionResult::decode(Decoder& decoder)
@@ -231,6 +242,13 @@
if (!blocks)
return std::nullopt;
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ std::optional<RetainPtr<VKCImageAnalysis>> platformData;
+ decoder >> platformData;
+ if (!platformData)
+ return std::nullopt;
+#endif
+
return {{
WTFMove(*lines),
#if ENABLE(DATA_DETECTION)
@@ -237,9 +255,16 @@
WTFMove(*dataDetectors),
#endif
WTFMove(*blocks),
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ WTFMove(*platformData),
+#endif
}};
}
} // namespace WebCore
+#if USE(APPLE_INTERNAL_SDK)
+#include <WebKitAdditions/TextRecognitionResultAdditions.h>
+#endif
+
#endif // ENABLE(IMAGE_ANALYSIS)
Copied: trunk/Source/WebCore/platform/cocoa/TextRecognitionResultCocoa.mm (from rev 289762, trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.h) (0 => 289763)
--- trunk/Source/WebCore/platform/cocoa/TextRecognitionResultCocoa.mm (rev 0)
+++ trunk/Source/WebCore/platform/cocoa/TextRecognitionResultCocoa.mm 2022-02-14 22:15:35 UTC (rev 289763)
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#import "config.h"
+#import "TextRecognitionResult.h"
+
+#if USE(APPLE_INTERNAL_SDK)
+#import <WebKitAdditions/TextRecognitionResultCocoaAdditions.mm>
+#endif
Modified: trunk/Source/WebKit/ChangeLog (289762 => 289763)
--- trunk/Source/WebKit/ChangeLog 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebKit/ChangeLog 2022-02-14 22:15:35 UTC (rev 289763)
@@ -1,3 +1,26 @@
+2022-02-14 Wenson Hsieh <wenson_hs...@apple.com>
+
+ [Live Text] Plumb platform image analysis objects to the web process
+ https://bugs.webkit.org/show_bug.cgi?id=236535
+ rdar://88845367
+
+ Reviewed by Devin Rousso.
+
+ Plumb the platform text recognition results object (VKCImageAnalysis) through TextRecognitionResults to the
+ web process when computing text recognition results for Live Text, and add a few new WebKitAdditions extension
+ points. In a subsequent patch, we'll use this VKCImageAnalysis on TextRecognitionResults to refactor how we
+ write Live Text to the system pasteboard when copying.
+
+ * Platform/cocoa/TextRecognitionUtilities.mm:
+ (WebKit::makeTextRecognitionResult):
+
+ Set `platformData`.
+
+ * Shared/Cocoa/WebCoreArgumentCodersCocoa.mm:
+ (IPC::ArgumentCoder<RetainPtr<VKCImageAnalysis>>::encode):
+ (IPC::ArgumentCoder<RetainPtr<VKCImageAnalysis>>::decode):
+ * Shared/WebCoreArgumentCoders.h:
+
2022-02-14 J Pascoe <j_pas...@apple.com>
[WebAuthn] Access group not set in add query for importLocalAuthenticatorCredential
Modified: trunk/Source/WebKit/Platform/cocoa/TextRecognitionUtilities.mm (289762 => 289763)
--- trunk/Source/WebKit/Platform/cocoa/TextRecognitionUtilities.mm 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebKit/Platform/cocoa/TextRecognitionUtilities.mm 2022-02-14 22:15:35 UTC (rev 289763)
@@ -120,7 +120,7 @@
#if ENABLE(DATA_DETECTION)
if ([analysis respondsToSelector:@selector(textDataDetectors)]) {
- auto dataDetectors = retainPtr(analysis.textDataDetectors);
+ auto dataDetectors = RetainPtr { analysis.textDataDetectors };
result.dataDetectors.reserveInitialCapacity([dataDetectors count]);
for (VKWKDataDetectorInfo *info in dataDetectors.get())
result.dataDetectors.uncheckedAppend({ info.result, floatQuads(info.boundingQuads) });
@@ -127,6 +127,10 @@
}
#endif // ENABLE(DATA_DETECTION)
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ result.platformData = { RetainPtr { analysis } };
+#endif
+
return result;
}
Modified: trunk/Source/WebKit/Shared/Cocoa/WebCoreArgumentCodersCocoa.mm (289762 => 289763)
--- trunk/Source/WebKit/Shared/Cocoa/WebCoreArgumentCodersCocoa.mm 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebKit/Shared/Cocoa/WebCoreArgumentCodersCocoa.mm 2022-02-14 22:15:35 UTC (rev 289763)
@@ -73,6 +73,8 @@
#import <WebCore/CoreVideoSoftLink.h>
#endif
+#import <pal/cocoa/VisionKitCoreSoftLink.h>
+
namespace IPC {
void ArgumentCoder<WebCore::AttributedString>::encode(Encoder& encoder, const WebCore::AttributedString& attributedString)
@@ -730,6 +732,20 @@
#endif // ENABLE(IMAGE_ANALYSIS) && ENABLE(DATA_DETECTION)
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
+void ArgumentCoder<RetainPtr<VKCImageAnalysis>>::encode(Encoder& encoder, const RetainPtr<VKCImageAnalysis>& data)
+{
+ encoder << data.get();
+}
+
+std::optional<RetainPtr<VKCImageAnalysis>> ArgumentCoder<RetainPtr<VKCImageAnalysis>>::decode(Decoder& decoder)
+{
+ return IPC::decode<VKCImageAnalysis>(decoder, @[ PAL::getVKCImageAnalysisClass() ]);
+}
+
+#endif // ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
#if USE(AVFOUNDATION)
void ArgumentCoder<RetainPtr<CVPixelBufferRef>>::encode(Encoder& encoder, const RetainPtr<CVPixelBufferRef>& pixelBuffer)
Modified: trunk/Source/WebKit/Shared/WebCoreArgumentCoders.h (289762 => 289763)
--- trunk/Source/WebKit/Shared/WebCoreArgumentCoders.h 2022-02-14 21:51:29 UTC (rev 289762)
+++ trunk/Source/WebKit/Shared/WebCoreArgumentCoders.h 2022-02-14 22:15:35 UTC (rev 289763)
@@ -103,6 +103,8 @@
}
#endif
+OBJC_CLASS VKCImageAnalysis;
+
#if USE(AVFOUNDATION)
typedef struct __CVBuffer* CVPixelBufferRef;
#endif
@@ -823,6 +825,15 @@
#endif // ENABLE(IMAGE_ANALYSIS) && ENABLE(DATA_DETECTION)
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
+template<> struct ArgumentCoder<RetainPtr<VKCImageAnalysis>> {
+ static void encode(Encoder&, const RetainPtr<VKCImageAnalysis>&);
+ static WARN_UNUSED_RETURN std::optional<RetainPtr<VKCImageAnalysis>> decode(Decoder&);
+};
+
+#endif // ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
#if USE(AVFOUNDATION)
template<> struct ArgumentCoder<RetainPtr<CVPixelBufferRef>> {
