- Revision
- 266993
- Author
- wenson_hs...@apple.com
- Date
- 2020-09-13 08:43:58 -0700 (Sun, 13 Sep 2020)
Log Message
Add a key to the text manipulation userInfo dictionary indicating whether the translated item is on-screen
https://bugs.webkit.org/show_bug.cgi?id=216452
<rdar://problem/68785397>
Reviewed by Darin Adler.
Source/WebCore:
For debugging purposes, and also to provide a hint as to what text should be prioritized when translating
web pages, WebKit clients have requested a new field in the userInfo metadata dictionary associated with each
token that indicates whether or not a translation (i.e. text manipulation) token represents an element that is
currently on-screen. See below for more details.
Test: TextManipulation.StartTextManipulationExtractsUserInfo
* editing/TextManipulationController.cpp:
(WebCore::tokenInfo):
Set the flag by checking whether or not the absolute bounding rect intersects with the visible content rect of
the enclosing frame. Note that since subframe content is currently never extracted for translation, we don't
need logic yet to recursively check that parent iframe elements are visible.
* editing/TextManipulationController.h:
Add a new `bool` flag in `ManipulationTokenInfo`.
(WebCore::TextManipulationController::ManipulationTokenInfo::encode const):
(WebCore::TextManipulationController::ManipulationTokenInfo::decode):
Source/WebKit:
Add `_WKTextManipulationTokenUserInfoVisibilityKey` and set its value to the value of the `isVisible` member in
`ManipulationTokenInfo`. See WebCore ChangeLog for more details.
* UIProcess/API/Cocoa/WKWebView.mm:
(createUserInfo):
* UIProcess/API/Cocoa/_WKTextManipulationToken.h:
* UIProcess/API/Cocoa/_WKTextManipulationToken.mm:
Tools:
Adjust an existing test so that it adds a fourth text paragraph with 2000px of top margin, and also
programmatically scrolls after loading the page so that only this last paragraph is visible. We expect the
metadata to indicate that none of the other tokens except this last one has a value of `YES` for
`_WKTextManipulationTokenUserInfoVisibilityKey`.
* TestWebKitAPI/Tests/WebKitCocoa/TextManipulation.mm:
(TestWebKitAPI::TEST):
Modified Paths
Diff
Modified: trunk/Source/WebCore/ChangeLog (266992 => 266993)
--- trunk/Source/WebCore/ChangeLog 2020-09-13 12:07:50 UTC (rev 266992)
+++ trunk/Source/WebCore/ChangeLog 2020-09-13 15:43:58 UTC (rev 266993)
@@ -1,3 +1,32 @@
+2020-09-13 Wenson Hsieh <wenson_hs...@apple.com>
+
+ Add a key to the text manipulation userInfo dictionary indicating whether the translated item is on-screen
+ https://bugs.webkit.org/show_bug.cgi?id=216452
+ <rdar://problem/68785397>
+
+ Reviewed by Darin Adler.
+
+ For debugging purposes, and also to provide a hint as to what text should be prioritized when translating
+ web pages, WebKit clients have requested a new field in the userInfo metadata dictionary associated with each
+ token that indicates whether or not a translation (i.e. text manipulation) token represents an element that is
+ currently on-screen. See below for more details.
+
+ Test: TextManipulation.StartTextManipulationExtractsUserInfo
+
+ * editing/TextManipulationController.cpp:
+ (WebCore::tokenInfo):
+
+ Set the flag by checking whether or not the absolute bounding rect intersects with the visible content rect of
+ the enclosing frame. Note that since subframe content is currently never extracted for translation, we don't
+ need logic yet to recursively check that parent iframe elements are visible.
+
+ * editing/TextManipulationController.h:
+
+ Add a new `bool` flag in `ManipulationTokenInfo`.
+
+ (WebCore::TextManipulationController::ManipulationTokenInfo::encode const):
+ (WebCore::TextManipulationController::ManipulationTokenInfo::decode):
+
2020-09-13 Philippe Normand <pnorm...@igalia.com>
Unreviewed, GTK/WPE EME build fixes after r266721
Modified: trunk/Source/WebCore/editing/TextManipulationController.cpp (266992 => 266993)
--- trunk/Source/WebCore/editing/TextManipulationController.cpp 2020-09-13 12:07:50 UTC (rev 266992)
+++ trunk/Source/WebCore/editing/TextManipulationController.cpp 2020-09-13 15:43:58 UTC (rev 266993)
@@ -297,6 +297,12 @@
result.tagName = element->tagName();
if (element->hasAttributeWithoutSynchronization(HTMLNames::roleAttr))
result.roleAttribute = element->attributeWithoutSynchronization(HTMLNames::roleAttr);
+ if (auto frame = makeRefPtr(node->document().frame()); frame && frame->view() && element->renderer()) {
+ // FIXME: This doesn't account for overflow clip.
+ auto elementRect = element->renderer()->absoluteAnchorRect();
+ auto visibleContentRect = frame->view()->visibleContentRect();
+ result.isVisible = visibleContentRect.intersects(enclosingIntRect(elementRect));
+ }
}
return result;
}
Modified: trunk/Source/WebCore/editing/TextManipulationController.h (266992 => 266993)
--- trunk/Source/WebCore/editing/TextManipulationController.h 2020-09-13 12:07:50 UTC (rev 266992)
+++ trunk/Source/WebCore/editing/TextManipulationController.h 2020-09-13 15:43:58 UTC (rev 266993)
@@ -52,6 +52,7 @@
String tagName;
String roleAttribute;
URL documentURL;
+ bool isVisible { false };
template<class Encoder> void encode(Encoder&) const;
template<class Decoder> static Optional<ManipulationTokenInfo> decode(Decoder&);
@@ -202,6 +203,7 @@
encoder << tagName;
encoder << roleAttribute;
encoder << documentURL;
+ encoder << isVisible;
}
template<class Decoder>
@@ -217,6 +219,9 @@
if (!decoder.decode(result.documentURL))
return WTF::nullopt;
+ if (!decoder.decode(result.isVisible))
+ return WTF::nullopt;
+
return result;
}
Modified: trunk/Source/WebKit/ChangeLog (266992 => 266993)
--- trunk/Source/WebKit/ChangeLog 2020-09-13 12:07:50 UTC (rev 266992)
+++ trunk/Source/WebKit/ChangeLog 2020-09-13 15:43:58 UTC (rev 266993)
@@ -1,3 +1,19 @@
+2020-09-13 Wenson Hsieh <wenson_hs...@apple.com>
+
+ Add a key to the text manipulation userInfo dictionary indicating whether the translated item is on-screen
+ https://bugs.webkit.org/show_bug.cgi?id=216452
+ <rdar://problem/68785397>
+
+ Reviewed by Darin Adler.
+
+ Add `_WKTextManipulationTokenUserInfoVisibilityKey` and set its value to the value of the `isVisible` member in
+ `ManipulationTokenInfo`. See WebCore ChangeLog for more details.
+
+ * UIProcess/API/Cocoa/WKWebView.mm:
+ (createUserInfo):
+ * UIProcess/API/Cocoa/_WKTextManipulationToken.h:
+ * UIProcess/API/Cocoa/_WKTextManipulationToken.mm:
+
2020-09-13 Pablo Saavedra <psaave...@igalia.com>
[GTK][WPE] Build fails when -DENABLE_SERVICE_WORKER=OFF
Modified: trunk/Source/WebKit/UIProcess/API/Cocoa/WKWebView.mm (266992 => 266993)
--- trunk/Source/WebKit/UIProcess/API/Cocoa/WKWebView.mm 2020-09-13 12:07:50 UTC (rev 266992)
+++ trunk/Source/WebKit/UIProcess/API/Cocoa/WKWebView.mm 2020-09-13 15:43:58 UTC (rev 266993)
@@ -1622,6 +1622,7 @@
[result setObject:(NSString *)info->tagName forKey:_WKTextManipulationTokenUserInfoTagNameKey];
if (!info->roleAttribute.isNull())
[result setObject:(NSString *)info->roleAttribute forKey:_WKTextManipulationTokenUserInfoRoleAttributeKey];
+ [result setObject:@(info->isVisible) forKey:_WKTextManipulationTokenUserInfoVisibilityKey];
return result;
}
Modified: trunk/Source/WebKit/UIProcess/API/Cocoa/_WKTextManipulationToken.h (266992 => 266993)
--- trunk/Source/WebKit/UIProcess/API/Cocoa/_WKTextManipulationToken.h 2020-09-13 12:07:50 UTC (rev 266992)
+++ trunk/Source/WebKit/UIProcess/API/Cocoa/_WKTextManipulationToken.h 2020-09-13 15:43:58 UTC (rev 266993)
@@ -31,6 +31,7 @@
WK_EXTERN NSString * const _WKTextManipulationTokenUserInfoDocumentURLKey WK_API_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA));
WK_EXTERN NSString * const _WKTextManipulationTokenUserInfoTagNameKey WK_API_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA));
WK_EXTERN NSString * const _WKTextManipulationTokenUserInfoRoleAttributeKey WK_API_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA));
+WK_EXTERN NSString * const _WKTextManipulationTokenUserInfoVisibilityKey WK_API_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA));
WK_CLASS_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA))
@interface _WKTextManipulationToken : NSObject
Modified: trunk/Source/WebKit/UIProcess/API/Cocoa/_WKTextManipulationToken.mm (266992 => 266993)
--- trunk/Source/WebKit/UIProcess/API/Cocoa/_WKTextManipulationToken.mm 2020-09-13 12:07:50 UTC (rev 266992)
+++ trunk/Source/WebKit/UIProcess/API/Cocoa/_WKTextManipulationToken.mm 2020-09-13 15:43:58 UTC (rev 266993)
@@ -31,6 +31,7 @@
NSString * const _WKTextManipulationTokenUserInfoDocumentURLKey = @"_WKTextManipulationTokenUserInfoDocumentURLKey";
NSString * const _WKTextManipulationTokenUserInfoTagNameKey = @"_WKTextManipulationTokenUserInfoTagNameKey";
NSString * const _WKTextManipulationTokenUserInfoRoleAttributeKey = @"_WKTextManipulationTokenUserInfoRoleAttributeKey";
+NSString * const _WKTextManipulationTokenUserInfoVisibilityKey = @"_WKTextManipulationTokenUserInfoVisibilityKey";
@implementation _WKTextManipulationToken {
RetainPtr<NSDictionary<NSString *, id>> _userInfo;
Modified: trunk/Tools/ChangeLog (266992 => 266993)
--- trunk/Tools/ChangeLog 2020-09-13 12:07:50 UTC (rev 266992)
+++ trunk/Tools/ChangeLog 2020-09-13 15:43:58 UTC (rev 266993)
@@ -1,3 +1,19 @@
+2020-09-13 Wenson Hsieh <wenson_hs...@apple.com>
+
+ Add a key to the text manipulation userInfo dictionary indicating whether the translated item is on-screen
+ https://bugs.webkit.org/show_bug.cgi?id=216452
+ <rdar://problem/68785397>
+
+ Reviewed by Darin Adler.
+
+ Adjust an existing test so that it adds a fourth text paragraph with 2000px of top margin, and also
+ programmatically scrolls after loading the page so that only this last paragraph is visible. We expect the
+ metadata to indicate that none of the other tokens except this last one has a value of `YES` for
+ `_WKTextManipulationTokenUserInfoVisibilityKey`.
+
+ * TestWebKitAPI/Tests/WebKitCocoa/TextManipulation.mm:
+ (TestWebKitAPI::TEST):
+
2020-09-12 Darin Adler <da...@apple.com>
Send TestRendered event after running a test but before dumping
Modified: trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/TextManipulation.mm (266992 => 266993)
--- trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/TextManipulation.mm 2020-09-13 12:07:50 UTC (rev 266992)
+++ trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/TextManipulation.mm 2020-09-13 15:43:58 UTC (rev 266993)
@@ -801,8 +801,12 @@
" <p>First</p>"
" <div role='button'>Second</div>"
" <span>Third</span>"
+ " <div style='margin-top: 2000px;'>Fourth</div>"
+ " <script>scrollTo(0, 2000);</script>"
"</body>"];
+ [webView waitForNextPresentationUpdate];
+
done = false;
[webView _startTextManipulationsWithConfiguration:nil completion:^{
done = true;
@@ -810,24 +814,28 @@
TestWebKitAPI::Util::run(&done);
auto items = [delegate items];
- EXPECT_EQ(items.count, 4UL);
+ EXPECT_EQ(items.count, 5UL);
EXPECT_EQ(items[0].tokens.count, 1UL);
EXPECT_EQ(items[1].tokens.count, 1UL);
EXPECT_EQ(items[2].tokens.count, 1UL);
EXPECT_EQ(items[3].tokens.count, 1UL);
+ EXPECT_EQ(items[4].tokens.count, 1UL);
EXPECT_WK_STREQ("This is a test", items[0].tokens[0].content);
EXPECT_WK_STREQ("First", items[1].tokens[0].content);
EXPECT_WK_STREQ("Second", items[2].tokens[0].content);
EXPECT_WK_STREQ("Third", items[3].tokens[0].content);
+ EXPECT_WK_STREQ("Fourth", items[4].tokens[0].content);
{
auto userInfo = items[0].tokens[0].userInfo;
EXPECT_WK_STREQ("TestWebKitAPI.resources", [(NSURL *)userInfo[_WKTextManipulationTokenUserInfoDocumentURLKey] lastPathComponent]);
EXPECT_WK_STREQ("TITLE", (NSString *)userInfo[_WKTextManipulationTokenUserInfoTagNameKey]);
+ EXPECT_FALSE([userInfo[_WKTextManipulationTokenUserInfoVisibilityKey] boolValue]);
}
{
auto userInfo = items[1].tokens[0].userInfo;
EXPECT_WK_STREQ("TestWebKitAPI.resources", [(NSURL *)userInfo[_WKTextManipulationTokenUserInfoDocumentURLKey] lastPathComponent]);
EXPECT_WK_STREQ("P", (NSString *)userInfo[_WKTextManipulationTokenUserInfoTagNameKey]);
+ EXPECT_FALSE([userInfo[_WKTextManipulationTokenUserInfoVisibilityKey] boolValue]);
}
{
auto userInfo = items[2].tokens[0].userInfo;
@@ -834,12 +842,20 @@
EXPECT_WK_STREQ("TestWebKitAPI.resources", [(NSURL *)userInfo[_WKTextManipulationTokenUserInfoDocumentURLKey] lastPathComponent]);
EXPECT_WK_STREQ("DIV", (NSString *)userInfo[_WKTextManipulationTokenUserInfoTagNameKey]);
EXPECT_WK_STREQ("button", (NSString *)userInfo[_WKTextManipulationTokenUserInfoRoleAttributeKey]);
+ EXPECT_FALSE([userInfo[_WKTextManipulationTokenUserInfoVisibilityKey] boolValue]);
}
{
auto userInfo = items[3].tokens[0].userInfo;
EXPECT_WK_STREQ("TestWebKitAPI.resources", [(NSURL *)userInfo[_WKTextManipulationTokenUserInfoDocumentURLKey] lastPathComponent]);
EXPECT_WK_STREQ("SPAN", (NSString *)userInfo[_WKTextManipulationTokenUserInfoTagNameKey]);
+ EXPECT_FALSE([userInfo[_WKTextManipulationTokenUserInfoVisibilityKey] boolValue]);
}
+ {
+ auto userInfo = items[4].tokens[0].userInfo;
+ EXPECT_WK_STREQ("TestWebKitAPI.resources", [(NSURL *)userInfo[_WKTextManipulationTokenUserInfoDocumentURLKey] lastPathComponent]);
+ EXPECT_WK_STREQ("DIV", (NSString *)userInfo[_WKTextManipulationTokenUserInfoTagNameKey]);
+ EXPECT_TRUE([userInfo[_WKTextManipulationTokenUserInfoVisibilityKey] boolValue]);
+ }
}
TEST(TextManipulation, StartTextManipulationExtractsValuesFromButtonInputs)
