Modified: trunk/Source/WebCore/ChangeLog (285861 => 285862)
--- trunk/Source/WebCore/ChangeLog 2021-11-16 15:24:18 UTC (rev 285861)
+++ trunk/Source/WebCore/ChangeLog 2021-11-16 16:37:05 UTC (rev 285862)
@@ -1,3 +1,41 @@
+2021-11-16 Wenson Hsieh <wenson_hs...@apple.com>
+
+ Add support for injecting and rendering text recognition blocks
+ https://bugs.webkit.org/show_bug.cgi?id=233044
+
+ Reviewed by Aditya Keerthi.
+
+ Adds support for rendering text recognition blocks, which appear as opaque div elements over images. See below
+ for more details; no change in behavior, since nothing currently generates TextRecognitionBlockData yet.
+
+ * dom/ImageOverlay.cpp:
+ (WebCore::ImageOverlay::imageOverlayDataDetectorClass):
+ (WebCore::ImageOverlay::imageOverlayBlockClass):
+ (WebCore::ImageOverlay::isDataDetectorResult):
+ (WebCore::ImageOverlay::updateSubtree):
+
+ Add support for creating text recognition block containers in the UA shadow root, if needed.
+
+ (WebCore::ImageOverlay::fitElementToQuad):
+
+ Factor out logic to adjust the width, height and transforms on a given element to fit the given quad, and return
+ rotated bounding rect info; we use this in three places below.
+
+ (WebCore::ImageOverlay::updateWithTextRecognitionResult):
+
+ Add support for adjusting the size and transforms on each of the block containers created above.
+
+ (WebCore::ImageOverlay::imageOverlayDataDetectorClassName): Deleted.
+
+ Rename this to just `imageOverlayDataDetectorClass()` to match the other static helper functions.
+
+ * html/shadow/imageOverlay.css:
+ (div#image-overlay):
+ (div.image-overlay-line):
+ (div.image-overlay-line, div.image-overlay-block):
+ (div.image-overlay-block):
+ (div.image-overlay-line, .image-overlay-text):
+
2021-11-16 Andreu Botella <and...@andreubotella.com>
Empty <input type=file> is represented incorrectly in FormData
Modified: trunk/Source/WebCore/dom/ImageOverlay.cpp (285861 => 285862)
--- trunk/Source/WebCore/dom/ImageOverlay.cpp 2021-11-16 15:24:18 UTC (rev 285861)
+++ trunk/Source/WebCore/dom/ImageOverlay.cpp 2021-11-16 16:37:05 UTC (rev 285862)
@@ -63,7 +63,7 @@
return identifier;
}
-static const AtomString& imageOverlayDataDetectorClassName()
+static const AtomString& imageOverlayDataDetectorClass()
{
static MainThreadNeverDestroyed<const AtomString> className("image-overlay-data-detector-result", AtomString::ConstructFromLiteral);
return className;
@@ -83,6 +83,12 @@
return className;
}
+static const AtomString& imageOverlayBlockClass()
+{
+ static MainThreadNeverDestroyed<const AtomString> className("image-overlay-block", AtomString::ConstructFromLiteral);
+ return className;
+}
+
#endif // ENABLE(IMAGE_ANALYSIS)
bool hasOverlay(const HTMLElement& element)
@@ -106,7 +112,7 @@
bool isDataDetectorResult(const HTMLElement& element)
{
- return imageOverlayHost(element) && element.hasClass() && element.classNames().contains(imageOverlayDataDetectorClassName());
+ return imageOverlayHost(element) && element.hasClass() && element.classNames().contains(imageOverlayDataDetectorClass());
}
bool isInsideOverlay(const SimpleRange& range)
@@ -191,6 +197,7 @@
RefPtr<HTMLDivElement> root;
Vector<LineElements> lines;
Vector<Ref<HTMLDivElement>> dataDetectors;
+ Vector<Ref<HTMLDivElement>> blocks;
};
static Elements updateSubtree(HTMLElement& element, const TextRecognitionResult& result)
@@ -229,17 +236,26 @@
}
if (elements.root) {
- for (auto& lineOrDataDetector : childrenOfType<HTMLDivElement>(*elements.root)) {
- if (!lineOrDataDetector.hasClass())
+ for (auto& childElement : childrenOfType<HTMLDivElement>(*elements.root)) {
+ if (!childElement.hasClass())
continue;
- if (lineOrDataDetector.classList().contains(imageOverlayLineClass())) {
- LineElements lineElements { lineOrDataDetector, { } };
- for (auto& text : childrenOfType<HTMLDivElement>(lineOrDataDetector))
- lineElements.children.append(text);
- elements.lines.append(WTFMove(lineElements));
- } else if (lineOrDataDetector.classList().contains(imageOverlayDataDetectorClassName()))
- elements.dataDetectors.append(lineOrDataDetector);
+ auto& classes = childElement.classList();
+ if (classes.contains(imageOverlayDataDetectorClass())) {
+ elements.dataDetectors.append(childElement);
+ continue;
+ }
+
+ if (classes.contains(imageOverlayBlockClass())) {
+ elements.blocks.append(childElement);
+ continue;
+ }
+
+ ASSERT(classes.contains(imageOverlayLineClass()));
+ LineElements lineElements { childElement, { } };
+ for (auto& text : childrenOfType<HTMLDivElement>(childElement))
+ lineElements.children.append(text);
+ elements.lines.append(WTFMove(lineElements));
}
bool canUseExistingElements = ([&] {
@@ -249,6 +265,9 @@
if (result.lines.size() != elements.lines.size())
return false;
+ if (result.blocks.size() != elements.blocks.size())
+ return false;
+
for (size_t lineIndex = 0; lineIndex < result.lines.size(); ++lineIndex) {
auto& childResults = result.lines[lineIndex].children;
auto& childTextElements = elements.lines[lineIndex].children;
@@ -261,6 +280,11 @@
}
}
+ for (size_t index = 0; index < result.blocks.size(); ++index) {
+ if (result.blocks[index].text != elements.blocks[index]->textContent())
+ return false;
+ }
+
return true;
})();
@@ -310,12 +334,21 @@
elements.dataDetectors.reserveInitialCapacity(result.dataDetectors.size());
for (auto& dataDetector : result.dataDetectors) {
auto dataDetectorContainer = DataDetection::createElementForImageOverlay(document.get(), dataDetector);
- dataDetectorContainer->classList().add(imageOverlayDataDetectorClassName());
+ dataDetectorContainer->classList().add(imageOverlayDataDetectorClass());
rootContainer->appendChild(dataDetectorContainer);
elements.dataDetectors.uncheckedAppend(WTFMove(dataDetectorContainer));
}
#endif // ENABLE(DATA_DETECTION)
+ elements.blocks.reserveInitialCapacity(result.blocks.size());
+ for (auto& block : result.blocks) {
+ auto blockContainer = HTMLDivElement::create(document.get());
+ blockContainer->classList().add(imageOverlayBlockClass());
+ rootContainer->appendChild(blockContainer);
+ blockContainer->appendChild(Text::create(document.get(), makeString('\n', block.text)));
+ elements.blocks.uncheckedAppend(WTFMove(blockContainer));
+ }
+
if (document->quirks().needsToForceUserSelectWhenInstallingImageOverlay())
element.setInlineStyleProperty(CSSPropertyWebkitUserSelect, CSSValueText);
}
@@ -330,6 +363,20 @@
return elements;
}
+static RotatedRect fitElementToQuad(HTMLElement& container, const FloatQuad& quad)
+{
+ auto bounds = rotatedBoundingRectWithMinimumAngleOfRotation(quad, 0.01);
+ container.setInlineStyleProperty(CSSPropertyWidth, bounds.size.width(), CSSUnitType::CSS_PX);
+ container.setInlineStyleProperty(CSSPropertyHeight, bounds.size.height(), CSSUnitType::CSS_PX);
+ container.setInlineStyleProperty(CSSPropertyTransform, makeString(
+ "translate("_s,
+ std::round(bounds.center.x() - (bounds.size.width() / 2)), "px, "_s,
+ std::round(bounds.center.y() - (bounds.size.height() / 2)), "px) "_s,
+ bounds.angleInRadians ? makeString("rotate("_s, bounds.angleInRadians, "rad) "_s) : emptyString()
+ ));
+ return bounds;
+}
+
void updateWithTextRecognitionResult(HTMLElement& element, const TextRecognitionResult& result, CacheTextRecognitionResults cacheTextRecognitionResults)
{
auto elements = updateSubtree(element, result);
@@ -362,16 +409,7 @@
if (lineQuad.isEmpty())
continue;
- auto lineBounds = rotatedBoundingRectWithMinimumAngleOfRotation(lineQuad, 0.01);
- lineContainer->setInlineStyleProperty(CSSPropertyWidth, lineBounds.size.width(), CSSUnitType::CSS_PX);
- lineContainer->setInlineStyleProperty(CSSPropertyHeight, lineBounds.size.height(), CSSUnitType::CSS_PX);
- lineContainer->setInlineStyleProperty(CSSPropertyTransform, makeString(
- "translate("_s,
- std::round(lineBounds.center.x() - (lineBounds.size.width() / 2)), "px, "_s,
- std::round(lineBounds.center.y() - (lineBounds.size.height() / 2)), "px) "_s,
- lineBounds.angleInRadians ? makeString("rotate("_s, lineBounds.angleInRadians, "rad) "_s) : emptyString()
- ));
-
+ auto lineBounds = fitElementToQuad(lineContainer.get(), lineQuad);
auto offsetAlongHorizontalAxis = [&](const FloatPoint& quadPoint1, const FloatPoint& quadPoint2) {
auto intervalLength = lineBounds.size.width();
auto mid = midPoint(quadPoint1, quadPoint2);
@@ -451,20 +489,28 @@
if (dataDetector.normalizedQuads.isEmpty())
continue;
+ auto firstQuad = dataDetector.normalizedQuads.first();
+ if (firstQuad.isEmpty())
+ continue;
+
// FIXME: We should come up with a way to coalesce the bounding quads into one or more rotated rects with the same angle of rotation.
- auto targetQuad = convertToContainerCoordinates(dataDetector.normalizedQuads.first());
- auto targetBounds = rotatedBoundingRectWithMinimumAngleOfRotation(targetQuad, 0.01);
- dataDetectorContainer->setInlineStyleProperty(CSSPropertyWidth, targetBounds.size.width(), CSSUnitType::CSS_PX);
- dataDetectorContainer->setInlineStyleProperty(CSSPropertyHeight, targetBounds.size.height(), CSSUnitType::CSS_PX);
- dataDetectorContainer->setInlineStyleProperty(CSSPropertyTransform, makeString(
- "translate("_s,
- std::round(targetBounds.center.x() - (targetBounds.size.width() / 2)), "px, "_s,
- std::round(targetBounds.center.y() - (targetBounds.size.height() / 2)), "px) "_s,
- targetBounds.angleInRadians ? makeString("rotate("_s, targetBounds.angleInRadians, "rad) "_s) : emptyString()
- ));
+ fitElementToQuad(dataDetectorContainer.get(), convertToContainerCoordinates(firstQuad));
}
#endif // ENABLE(DATA_DETECTION)
+ ASSERT(result.blocks.size() == elements.blocks.size());
+ for (size_t index = 0; index < result.blocks.size(); ++index) {
+ auto& block = result.blocks[index];
+ if (block.normalizedQuad.isEmpty())
+ continue;
+
+ auto blockContainer = elements.blocks[index];
+ auto bounds = fitElementToQuad(blockContainer.get(), convertToContainerCoordinates(block.normalizedQuad));
+ // FIXME: We'll need a smarter algorithm here that chooses the largest font size for the container without
+ // vertically overflowing the container.
+ blockContainer->setInlineStyleProperty(CSSPropertyFontSize, std::round(0.8 * bounds.size.height()), CSSUnitType::CSS_PX);
+ }
+
if (RefPtr frame = document->frame())
frame->eventHandler().scheduleCursorUpdate();
Modified: trunk/Source/WebCore/html/shadow/imageOverlay.css (285861 => 285862)
--- trunk/Source/WebCore/html/shadow/imageOverlay.css 2021-11-16 15:24:18 UTC (rev 285861)
+++ trunk/Source/WebCore/html/shadow/imageOverlay.css 2021-11-16 16:37:05 UTC (rev 285862)
@@ -30,21 +30,37 @@
color: transparent;
text-shadow: none;
text-align: center;
+ font-family: system-ui;
+}
+
+div.image-overlay-line {
white-space: nowrap;
line-height: 100%;
- font-family: system-ui;
font-size: 1024px; /* This large font size is chosen to minimize gaps when painting selection quads. */
}
-div.image-overlay-line, .image-overlay-text {
+div.image-overlay-line, div.image-overlay-block {
+ pointer-events: auto;
+}
+
+div.image-overlay-block {
+ background-color: rgba(255, 255, 255, 0.75);
+ border-radius: calc(clamp(2px, 0.1em, 12px));
+ box-shadow: rgba(100, 100, 100, 0.2) 3px 4px 8px 4px;
+ color: rgb(90, 90, 90);
+ font-weight: bold;
+ display: flex;
+ justify-content: center;
+ align-content: center;
+ flex-direction: column;
+ -webkit-backdrop-filter: blur(8px);
+}
+
+div.image-overlay-line, .image-overlay-text, div.image-overlay-block {
position: absolute;
overflow: hidden;
}
-div.image-overlay-line {
- pointer-events: auto;
-}
-
.image-overlay-text::selection {
color: transparent;
background-color: highlight;
