Diff
Modified: trunk/LayoutTests/ChangeLog (219318 => 219319)
--- trunk/LayoutTests/ChangeLog 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/LayoutTests/ChangeLog 2017-07-11 02:52:07 UTC (rev 219319)
@@ -1,3 +1,17 @@
+2017-07-10 John Wilander <wilan...@apple.com>
+
+ Resource Load Statistics: Prune statistics in orders of importance
+ https://bugs.webkit.org/show_bug.cgi?id=174215
+ <rdar://problem/33164403>
+
+ Reviewed by Chris Dumez.
+
+ * http/tests/loading/resourceLoadStatistics/prune-statistics-expected.txt: Added.
+ * http/tests/loading/resourceLoadStatistics/prune-statistics.html: Added.
+ * platform/wk2/TestExpectations:
+ Added http/tests/loading/resourceLoadStatistics/prune-statistics.html as
+ [ Pass ] since Resource Load Statistics is WK2-only.
+
2017-07-10 Devin Rousso <drou...@apple.com>
Web Inspector: Highlight matching CSS canvas clients when hovering contexts in the Resources tab
Added: trunk/LayoutTests/http/tests/loading/resourceLoadStatistics/prune-statistics-expected.txt (0 => 219319)
--- trunk/LayoutTests/http/tests/loading/resourceLoadStatistics/prune-statistics-expected.txt (rev 0)
+++ trunk/LayoutTests/http/tests/loading/resourceLoadStatistics/prune-statistics-expected.txt 2017-07-11 02:52:07 UTC (rev 219319)
@@ -0,0 +1,23 @@
+main frame - didStartProvisionalLoadForFrame
+main frame - didCommitLoadForFrame
+main frame - didReceiveTitle: Test for Resource Load Statistics Pruning
+main frame - didFinishDocumentLoadForFrame
+main frame - didHandleOnloadEventsForFrame
+main frame - didFinishLoadForFrame
+Tests that statistics are pruned in the right order.
+
+On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
+
+
+PASS Test iteration 1 passed.
+PASS Test iteration 2 passed.
+PASS Test iteration 3 passed.
+PASS Test iteration 4 passed.
+PASS Test iteration 5 passed.
+PASS Test iteration 6 passed.
+PASS Test iteration 7 passed.
+PASS Test iteration 8 passed.
+PASS successfullyParsed is true
+
+TEST COMPLETE
+
Added: trunk/LayoutTests/http/tests/loading/resourceLoadStatistics/prune-statistics.html (0 => 219319)
--- trunk/LayoutTests/http/tests/loading/resourceLoadStatistics/prune-statistics.html (rev 0)
+++ trunk/LayoutTests/http/tests/loading/resourceLoadStatistics/prune-statistics.html 2017-07-11 02:52:07 UTC (rev 219319)
@@ -0,0 +1,119 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <meta charset="UTF-8">
+ <title>Test for Resource Load Statistics Pruning</title>
+ <script src=""
+</head>
+<body>
+<script>
+ description("Tests that statistics are pruned in the right order.");
+ jsTestIsAsync = true;
+
+ const olderTimestamp = Math.round((new Date()).getTime() / 1000);
+ const newerTimestamp = olderTimestamp + 10;
+ const newestTimestamp = newerTimestamp + 10;
+
+ const urlsToBePruned = [
+ { url: "http://127.0.0.1:8000/temp", prevalent: false },
+ { url: "http://127.0.0.2:8000/temp", prevalent: false },
+ { url: "http://127.0.0.3:8000/temp", prevalent: true },
+ { url: "http://127.0.0.4:8000/temp", prevalent: true },
+ { url: "http://127.0.0.5:8000/temp", prevalent: false },
+ { url: "http://127.0.0.6:8000/temp", prevalent: false },
+ { url: "http://127.0.0.7:8000/temp", prevalent: true },
+ { url: "http://127.0.0.8:8000/temp", prevalent: true }
+ ];
+
+ function checkIfPrevalentAccordingToInitialExpectation(begin, end) {
+ var failed = false;
+ for (var i = begin; i < end; ++i) {
+ if (testRunner.isStatisticsPrevalentResource(urlsToBePruned[i].url) !== urlsToBePruned[i].prevalent) {
+ testFailed("checkIfPrevalentAccordingToInitialExpectation: Test iteration " + currentTest + " failed. " + urlsToBePruned[i].url + (urlsToBePruned[i].prevalent ? " wasn't " : " was ") + "prevalent");
+ failed = true;
+ }
+ }
+ if (failed)
+ finishJSTest();
+ }
+
+ function checkIfPrevalent(begin, end, expected) {
+ var failed = false;
+ for (var i = begin; i < end; ++i) {
+ if (testRunner.isStatisticsPrevalentResource(urlsToBePruned[i].url) !== expected) {
+ testFailed("checkIfPrevalent: Test iteration " + currentTest + " failed. " + urlsToBePruned[i].url + (expected ? " wasn't " : " was ") + "prevalent");
+ failed = true;
+ }
+ }
+ if (failed)
+ finishJSTest();
+ }
+
+ function initializeStatistics() {
+ testRunner.installStatisticsDidScanDataRecordsCallback(checkStatisticsAfterPruning);
+
+ // Non-prevalent without user interaction to be pruned first.
+ testRunner.setStatisticsLastSeen(urlsToBePruned[0].url, olderTimestamp);
+ testRunner.setStatisticsLastSeen(urlsToBePruned[1].url, newerTimestamp);
+
+ // Prevalent without user interaction to be pruned second.
+ testRunner.setStatisticsPrevalentResource(urlsToBePruned[2].url, true);
+ testRunner.setStatisticsLastSeen(urlsToBePruned[2].url, olderTimestamp);
+ testRunner.setStatisticsPrevalentResource(urlsToBePruned[3].url, true);
+ testRunner.setStatisticsLastSeen(urlsToBePruned[3].url, newerTimestamp);
+
+ // Non-prevalent with user interaction to be pruned third.
+ testRunner.setStatisticsHasHadUserInteraction(urlsToBePruned[4].url, true);
+ testRunner.setStatisticsLastSeen(urlsToBePruned[4].url, olderTimestamp);
+ testRunner.setStatisticsHasHadUserInteraction(urlsToBePruned[5].url, true);
+ testRunner.setStatisticsLastSeen(urlsToBePruned[5].url, newerTimestamp);
+
+ // Prevalent with user interaction to be pruned last.
+ testRunner.setStatisticsPrevalentResource(urlsToBePruned[6].url, true);
+ testRunner.setStatisticsHasHadUserInteraction(urlsToBePruned[6].url, true);
+ testRunner.setStatisticsLastSeen(urlsToBePruned[6].url, olderTimestamp);
+ testRunner.setStatisticsPrevalentResource(urlsToBePruned[7].url, true);
+ testRunner.setStatisticsHasHadUserInteraction(urlsToBePruned[7].url, true);
+ testRunner.setStatisticsLastSeen(urlsToBePruned[7].url, newerTimestamp);
+
+ checkIfPrevalentAccordingToInitialExpectation(0, urlsToBePruned.length);
+ }
+
+ var currentTest;
+ function checkStatisticsAfterPruning() {
+ // Pruned entries should not be prevalent.
+ checkIfPrevalent(0, currentTest, false);
+ // Non-pruned entries should keep their expected status.
+ checkIfPrevalentAccordingToInitialExpectation(currentTest, urlsToBePruned.length);
+ testPassed("Test iteration " + currentTest + " passed.");
+ if (currentTest < urlsToBePruned.length) {
+ ++currentTest;
+ runTest();
+ } else {
+ finishJSTest();
+ }
+ }
+
+ function runTest() {
+ initializeStatistics();
+
+ var fillerUrl = "http://127.0." + currentTest + ".1:8000/temp";
+ testRunner.setStatisticsPrevalentResource(fillerUrl, true);
+ testRunner.setStatisticsHasHadUserInteraction(fillerUrl, true);
+ testRunner.setStatisticsLastSeen(fillerUrl, newestTimestamp);
+ testRunner.statisticsProcessStatisticsAndDataRecords();
+ }
+
+ if (window.testRunner) {
+ testRunner.setStatisticsShouldClassifyResourcesBeforeDataRecordsRemoval(false);
+ testRunner.setStatisticsMinimumTimeBetweenDataRecordsRemoval(0);
+ testRunner.setStatisticsNotifyPagesWhenDataRecordsWereScanned(true);
+ testRunner.setStatisticsMaxStatisticsEntries(urlsToBePruned.length);
+ testRunner.setStatisticsPruneEntriesDownTo(urlsToBePruned.length);
+
+ currentTest = 1;
+ runTest();
+ }
+</script>
+</body>
+</html>
\ No newline at end of file
Modified: trunk/LayoutTests/platform/wk2/TestExpectations (219318 => 219319)
--- trunk/LayoutTests/platform/wk2/TestExpectations 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/LayoutTests/platform/wk2/TestExpectations 2017-07-11 02:52:07 UTC (rev 219319)
@@ -706,6 +706,7 @@
http/tests/loading/resourceLoadStatistics/clear-in-memory-and-persistent-store.html [ Pass ]
webkit.org/b/172452 http/tests/loading/resourceLoadStatistics/grandfathering.html [ Pass Failure Timeout ]
webkit.org/b/173499 http/tests/loading/resourceLoadStatistics/telemetry-generation.html [ Pass Failure ]
+http/tests/loading/resourceLoadStatistics/prune-statistics.html [ Pass ]
### END OF (5) Progressions, expected successes that are expected failures in WebKit1.
########################################
Modified: trunk/Source/WebCore/ChangeLog (219318 => 219319)
--- trunk/Source/WebCore/ChangeLog 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Source/WebCore/ChangeLog 2017-07-11 02:52:07 UTC (rev 219319)
@@ -1,3 +1,28 @@
+2017-07-10 John Wilander <wilan...@apple.com>
+
+ Resource Load Statistics: Prune statistics in orders of importance
+ https://bugs.webkit.org/show_bug.cgi?id=174215
+ <rdar://problem/33164403>
+
+ Reviewed by Chris Dumez.
+
+ Test: http/tests/loading/resourceLoadStatistics/prune-statistics.html
+
+ * loader/ResourceLoadObserver.cpp:
+ (WebCore::reduceTimeResolution):
+ (WebCore::ResourceLoadObserver::logFrameNavigation):
+ (WebCore::ResourceLoadObserver::logSubresourceLoading):
+ (WebCore::ResourceLoadObserver::logWebSocketLoading):
+ (WebCore::ResourceLoadObserver::logUserInteractionWithReducedTimeResolution):
+ Now all set the new statistics field lastSeen.
+ * loader/ResourceLoadStatistics.cpp:
+ (WebCore::ResourceLoadStatistics::encode):
+ (WebCore::ResourceLoadStatistics::decode):
+ (WebCore::ResourceLoadStatistics::toString):
+ (WebCore::ResourceLoadStatistics::merge):
+ Handling of the new statistics field lastSeen.
+ * loader/ResourceLoadStatistics.h:
+
2017-07-10 Devin Rousso <drou...@apple.com>
Web Inspector: Highlight matching CSS canvas clients when hovering contexts in the Resources tab
Modified: trunk/Source/WebCore/loader/ResourceLoadObserver.cpp (219318 => 219319)
--- trunk/Source/WebCore/loader/ResourceLoadObserver.cpp 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Source/WebCore/loader/ResourceLoadObserver.cpp 2017-07-11 02:52:07 UTC (rev 219319)
@@ -92,6 +92,11 @@
return Settings::resourceLoadStatisticsEnabled() && !page->usesEphemeralSession() && m_notificationCallback;
}
+static WallTime reduceToHourlyTimeResolution(WallTime time)
+{
+ return WallTime::fromRawSeconds(std::floor(time.secondsSinceEpoch() / timestampResolution) * timestampResolution.seconds());
+}
+
void ResourceLoadObserver::logFrameNavigation(const Frame& frame, const Frame& topFrame, const ResourceRequest& newRequest)
{
ASSERT(frame.document());
@@ -125,6 +130,7 @@
return;
auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
+ targetStatistics.lastSeen = reduceToHourlyTimeResolution(WallTime::now());
auto subframeUnderTopFrameOriginsResult = targetStatistics.subframeUnderTopFrameOrigins.add(mainFramePrimaryDomain);
if (subframeUnderTopFrameOriginsResult.isNewEntry)
scheduleNotificationIfNeeded();
@@ -158,6 +164,7 @@
bool shouldCallNotificationCallback = false;
{
auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
+ targetStatistics.lastSeen = reduceToHourlyTimeResolution(WallTime::now());
if (targetStatistics.subresourceUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
shouldCallNotificationCallback = true;
}
@@ -197,15 +204,11 @@
return;
auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
+ targetStatistics.lastSeen = reduceToHourlyTimeResolution(WallTime::now());
if (targetStatistics.subresourceUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
scheduleNotificationIfNeeded();
}
-static WallTime reduceTimeResolution(WallTime time)
-{
- return WallTime::fromRawSeconds(std::floor(time.secondsSinceEpoch() / timestampResolution) * timestampResolution.seconds());
-}
-
void ResourceLoadObserver::logUserInteractionWithReducedTimeResolution(const Document& document)
{
ASSERT(document.page());
@@ -218,11 +221,12 @@
return;
auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain(url));
- auto newTime = reduceTimeResolution(WallTime::now());
+ auto newTime = reduceToHourlyTimeResolution(WallTime::now());
if (newTime == statistics.mostRecentUserInteractionTime)
return;
statistics.hadUserInteraction = true;
+ statistics.lastSeen = newTime;
statistics.mostRecentUserInteractionTime = newTime;
scheduleNotificationIfNeeded();
Modified: trunk/Source/WebCore/loader/ResourceLoadStatistics.cpp (219318 => 219319)
--- trunk/Source/WebCore/loader/ResourceLoadStatistics.cpp 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Source/WebCore/loader/ResourceLoadStatistics.cpp 2017-07-11 02:52:07 UTC (rev 219319)
@@ -50,6 +50,8 @@
{
encoder.encodeString("PrevalentResourceOrigin", highLevelDomain);
+ encoder.encodeDouble("lastSeen", lastSeen.secondsSinceEpoch().value());
+
// User interaction
encoder.encodeBool("hadUserInteraction", hadUserInteraction);
encoder.encodeDouble("mostRecentUserInteraction", mostRecentUserInteractionTime.secondsSinceEpoch().value());
@@ -114,6 +116,11 @@
if (!decoder.decodeBool("grandfathered", grandfathered))
return false;
+ double lastSeenTimeAsDouble;
+ if (!decoder.decodeDouble("lastSeen", lastSeenTimeAsDouble))
+ return false;
+ lastSeen = WallTime::fromRawSeconds(lastSeenTimeAsDouble);
+
return true;
}
@@ -147,6 +154,10 @@
{
StringBuilder builder;
+ builder.appendLiteral("lastSeen");
+ builder.appendNumber(lastSeen.secondsSinceEpoch().value());
+ builder.append('\n');
+
// User interaction
appendBoolean(builder, "hadUserInteraction", hadUserInteraction);
builder.append('\n');
@@ -189,6 +200,9 @@
{
ASSERT(other.highLevelDomain == highLevelDomain);
+ if (lastSeen < other.lastSeen)
+ lastSeen = other.lastSeen;
+
if (!other.hadUserInteraction) {
// If user interaction has been reset do so here too.
// Else, do nothing.
Modified: trunk/Source/WebCore/loader/ResourceLoadStatistics.h (219318 => 219319)
--- trunk/Source/WebCore/loader/ResourceLoadStatistics.h 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Source/WebCore/loader/ResourceLoadStatistics.h 2017-07-11 02:52:07 UTC (rev 219319)
@@ -61,6 +61,8 @@
String highLevelDomain;
+ WallTime lastSeen;
+
// User interaction
bool hadUserInteraction { false };
// Timestamp. Default value is negative, 0 means it was reset.
Modified: trunk/Source/WebKit2/ChangeLog (219318 => 219319)
--- trunk/Source/WebKit2/ChangeLog 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Source/WebKit2/ChangeLog 2017-07-11 02:52:07 UTC (rev 219319)
@@ -1,3 +1,49 @@
+2017-07-10 John Wilander <wilan...@apple.com>
+
+ Resource Load Statistics: Prune statistics in orders of importance
+ https://bugs.webkit.org/show_bug.cgi?id=174215
+ <rdar://problem/33164403>
+
+ Reviewed by Chris Dumez.
+
+ New functionality. Prunes statistics in this order:
+ 1. Non-prevalent resources without user interaction.
+ 2. Prevalent resources without user interaction.
+ 3. Non-prevalent resources with user interaction.
+ 4. Prevalent resources with user interaction.
+
+ * Shared/WebCoreArgumentCoders.cpp:
+ (IPC::ArgumentCoder<ResourceLoadStatistics>::encode):
+ (IPC::ArgumentCoder<ResourceLoadStatistics>::decode):
+ Added timestamp field lastSeen.
+ * UIProcess/API/Cocoa/WKWebsiteDataStore.mm:
+ (-[WKWebsiteDataStore _resourceLoadStatisticsSetLastSeen:forHost:]):
+ (-[WKWebsiteDataStore _resourceLoadStatisticsSetMaxStatisticsEntries:]):
+ (-[WKWebsiteDataStore _resourceLoadStatisticsSetPruneEntriesDownTo:]):
+ (-[WKWebsiteDataStore _resourceLoadStatisticsResetToConsistentState]):
+ Test infrastructure.
+ * UIProcess/API/Cocoa/WKWebsiteDataStorePrivate.h:
+ * UIProcess/Storage/ResourceLoadStatisticsStore.cpp:
+ (WebKit::ResourceLoadStatisticsStore::setMaxStatisticsEntries):
+ Test infrastructure.
+ (WebKit::ResourceLoadStatisticsStore::setPruneEntriesDownTo):
+ Test infrastructure.
+ (WebKit::sortAndPrune):
+ Convenience function.
+ (WebKit::ResourceLoadStatisticsStore::pruneStatisticsIfNeeded):
+ The new pruning function.
+ * UIProcess/Storage/ResourceLoadStatisticsStore.h:
+ * UIProcess/WebResourceLoadStatisticsStore.cpp:
+ (WebKit::WebResourceLoadStatisticsStore::processStatisticsAndDataRecords):
+ Now calls ResourceLoadStatisticsStore::pruneStatisticsIfNeeded().
+ (WebKit::WebResourceLoadStatisticsStore::setLastSeen):
+ Test infrastructure.
+ (WebKit::WebResourceLoadStatisticsStore::setMaxStatisticsEntries):
+ Test infrastructure.
+ (WebKit::WebResourceLoadStatisticsStore::setPruneEntriesDownTo):
+ Test infrastructure.
+ * UIProcess/WebResourceLoadStatisticsStore.h:
+
2017-07-10 Dean Jackson <d...@apple.com>
const() experimental feature should always be on by default
Modified: trunk/Source/WebKit2/Shared/WebCoreArgumentCoders.cpp (219318 => 219319)
--- trunk/Source/WebKit2/Shared/WebCoreArgumentCoders.cpp 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Source/WebKit2/Shared/WebCoreArgumentCoders.cpp 2017-07-11 02:52:07 UTC (rev 219319)
@@ -2242,6 +2242,8 @@
{
encoder << statistics.highLevelDomain;
+ encoder << statistics.lastSeen.secondsSinceEpoch().value();
+
// User interaction
encoder << statistics.hadUserInteraction;
encoder << statistics.mostRecentUserInteractionTime.secondsSinceEpoch().value();
@@ -2264,6 +2266,11 @@
if (!decoder.decode(statistics.highLevelDomain))
return false;
+ double lastSeenTimeAsDouble;
+ if (!decoder.decode(lastSeenTimeAsDouble))
+ return false;
+ statistics.lastSeen = WallTime::fromRawSeconds(lastSeenTimeAsDouble);
+
// User interaction
if (!decoder.decode(statistics.hadUserInteraction))
return false;
Modified: trunk/Source/WebKit2/UIProcess/API/Cocoa/WKWebsiteDataStore.mm (219318 => 219319)
--- trunk/Source/WebKit2/UIProcess/API/Cocoa/WKWebsiteDataStore.mm 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Source/WebKit2/UIProcess/API/Cocoa/WKWebsiteDataStore.mm 2017-07-11 02:52:07 UTC (rev 219319)
@@ -207,6 +207,15 @@
_websiteDataStore->websiteDataStore().setResourceLoadStatisticsEnabled(enabled);
}
+- (void)_resourceLoadStatisticsSetLastSeen:(double)seconds forHost:(NSString *)host
+{
+ auto* store = _websiteDataStore->websiteDataStore().resourceLoadStatistics();
+ if (!store)
+ return;
+
+ store->setLastSeen(URL(URL(), host), Seconds { seconds });
+}
+
- (void)_resourceLoadStatisticsSetIsPrevalentResource:(BOOL)value forHost:(NSString *)host
{
auto* store = _websiteDataStore->websiteDataStore().resourceLoadStatistics();
@@ -345,6 +354,24 @@
store->setGrandfatheringTime(Seconds {seconds });
}
+- (void)_resourceLoadStatisticsSetMaxStatisticsEntries:(size_t)entries
+{
+ auto* store = _websiteDataStore->websiteDataStore().resourceLoadStatistics();
+ if (!store)
+ return;
+
+ store->setMaxStatisticsEntries(entries);
+}
+
+- (void)_resourceLoadStatisticsSetPruneEntriesDownTo:(size_t)entries
+{
+ auto* store = _websiteDataStore->websiteDataStore().resourceLoadStatistics();
+ if (!store)
+ return;
+
+ store->setPruneEntriesDownTo(entries);
+}
+
- (void)_resourceLoadStatisticsProcessStatisticsAndDataRecords
{
auto* store = _websiteDataStore->websiteDataStore().resourceLoadStatistics();
@@ -441,6 +468,8 @@
return;
// FIXME: These needs to match the default data member values in ResourceLoadStatistics, which is fragile.
+ store->setMaxStatisticsEntries(1000);
+ store->setPruneEntriesDownTo(800);
store->setTimeToLiveUserInteraction(std::nullopt);
store->setTimeToLiveCookiePartitionFree(24_h);
store->setMinimumTimeBetweenDataRecordsRemoval(1_h);
Modified: trunk/Source/WebKit2/UIProcess/API/Cocoa/WKWebsiteDataStorePrivate.h (219318 => 219319)
--- trunk/Source/WebKit2/UIProcess/API/Cocoa/WKWebsiteDataStorePrivate.h 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Source/WebKit2/UIProcess/API/Cocoa/WKWebsiteDataStorePrivate.h 2017-07-11 02:52:07 UTC (rev 219319)
@@ -44,6 +44,7 @@
@property (nonatomic, setter=_setResourceLoadStatisticsEnabled:) BOOL _resourceLoadStatisticsEnabled WK_API_AVAILABLE(macosx(10.12), ios(10.0));
// ResourceLoadStatistics SPI for testing.
+- (void)_resourceLoadStatisticsSetLastSeen:(double)seconds forHost:(NSString *)host WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
- (void)_resourceLoadStatisticsSetIsPrevalentResource:(BOOL)value forHost:(NSString *)host WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
- (void)_resourceLoadStatisticsIsPrevalentResource:(NSString *)host completionHandler:(void (^)(BOOL))completionHandler WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
- (void)_resourceLoadStatisticsSetHadUserInteraction:(BOOL)value forHost:(NSString *)host WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
@@ -57,6 +58,8 @@
- (void)_resourceLoadStatisticsSetTimeToLiveCookiePartitionFree:(double)seconds WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
- (void)_resourceLoadStatisticsSetMinimumTimeBetweenDataRecordsRemoval:(double)seconds WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
- (void)_resourceLoadStatisticsSetGrandfatheringTime:(double)seconds WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
+- (void)_resourceLoadStatisticsSetMaxStatisticsEntries:(size_t)entries WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
+- (void)_resourceLoadStatisticsSetPruneEntriesDownTo:(size_t)entries WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
- (void)_resourceLoadStatisticsProcessStatisticsAndDataRecords WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
- (void)_resourceLoadStatisticsUpdateCookiePartitioning WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
- (void)_resourceLoadStatisticsSetShouldPartitionCookies:(BOOL)value forHost:(NSString *)host WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
Modified: trunk/Source/WebKit2/UIProcess/WebResourceLoadStatisticsStore.cpp (219318 => 219319)
--- trunk/Source/WebKit2/UIProcess/WebResourceLoadStatisticsStore.cpp 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Source/WebKit2/UIProcess/WebResourceLoadStatisticsStore.cpp 2017-07-11 02:52:07 UTC (rev 219319)
@@ -50,7 +50,8 @@
constexpr Seconds minimumStatisticsFileWriteInterval { 5_min };
constexpr unsigned operatingDatesWindow { 30 };
-constexpr unsigned statisticsModelVersion { 6 };
+constexpr unsigned statisticsModelVersion { 7 };
+constexpr unsigned maxImportance { 3 };
template<typename T> static inline String primaryDomain(const T& value)
{
@@ -146,6 +147,8 @@
}
removeDataRecords();
+ pruneStatisticsIfNeeded();
+
if (m_shouldNotifyPagesWhenDataRecordsWereScanned) {
RunLoop::main().dispatch([] {
WebProcessProxy::notifyPageStatisticsAndDataRecordsProcessed();
@@ -500,6 +503,17 @@
});
}
+void WebResourceLoadStatisticsStore::setLastSeen(const URL& url, Seconds seconds)
+{
+ if (url.isBlankURL() || url.isEmpty())
+ return;
+
+ m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), url = "" seconds] {
+ auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain(url));
+ statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
+ });
+}
+
void WebResourceLoadStatisticsStore::setPrevalentResource(const URL& url)
{
if (url.isBlankURL() || url.isEmpty())
@@ -833,7 +847,7 @@
for (auto& domain : domainsToAdd)
ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = true;
}
-
+
void WebResourceLoadStatisticsStore::processStatistics(const WTF::Function<void (const ResourceLoadStatistics&)>& processFunction) const
{
ASSERT(!RunLoop::isMain());
@@ -904,4 +918,62 @@
return false;
}
+void WebResourceLoadStatisticsStore::setMaxStatisticsEntries(size_t maximumEntryCount)
+{
+ m_maxStatisticsEntries = maximumEntryCount;
+}
+
+void WebResourceLoadStatisticsStore::setPruneEntriesDownTo(size_t pruneTargetCount)
+{
+ m_pruneEntriesDownTo = pruneTargetCount;
+}
+
+struct StatisticsLastSeen {
+ String topPrivatelyOwnedDomain;
+ WallTime lastSeen;
+};
+
+static void pruneResources(HashMap<String, WebCore::ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
+{
+ if (statisticsToPrune.size() > numberOfEntriesToPrune) {
+ std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
+ return a.lastSeen < b.lastSeen;
+ });
+ }
+
+ for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i)
+ statisticsMap.remove(statisticsToPrune[i].topPrivatelyOwnedDomain);
+}
+
+static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
+{
+ unsigned importance = maxImportance;
+ if (!resourceStatistic.isPrevalentResource)
+ importance -= 1;
+ if (!resourceStatistic.hadUserInteraction)
+ importance -= 2;
+ return importance;
+}
+
+void WebResourceLoadStatisticsStore::pruneStatisticsIfNeeded()
+{
+ ASSERT(!RunLoop::isMain());
+ if (m_resourceStatisticsMap.size() <= m_maxStatisticsEntries)
+ return;
+
+ ASSERT(m_pruneEntriesDownTo <= m_maxStatisticsEntries);
+
+ size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_pruneEntriesDownTo;
+ ASSERT(numberOfEntriesLeftToPrune);
+
+ Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
+ for (auto& resourceStatistic : m_resourceStatisticsMap.values())
+ resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.highLevelDomain, resourceStatistic.lastSeen });
+
+ for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
+ pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
+
+ ASSERT(!numberOfEntriesLeftToPrune);
+}
+
} // namespace WebKit
Modified: trunk/Source/WebKit2/UIProcess/WebResourceLoadStatisticsStore.h (219318 => 219319)
--- trunk/Source/WebKit2/UIProcess/WebResourceLoadStatisticsStore.h 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Source/WebKit2/UIProcess/WebResourceLoadStatisticsStore.h 2017-07-11 02:52:07 UTC (rev 219319)
@@ -79,6 +79,7 @@
void logUserInteraction(const WebCore::URL&);
void clearUserInteraction(const WebCore::URL&);
void hasHadUserInteraction(const WebCore::URL&, WTF::Function<void (bool)>&&);
+ void setLastSeen(const WebCore::URL&, Seconds);
void setPrevalentResource(const WebCore::URL&);
void isPrevalentResource(const WebCore::URL&, WTF::Function<void (bool)>&&);
void clearPrevalentResource(const WebCore::URL&);
@@ -100,9 +101,12 @@
void setTimeToLiveCookiePartitionFree(Seconds);
void setMinimumTimeBetweenDataRecordsRemoval(Seconds);
void setGrandfatheringTime(Seconds);
-
+ void setMaxStatisticsEntries(size_t);
+ void setPruneEntriesDownTo(size_t);
+
void processStatistics(const WTF::Function<void (const WebCore::ResourceLoadStatistics&)>&) const;
-
+ void pruneStatisticsIfNeeded();
+
private:
WebResourceLoadStatisticsStore(const String&, UpdateCookiePartitioningForDomainsHandler&&);
@@ -172,6 +176,8 @@
std::optional<Seconds> m_timeToLiveUserInteraction;
Seconds m_timeToLiveCookiePartitionFree { 24_h };
Seconds m_grandfatheringTime { 1_h };
+ size_t m_maxStatisticsEntries { 1000 };
+ size_t m_pruneEntriesDownTo { 800 };
bool m_dataRecordsBeingRemoved { false };
bool m_didScheduleWrite { false };
bool m_shouldNotifyPagesWhenDataRecordsWereScanned { false };
Modified: trunk/Tools/ChangeLog (219318 => 219319)
--- trunk/Tools/ChangeLog 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Tools/ChangeLog 2017-07-11 02:52:07 UTC (rev 219319)
@@ -1,3 +1,34 @@
+2017-07-10 John Wilander <wilan...@apple.com>
+
+ Resource Load Statistics: Prune statistics in orders of importance
+ https://bugs.webkit.org/show_bug.cgi?id=174215
+ <rdar://problem/33164403>
+
+ Reviewed by Chris Dumez.
+
+ Nest infrastructure. Adds these functions:
+ 1. testRunner.setStatisticsLastSeen()
+ 2. setStatisticsMaxStatisticsEntries()
+ 3. setStatisticsPruneEntriesDownTo()
+
+ * WebKitTestRunner/InjectedBundle/Bindings/TestRunner.idl:
+ * WebKitTestRunner/InjectedBundle/TestRunner.cpp:
+ (WTR::TestRunner::setStatisticsLastSeen):
+ (WTR::TestRunner::setStatisticsMaxStatisticsEntries):
+ (WTR::TestRunner::setStatisticsPruneEntriesDownTo):
+ * WebKitTestRunner/InjectedBundle/TestRunner.h:
+ * WebKitTestRunner/TestController.cpp:
+ (WTR::TestController::setStatisticsLastSeen):
+ (WTR::TestController::setMaxStatisticsEntries):
+ (WTR::TestController::setPruneEntriesDownTo):
+ * WebKitTestRunner/TestController.h:
+ * WebKitTestRunner/TestInvocation.cpp:
+ (WTR::TestInvocation::didReceiveSynchronousMessageFromInjectedBundle):
+ * WebKitTestRunner/cocoa/TestControllerCocoa.mm:
+ (WTR::TestController::setStatisticsLastSeen):
+ (WTR::TestController::setStatisticsMaxStatisticsEntries):
+ (WTR::TestController::setStatisticsPruneEntriesDownTo):
+
2017-07-03 Brian Burg <bb...@apple.com>
Web Replay: remove some unused code
Modified: trunk/Tools/WebKitTestRunner/InjectedBundle/Bindings/TestRunner.idl (219318 => 219319)
--- trunk/Tools/WebKitTestRunner/InjectedBundle/Bindings/TestRunner.idl 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Tools/WebKitTestRunner/InjectedBundle/Bindings/TestRunner.idl 2017-07-11 02:52:07 UTC (rev 219319)
@@ -254,6 +254,7 @@
void installStatisticsDidModifyDataRecordsCallback(object callback);
void installStatisticsDidScanDataRecordsCallback(object callback);
void installStatisticsDidRunTelemetryCallback(object callback);
+ void setStatisticsLastSeen(DOMString hostName, double seconds);
void setStatisticsPrevalentResource(DOMString hostName, boolean value);
boolean isStatisticsPrevalentResource(DOMString hostName);
void setStatisticsHasHadUserInteraction(DOMString hostName, boolean value);
@@ -274,6 +275,8 @@
void setStatisticsNotifyPagesWhenTelemetryWasCaptured(boolean value);
void setStatisticsMinimumTimeBetweenDataRecordsRemoval(double seconds);
void setStatisticsGrandfatheringTime(double seconds);
+ void setStatisticsMaxStatisticsEntries(unsigned long entries);
+ void setStatisticsPruneEntriesDownTo(unsigned long entries);
void statisticsClearInMemoryAndPersistentStore();
void statisticsClearInMemoryAndPersistentStoreModifiedSinceHours(unsigned long hours);
void statisticsResetToConsistentState();
Modified: trunk/Tools/WebKitTestRunner/InjectedBundle/TestRunner.cpp (219318 => 219319)
--- trunk/Tools/WebKitTestRunner/InjectedBundle/TestRunner.cpp 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Tools/WebKitTestRunner/InjectedBundle/TestRunner.cpp 2017-07-11 02:52:07 UTC (rev 219319)
@@ -1175,6 +1175,31 @@
callTestRunnerCallback(DidRemoveSwipeSnapshotCallbackID);
}
+void TestRunner::setStatisticsLastSeen(JSStringRef hostName, double seconds)
+{
+ Vector<WKRetainPtr<WKStringRef>> keys;
+ Vector<WKRetainPtr<WKTypeRef>> values;
+
+ keys.append({ AdoptWK, WKStringCreateWithUTF8CString("HostName") });
+ values.append({ AdoptWK, WKStringCreateWithJSString(hostName) });
+
+ keys.append({ AdoptWK, WKStringCreateWithUTF8CString("Value") });
+ values.append({ AdoptWK, WKDoubleCreate(seconds) });
+
+ Vector<WKStringRef> rawKeys(keys.size());
+ Vector<WKTypeRef> rawValues(values.size());
+
+ for (size_t i = 0; i < keys.size(); ++i) {
+ rawKeys[i] = keys[i].get();
+ rawValues[i] = values[i].get();
+ }
+
+ WKRetainPtr<WKStringRef> messageName(AdoptWK, WKStringCreateWithUTF8CString("SetStatisticsLastSeen"));
+ WKRetainPtr<WKDictionaryRef> messageBody(AdoptWK, WKDictionaryCreate(rawKeys.data(), rawValues.data(), rawKeys.size()));
+
+ WKBundlePostSynchronousMessage(InjectedBundle::singleton().bundle(), messageName.get(), messageBody.get(), nullptr);
+}
+
void TestRunner::setStatisticsPrevalentResource(JSStringRef hostName, bool value)
{
Vector<WKRetainPtr<WKStringRef>> keys;
@@ -1491,6 +1516,20 @@
WKBundlePostSynchronousMessage(InjectedBundle::singleton().bundle(), messageName.get(), messageBody.get(), nullptr);
}
+void TestRunner::setStatisticsMaxStatisticsEntries(unsigned entries)
+{
+ WKRetainPtr<WKStringRef> messageName(AdoptWK, WKStringCreateWithUTF8CString("SetMaxStatisticsEntries"));
+ WKRetainPtr<WKTypeRef> messageBody(AdoptWK, WKUInt64Create(entries));
+ WKBundlePostSynchronousMessage(InjectedBundle::singleton().bundle(), messageName.get(), messageBody.get(), nullptr);
+}
+
+void TestRunner::setStatisticsPruneEntriesDownTo(unsigned entries)
+{
+ WKRetainPtr<WKStringRef> messageName(AdoptWK, WKStringCreateWithUTF8CString("SetPruneEntriesDownTo"));
+ WKRetainPtr<WKTypeRef> messageBody(AdoptWK, WKUInt64Create(entries));
+ WKBundlePostSynchronousMessage(InjectedBundle::singleton().bundle(), messageName.get(), messageBody.get(), nullptr);
+}
+
void TestRunner::statisticsClearInMemoryAndPersistentStore()
{
WKRetainPtr<WKStringRef> messageName(AdoptWK, WKStringCreateWithUTF8CString("StatisticsClearInMemoryAndPersistentStore"));
Modified: trunk/Tools/WebKitTestRunner/InjectedBundle/TestRunner.h (219318 => 219319)
--- trunk/Tools/WebKitTestRunner/InjectedBundle/TestRunner.h 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Tools/WebKitTestRunner/InjectedBundle/TestRunner.h 2017-07-11 02:52:07 UTC (rev 219319)
@@ -357,6 +357,7 @@
void statisticsUpdateCookiePartitioning();
void statisticsSetShouldPartitionCookiesForHost(JSStringRef hostName, bool value);
void statisticsSubmitTelemetry();
+ void setStatisticsLastSeen(JSStringRef hostName, double seconds);
void setStatisticsPrevalentResource(JSStringRef hostName, bool value);
bool isStatisticsPrevalentResource(JSStringRef hostName);
void setStatisticsHasHadUserInteraction(JSStringRef hostName, bool value);
@@ -373,6 +374,8 @@
void setStatisticsNotifyPagesWhenTelemetryWasCaptured(bool value);
void setStatisticsMinimumTimeBetweenDataRecordsRemoval(double);
void setStatisticsGrandfatheringTime(double seconds);
+ void setStatisticsMaxStatisticsEntries(unsigned);
+ void setStatisticsPruneEntriesDownTo(unsigned);
void statisticsClearInMemoryAndPersistentStore();
void statisticsClearInMemoryAndPersistentStoreModifiedSinceHours(unsigned hours);
void statisticsResetToConsistentState();
Modified: trunk/Tools/WebKitTestRunner/TestController.cpp (219318 => 219319)
--- trunk/Tools/WebKitTestRunner/TestController.cpp 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Tools/WebKitTestRunner/TestController.cpp 2017-07-11 02:52:07 UTC (rev 219319)
@@ -2249,6 +2249,10 @@
#if !PLATFORM(COCOA) || !WK_API_ENABLED
+void TestController::setStatisticsLastSeen(WKStringRef, double)
+{
+}
+
void TestController::setStatisticsPrevalentResource(WKStringRef, bool)
{
}
@@ -2332,6 +2336,14 @@
{
}
+void TestController::setStatisticsMaxStatisticsEntries(unsigned)
+{
+}
+
+void TestController::setStatisticsPruneEntriesDownTo(unsigned)
+{
+}
+
void TestController::statisticsClearInMemoryAndPersistentStore()
{
}
Modified: trunk/Tools/WebKitTestRunner/TestController.h (219318 => 219319)
--- trunk/Tools/WebKitTestRunner/TestController.h 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Tools/WebKitTestRunner/TestController.h 2017-07-11 02:52:07 UTC (rev 219319)
@@ -150,6 +150,7 @@
void setShouldDownloadUndisplayableMIMETypes(bool value) { m_shouldDownloadUndisplayableMIMETypes = value; }
+ void setStatisticsLastSeen(WKStringRef hostName, double seconds);
void setStatisticsPrevalentResource(WKStringRef hostName, bool value);
bool isStatisticsPrevalentResource(WKStringRef hostName);
void setStatisticsHasHadUserInteraction(WKStringRef hostName, bool value);
@@ -170,6 +171,8 @@
void setStatisticsNotifyPagesWhenTelemetryWasCaptured(bool value);
void setStatisticsMinimumTimeBetweenDataRecordsRemoval(double);
void setStatisticsGrandfatheringTime(double seconds);
+ void setStatisticsMaxStatisticsEntries(unsigned);
+ void setStatisticsPruneEntriesDownTo(unsigned);
void statisticsClearInMemoryAndPersistentStore();
void statisticsClearInMemoryAndPersistentStoreModifiedSinceHours(unsigned);
void statisticsResetToConsistentState();
Modified: trunk/Tools/WebKitTestRunner/TestInvocation.cpp (219318 => 219319)
--- trunk/Tools/WebKitTestRunner/TestInvocation.cpp 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Tools/WebKitTestRunner/TestInvocation.cpp 2017-07-11 02:52:07 UTC (rev 219319)
@@ -912,6 +912,21 @@
return result;
}
+ if (WKStringIsEqualToUTF8CString(messageName, "SetStatisticsLastSeen")) {
+ ASSERT(WKGetTypeID(messageBody) == WKDictionaryGetTypeID());
+
+ WKDictionaryRef messageBodyDictionary = static_cast<WKDictionaryRef>(messageBody);
+ WKRetainPtr<WKStringRef> hostNameKey(AdoptWK, WKStringCreateWithUTF8CString("HostName"));
+ WKRetainPtr<WKStringRef> valueKey(AdoptWK, WKStringCreateWithUTF8CString("Value"));
+
+ WKStringRef hostName = static_cast<WKStringRef>(WKDictionaryGetItemForKey(messageBodyDictionary, hostNameKey.get()));
+ WKDoubleRef value = static_cast<WKDoubleRef>(WKDictionaryGetItemForKey(messageBodyDictionary, valueKey.get()));
+
+ TestController::singleton().setStatisticsLastSeen(hostName, WKDoubleGetValue(value));
+
+ return nullptr;
+ }
+
if (WKStringIsEqualToUTF8CString(messageName, "SetStatisticsPrevalentResource")) {
ASSERT(WKGetTypeID(messageBody) == WKDictionaryGetTypeID());
@@ -1101,6 +1116,20 @@
return nullptr;
}
+ if (WKStringIsEqualToUTF8CString(messageName, "SetMaxStatisticsEntries")) {
+ ASSERT(WKGetTypeID(messageBody) == WKUInt64GetTypeID());
+ WKUInt64Ref entries = static_cast<WKUInt64Ref>(messageBody);
+ TestController::singleton().setStatisticsMaxStatisticsEntries(WKUInt64GetValue(entries));
+ return nullptr;
+ }
+
+ if (WKStringIsEqualToUTF8CString(messageName, "SetPruneEntriesDownTo")) {
+ ASSERT(WKGetTypeID(messageBody) == WKUInt64GetTypeID());
+ WKUInt64Ref entries = static_cast<WKUInt64Ref>(messageBody);
+ TestController::singleton().setStatisticsPruneEntriesDownTo(WKUInt64GetValue(entries));
+ return nullptr;
+ }
+
if (WKStringIsEqualToUTF8CString(messageName, "StatisticsClearInMemoryAndPersistentStore")) {
TestController::singleton().statisticsClearInMemoryAndPersistentStore();
return nullptr;
Modified: trunk/Tools/WebKitTestRunner/cocoa/TestControllerCocoa.mm (219318 => 219319)
--- trunk/Tools/WebKitTestRunner/cocoa/TestControllerCocoa.mm 2017-07-11 02:39:16 UTC (rev 219318)
+++ trunk/Tools/WebKitTestRunner/cocoa/TestControllerCocoa.mm 2017-07-11 02:52:07 UTC (rev 219319)
@@ -222,6 +222,11 @@
}
#if WK_API_ENABLED
+void TestController::setStatisticsLastSeen(WKStringRef hostName, double seconds)
+{
+ [globalWebViewConfiguration.websiteDataStore _resourceLoadStatisticsSetLastSeen:seconds forHost:toNSString(hostName)];
+}
+
void TestController::setStatisticsPrevalentResource(WKStringRef hostName, bool value)
{
[globalWebViewConfiguration.websiteDataStore _resourceLoadStatisticsSetIsPrevalentResource:value forHost:toNSString(hostName)];
@@ -346,6 +351,16 @@
[globalWebViewConfiguration.websiteDataStore _resourceLoadStatisticsSetGrandfatheringTime:seconds];
}
+void TestController::setStatisticsMaxStatisticsEntries(unsigned entries)
+{
+ [globalWebViewConfiguration.websiteDataStore _resourceLoadStatisticsSetMaxStatisticsEntries:entries];
+}
+
+void TestController::setStatisticsPruneEntriesDownTo(unsigned entries)
+{
+ [globalWebViewConfiguration.websiteDataStore _resourceLoadStatisticsSetPruneEntriesDownTo:entries];
+}
+
void TestController::statisticsClearInMemoryAndPersistentStore()
{
[globalWebViewConfiguration.websiteDataStore _resourceLoadStatisticsClearInMemoryAndPersistentStore];
