Modified: trunk/Source/WebKit/ChangeLog (219498 => 219499)
--- trunk/Source/WebKit/ChangeLog 2017-07-14 06:18:15 UTC (rev 219498)
+++ trunk/Source/WebKit/ChangeLog 2017-07-14 06:39:08 UTC (rev 219499)
@@ -1,3 +1,42 @@
+2017-07-13 Chris Dumez <cdu...@apple.com>
+
+ Better deal with changes to the ResourceLoadStatisticsStore file on disk
+ https://bugs.webkit.org/show_bug.cgi?id=174487
+
+ Reviewed by Brent Fulgham.
+
+ Rename WebResourceLoadStatisticsStore's resetDataFromDecoder() to mergeWithDataFromDecoder()
+ and update it so that it actual merges the data from the disk with the one we have in memory,
+ instead of replacing it. This avoid data loss.
+
+ We leverage the existing WebResourceLoadStatisticsStore::mergeStatistics() to merge the
+ statistics.
+
+ To faciliate merging the operating dates, they are now represented as OperatingDate objects
+ instead of WallTime objects. OperatingDate only include date precision, no time information.
+ As a result, the merge 2 vectors of OperatingDate objects, we can:
+ 1. Merge the 2 sorted vectors (using std::merge())
+ 2. Get rid of duplicates (easy because of date-level precision)
+ 3. Drop old dates until the vector has at most 30 items.
+
+ * UIProcess/Storage/ResourceLoadStatisticsPersistentStorage.cpp:
+ (WebKit::ResourceLoadStatisticsPersistentStorage::refreshMemoryStoreFromDisk):
+ (WebKit::ResourceLoadStatisticsPersistentStorage::populateMemoryStoreFromDisk):
+ * UIProcess/WebResourceLoadStatisticsStore.cpp:
+ (WebKit::OperatingDate::fromWallTime):
+ (WebKit::OperatingDate::today):
+ (WebKit::OperatingDate::secondsSinceEpoch):
+ (WebKit::OperatingDate::operator==):
+ (WebKit::OperatingDate::operator<):
+ (WebKit::OperatingDate::operator<=):
+ (WebKit::OperatingDate::OperatingDate):
+ (WebKit::mergeOperatingDates):
+ (WebKit::WebResourceLoadStatisticsStore::createEncoderFromData):
+ (WebKit::WebResourceLoadStatisticsStore::mergeWithDataFromDecoder):
+ (WebKit::WebResourceLoadStatisticsStore::includeTodayAsOperatingDateIfNecessary):
+ (WebKit::WebResourceLoadStatisticsStore::hasStatisticsExpired):
+ * UIProcess/WebResourceLoadStatisticsStore.h:
+
2017-07-13 Dan Bernstein <m...@apple.com>
Removed empty project directories left behind after the rename
Modified: trunk/Source/WebKit/UIProcess/Storage/ResourceLoadStatisticsPersistentStorage.cpp (219498 => 219499)
--- trunk/Source/WebKit/UIProcess/Storage/ResourceLoadStatisticsPersistentStorage.cpp 2017-07-14 06:18:15 UTC (rev 219498)
+++ trunk/Source/WebKit/UIProcess/Storage/ResourceLoadStatisticsPersistentStorage.cpp 2017-07-14 06:39:08 UTC (rev 219499)
@@ -166,7 +166,7 @@
if (!decoder)
return;
- m_memoryStore.resetDataFromDecoder(*decoder);
+ m_memoryStore.mergeWithDataFromDecoder(*decoder);
m_lastStatisticsFileSyncTime = readTime;
}
@@ -193,7 +193,8 @@
return;
}
- m_memoryStore.resetDataFromDecoder(*decoder);
+ ASSERT_WITH_MESSAGE(m_memoryStore.isEmpty(), "This is the initial import so the store should be empty");
+ m_memoryStore.mergeWithDataFromDecoder(*decoder);
m_lastStatisticsFileSyncTime = readTime;
Modified: trunk/Source/WebKit/UIProcess/WebResourceLoadStatisticsStore.cpp (219498 => 219499)
--- trunk/Source/WebKit/UIProcess/WebResourceLoadStatisticsStore.cpp 2017-07-14 06:18:15 UTC (rev 219498)
+++ trunk/Source/WebKit/UIProcess/WebResourceLoadStatisticsStore.cpp 2017-07-14 06:39:08 UTC (rev 219499)
@@ -37,6 +37,7 @@
#include <WebCore/KeyedCoding.h>
#include <WebCore/ResourceLoadStatistics.h>
#include <wtf/CrossThreadCopier.h>
+#include <wtf/DateMath.h>
#include <wtf/MathExtras.h>
#include <wtf/NeverDestroyed.h>
@@ -77,6 +78,77 @@
return dataTypes;
}
+class OperatingDate {
+public:
+ OperatingDate() = default;
+
+ static OperatingDate fromWallTime(WallTime time)
+ {
+ double ms = time.secondsSinceEpoch().milliseconds();
+ int year = msToYear(ms);
+ int yearDay = dayInYear(ms, year);
+ int month = monthFromDayInYear(yearDay, isLeapYear(year));
+ int monthDay = dayInMonthFromDayInYear(yearDay, isLeapYear(year));
+
+ return OperatingDate { year, month, monthDay };
+ }
+
+ static OperatingDate today()
+ {
+ return OperatingDate::fromWallTime(WallTime::now());
+ }
+
+ Seconds secondsSinceEpoch() const
+ {
+ return Seconds { dateToDaysFrom1970(m_year, m_month, m_monthDay) * secondsPerDay };
+ }
+
+ bool operator==(const OperatingDate& other) const
+ {
+ return m_monthDay == other.m_monthDay && m_month == other.m_month && m_year == other.m_year;
+ }
+
+ bool operator<(const OperatingDate& other) const
+ {
+ return secondsSinceEpoch() < other.secondsSinceEpoch();
+ }
+
+ bool operator<=(const OperatingDate& other) const
+ {
+ return secondsSinceEpoch() <= other.secondsSinceEpoch();
+ }
+
+private:
+ OperatingDate(int year, int month, int monthDay)
+ : m_year(year)
+ , m_month(month)
+ , m_monthDay(monthDay)
+ { }
+
+ int m_year { 0 };
+ int m_month { 0 }; // [0, 11].
+ int m_monthDay { 0 }; // [1, 31].
+};
+
+static Vector<OperatingDate> mergeOperatingDates(const Vector<OperatingDate>& existingDates, Vector<OperatingDate>&& newDates)
+{
+ if (existingDates.isEmpty())
+ return WTFMove(newDates);
+
+ Vector<OperatingDate> mergedDates(existingDates.size() + newDates.size());
+
+ // Merge the two sorted vectors of dates.
+ std::merge(existingDates.begin(), existingDates.end(), newDates.begin(), newDates.end(), mergedDates.begin());
+ // Remove duplicate dates.
+ removeRepeatedElements(mergedDates);
+
+ // Drop old dates until the Vector size reaches operatingDatesWindow.
+ while (mergedDates.size() > operatingDatesWindow)
+ mergedDates.remove(0);
+
+ return mergedDates;
+}
+
WebResourceLoadStatisticsStore::WebResourceLoadStatisticsStore(const String& resourceLoadStatisticsDirectory, UpdateCookiePartitioningForDomainsHandler&& updateCookiePartitioningForDomainsHandler)
: m_statisticsQueue(WorkQueue::create("WebResourceLoadStatisticsStore Process Data Queue", WorkQueue::Type::Serial, WorkQueue::QOS::Utility))
, m_persistentStorage(*this, resourceLoadStatisticsDirectory)
@@ -478,7 +550,7 @@
origin.value.encode(encoderInner);
});
- encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, WallTime date) {
+ encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
});
@@ -485,12 +557,10 @@
return encoder;
}
-void WebResourceLoadStatisticsStore::resetDataFromDecoder(KeyedDecoder& decoder)
+void WebResourceLoadStatisticsStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
{
ASSERT(!RunLoop::isMain());
- clearInMemory();
-
unsigned versionOnDisk;
if (!decoder.decodeUInt32("version", versionOnDisk))
return;
@@ -512,22 +582,16 @@
if (!succeeded)
return;
- Vector<String> prevalentResourceDomainsWithoutUserInteraction;
- prevalentResourceDomainsWithoutUserInteraction.reserveInitialCapacity(loadedStatistics.size());
- for (auto& statistics : loadedStatistics) {
- if (statistics.isPrevalentResource && !statistics.hadUserInteraction) {
- prevalentResourceDomainsWithoutUserInteraction.uncheckedAppend(statistics.highLevelDomain);
- statistics.isMarkedForCookiePartitioning = true;
- }
- m_resourceStatisticsMap.add(statistics.highLevelDomain, WTFMove(statistics));
- }
+ mergeStatistics(WTFMove(loadedStatistics));
+ updateCookiePartitioning();
- succeeded = decoder.decodeObjects("operatingDates", m_operatingDates, [](KeyedDecoder& decoder, WallTime& wallTime) {
+ Vector<OperatingDate> operatingDates;
+ succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
double value;
if (!decoder.decodeDouble("date", value))
return false;
- wallTime = WallTime::fromRawSeconds(value);
+ date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
return true;
});
@@ -534,7 +598,7 @@
if (!succeeded)
return;
- updateCookiePartitioningForDomains({ }, prevalentResourceDomainsWithoutUserInteraction, ShouldClearFirst::Yes);
+ m_operatingDates = mergeOperatingDates(m_operatingDates, WTFMove(operatingDates));
}
void WebResourceLoadStatisticsStore::clearInMemory()
@@ -660,19 +724,20 @@
void WebResourceLoadStatisticsStore::includeTodayAsOperatingDateIfNecessary()
{
- if (!m_operatingDates.isEmpty() && (WallTime::now() - m_operatingDates.last() < 24_h))
+ auto today = OperatingDate::today();
+ if (!m_operatingDates.isEmpty() && today <= m_operatingDates.last())
return;
while (m_operatingDates.size() >= operatingDatesWindow)
- m_operatingDates.removeFirst();
+ m_operatingDates.remove(0);
- m_operatingDates.append(WallTime::now());
+ m_operatingDates.append(today);
}
bool WebResourceLoadStatisticsStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
{
if (m_operatingDates.size() >= operatingDatesWindow) {
- if (resourceStatistic.mostRecentUserInteractionTime < m_operatingDates.first())
+ if (OperatingDate::fromWallTime(resourceStatistic.mostRecentUserInteractionTime) < m_operatingDates.first())
return true;
}
Modified: trunk/Source/WebKit/UIProcess/WebResourceLoadStatisticsStore.h (219498 => 219499)
--- trunk/Source/WebKit/UIProcess/WebResourceLoadStatisticsStore.h 2017-07-14 06:18:15 UTC (rev 219498)
+++ trunk/Source/WebKit/UIProcess/WebResourceLoadStatisticsStore.h 2017-07-14 06:39:08 UTC (rev 219499)
@@ -51,6 +51,7 @@
namespace WebKit {
+class OperatingDate;
class WebProcessProxy;
enum class ShouldClearFirst;
@@ -113,7 +114,7 @@
void resetParametersToDefaultValues();
std::unique_ptr<WebCore::KeyedEncoder> createEncoderFromData() const;
- void resetDataFromDecoder(WebCore::KeyedDecoder&);
+ void mergeWithDataFromDecoder(WebCore::KeyedDecoder&);
void clearInMemory();
void grandfatherExistingWebsiteData();
@@ -165,7 +166,7 @@
#endif
Ref<WTF::WorkQueue> m_statisticsQueue;
ResourceLoadStatisticsPersistentStorage m_persistentStorage;
- Deque<WTF::WallTime> m_operatingDates;
+ Vector<OperatingDate> m_operatingDates;
UpdateCookiePartitioningForDomainsHandler m_updateCookiePartitioningForDomainsHandler;
