Modified: branches/safari-611-branch/PerformanceTests/ChangeLog (273389 => 273390)
--- branches/safari-611-branch/PerformanceTests/ChangeLog 2021-02-24 17:10:07 UTC (rev 273389)
+++ branches/safari-611-branch/PerformanceTests/ChangeLog 2021-02-24 17:10:10 UTC (rev 273390)
@@ -1,3 +1,55 @@
+2021-02-23 Alan Coon <alanc...@apple.com>
+
+ Cherry-pick r273122. rdar://problem/74623473
+
+ MotionMark scores are super sensitive to a single long frame
+ https://bugs.webkit.org/show_bug.cgi?id=220847
+ <rdar://problem/74152743>
+
+ Reviewed by Maciej Stachowiak and Jon Lee.
+
+ Currently, "ramp" tests have three phases. The middle phase is where they try to determine a maximum reasonable
+ complexity, and the third one is where they try various complexities between 0 and the maximum. The calculation
+ of this maximum reasonable complexity is currently very sensitive to outlier frame times. If there is a single
+ outlier frame time, the failure mode is to assume that the maximum complexity is ~10. So, the solution is to
+ ignore outlier frame times during this first phase, and to ensure that there are at least 9 frames measured that
+ have non-outlier times.
+
+ * MotionMark/tests/resources/main.js:
+ (filterOutOutliers):
+ (_measureAndResetInterval):
+ (update):
+ (registerFrameTime):
+ (intervalHasConcluded):
+ (start):
+ (didFinishInterval):
+
+ git-svn-id: https://svn.webkit.org/repository/webkit/trunk@273122 268f45cc-cd09-0410-ab3c-d52691b4dbfc
+
+ 2021-02-18 Myles C. Maxfield <mmaxfi...@apple.com>
+
+ MotionMark scores are super sensitive to a single long frame
+ https://bugs.webkit.org/show_bug.cgi?id=220847
+ <rdar://problem/74152743>
+
+ Reviewed by Maciej Stachowiak and Jon Lee.
+
+ Currently, "ramp" tests have three phases. The middle phase is where they try to determine a maximum reasonable
+ complexity, and the third one is where they try various complexities between 0 and the maximum. The calculation
+ of this maximum reasonable complexity is currently very sensitive to outlier frame times. If there is a single
+ outlier frame time, the failure mode is to assume that the maximum complexity is ~10. So, the solution is to
+ ignore outlier frame times during this first phase, and to ensure that there are at least 9 frames measured that
+ have non-outlier times.
+
+ * MotionMark/tests/resources/main.js:
+ (filterOutOutliers):
+ (_measureAndResetInterval):
+ (update):
+ (registerFrameTime):
+ (intervalHasConcluded):
+ (start):
+ (didFinishInterval):
+
2021-02-17 Ruben Turcios <rubent...@apple.com>
Cherry-pick r270665. rdar://problem/74409412
Modified: branches/safari-611-branch/PerformanceTests/MotionMark/tests/resources/main.js (273389 => 273390)
--- branches/safari-611-branch/PerformanceTests/MotionMark/tests/resources/main.js 2021-02-24 17:10:07 UTC (rev 273389)
+++ branches/safari-611-branch/PerformanceTests/MotionMark/tests/resources/main.js 2021-02-24 17:10:10 UTC (rev 273390)
@@ -110,6 +110,20 @@
return comment in this._marks;
},
+ filterOutOutliers: function(array)
+ {
+ if (array.length == 0)
+ return [];
+
+ array.sort();
+ var q1 = array[Math.min(Math.round(array.length * 1 / 4), array.length - 1)];
+ var q3 = array[Math.min(Math.round(array.length * 3 / 4), array.length - 1)];
+ var interquartileRange = q3 - q1;
+ var minimum = q1 - interquartileRange * 1.5;
+ var maximum = q3 + interquartileRange * 1.5;
+ return array.filter(x => x >= minimum && x <= maximum);
+ },
+
_measureAndResetInterval: function(currentTimestamp)
{
var sampleCount = this._sampler.sampleCount;
@@ -116,8 +130,13 @@
var averageFrameLength = 0;
if (this._intervalEndTimestamp) {
- var intervalStartTimestamp = this._sampler.samples[0][this._intervalStartIndex];
- averageFrameLength = (currentTimestamp - intervalStartTimestamp) / (sampleCount - this._intervalStartIndex);
+ var durations = [];
+ for (var i = Math.max(this._intervalStartIndex, 1); i < sampleCount; ++i) {
+ durations.push(this._sampler.samples[0][i] - this._sampler.samples[0][i - 1]);
+ }
+ var filteredDurations = this.filterOutOutliers(durations);
+ if (filteredDurations.length > 0)
+ averageFrameLength = filteredDurations.reduce((a, b) => a + b, 0) / filteredDurations.length;
}
this._intervalStartIndex = sampleCount;
@@ -138,15 +157,19 @@
this._frameLengthEstimator.sample(lastFrameLength);
frameLengthEstimate = this._frameLengthEstimator.estimate;
}
- } else if (timestamp >= this._intervalEndTimestamp) {
- var intervalStartTimestamp = this._sampler.samples[0][this._intervalStartIndex];
- intervalAverageFrameLength = this._measureAndResetInterval(timestamp);
- if (this._isFrameLengthEstimatorEnabled) {
- this._frameLengthEstimator.sample(intervalAverageFrameLength);
- frameLengthEstimate = this._frameLengthEstimator.estimate;
+ } else {
+ this.registerFrameTime(lastFrameLength);
+ if (this.intervalHasConcluded(timestamp)) {
+ var intervalStartTimestamp = this._sampler.samples[0][this._intervalStartIndex];
+ intervalAverageFrameLength = this._measureAndResetInterval(timestamp);
+ if (this._isFrameLengthEstimatorEnabled) {
+ this._frameLengthEstimator.sample(intervalAverageFrameLength);
+ frameLengthEstimate = this._frameLengthEstimator.estimate;
+ }
+ didFinishInterval = true;
+ this.didFinishInterval(timestamp, stage, intervalAverageFrameLength);
+ this._frameLengthEstimator.reset();
}
- didFinishInterval = true;
- this.didFinishInterval(timestamp, stage, intervalAverageFrameLength);
}
this._sampler.record(timestamp, stage.complexity(), frameLengthEstimate);
@@ -153,6 +176,15 @@
this.tune(timestamp, stage, lastFrameLength, didFinishInterval, intervalAverageFrameLength);
},
+ registerFrameTime: function(lastFrameLength)
+ {
+ },
+
+ intervalHasConcluded: function(timestamp)
+ {
+ return timestamp >= this._intervalEndTimestamp;
+ },
+
didFinishInterval: function(timestamp, stage, intervalAverageFrameLength)
{
},
@@ -336,6 +368,8 @@
tierFastTestLength: 250,
// If the engine is under stress, let the test run a little longer to let the measurement settle
tierSlowTestLength: 750,
+ // Tier intervals must have this number of non-outlier frames in order to end.
+ numberOfFramesRequiredInInterval: 9,
rampWarmupLength: 200,
@@ -355,10 +389,25 @@
Controller.prototype.start.call(this, startTimestamp, stage);
this._rampStartTimestamp = 0;
this.intervalSamplingLength = 100;
+ this._frameTimeHistory = [];
},
+ registerFrameTime: function(lastFrameLength)
+ {
+ this._frameTimeHistory.push(lastFrameLength);
+ },
+
+ intervalHasConcluded: function(timestamp)
+ {
+ if (!Controller.prototype.intervalHasConcluded.call(this, timestamp))
+ return false;
+
+ return this._finishedTierSampling || this.filterOutOutliers(this._frameTimeHistory).length > this.numberOfFramesRequiredInInterval;
+ },
+
didFinishInterval: function(timestamp, stage, intervalAverageFrameLength)
{
+ this._frameTimeHistory = [];
if (!this._finishedTierSampling) {
if (this._tierStartTimestamp > 0 && timestamp < this._tierStartTimestamp + this.tierFastTestLength)
return;
