Diff
Modified: trunk/Tools/ChangeLog (90770 => 90771)
--- trunk/Tools/ChangeLog 2011-07-11 18:51:13 UTC (rev 90770)
+++ trunk/Tools/ChangeLog 2011-07-11 18:54:13 UTC (rev 90771)
@@ -1,5 +1,24 @@
2011-07-11 Adam Barth <aba...@webkit.org>
+ garden-o-matic should be able to determine which revisions caused a given failure
+ https://bugs.webkit.org/show_bug.cgi?id=64189
+
+ Reviewed by Adam Roben.
+
+ Walking the failure history looking for failures turns out to be
+ slightly tricky because the network requests are asynchronous.
+ Currently we do all the fetches serially and our cache is unbounded.
+ We'll probably optimize both those parameters eventually.
+
+ This patch also generalizes some functionality in the unit testing
+ framework to make testing this sort of code easier.
+
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/base.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
+
+2011-07-11 Adam Barth <aba...@webkit.org>
+
Add a webkit-patch command for rebaselining an individual test
https://bugs.webkit.org/show_bug.cgi?id=64246
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js (90770 => 90771)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js 2011-07-11 18:51:13 UTC (rev 90770)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js 2011-07-11 18:54:13 UTC (rev 90771)
@@ -64,4 +64,15 @@
document.head.appendChild(scriptElement);
};
+// jQuery makes jsonp requests somewhat ugly (which is fair given that they're
+// terrible for security). We use this wrapper to make our lives slightly easier.
+base.jsonp = function(url, onsuccess)
+{
+ $.ajax({
+ url: url,
+ dataType: 'jsonp',
+ success: onsuccess
+ });
+};
+
})();
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js (90770 => 90771)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js 2011-07-11 18:51:13 UTC (rev 90770)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js 2011-07-11 18:54:13 UTC (rev 90771)
@@ -3,6 +3,7 @@
(function() {
var kTestResultsServer = 'http://test-results.appspot.com/';
+var kTestResultsQuery = kTestResultsServer + 'testfile?'
var kTestType = 'layout-tests';
var kResultsName = 'full_results.json';
var kMasterName = 'ChromiumWebkit';
@@ -57,6 +58,57 @@
return result === PASS;
}
+function resultsParameters(builderName, testName)
+{
+ return {
+ builder: builderName,
+ master: kMasterName,
+ testtype: kTestType,
+ name: name,
+ };
+}
+
+function resultsSummaryURL(builderName, testName)
+{
+ return kTestResultsQuery + $.param(resultsParameters(builderName, testName));
+}
+
+function directoryOfResultsSummaryURL(builderName, testName)
+{
+ var parameters = resultsParameters(builderName, testName);
+ parameters['dir'] = 1;
+ return kTestResultsQuery + $.param(parameters);
+}
+
+function ResultsCache()
+{
+ this._cache = {};
+}
+
+ResultsCache.prototype._fetch = function(key, callback)
+{
+ var self = this;
+
+ var url = "" + 'testfile?key=' + key;
+ base.jsonp(url, function (resultsTree) {
+ self._cache[key] = resultsTree;
+ callback(resultsTree);
+ });
+};
+
+// Warning! This function can call callback either synchronously or asynchronously.
+// FIXME: Consider using setTimeout to make this method always asynchronous.
+ResultsCache.prototype.get = function(key, callback)
+{
+ if (key in this._cache) {
+ callback(this._cache[key]);
+ return;
+ }
+ this._fetch(key, callback);
+};
+
+var g_resultsCache = new ResultsCache();
+
function anyIsFailure(resultsList)
{
return $.grep(resultsList, isFailure).length > 0;
@@ -98,28 +150,112 @@
results.BuilderResults = function(resultsJSON)
{
- this.m_resultsJSON = resultsJSON;
+ this._resultsJSON = resultsJSON;
};
results.BuilderResults.prototype.unexpectedFailures = function()
{
- return base.filterTree(this.m_resultsJSON.tests, isResultNode, isUnexpectedFailure);
+ return base.filterTree(this._resultsJSON.tests, isResultNode, isUnexpectedFailure);
};
results.unexpectedFailuresByTest = function(resultsByBuilder)
{
var unexpectedFailures = {};
- $.each(resultsByBuilder, function(buildName, builderResults) {
+ $.each(resultsByBuilder, function(builderName, builderResults) {
$.each(builderResults.unexpectedFailures(), function(testName, resultNode) {
unexpectedFailures[testName] = unexpectedFailures[testName] || {};
- unexpectedFailures[testName][buildName] = resultNode;
+ unexpectedFailures[testName][builderName] = resultNode;
});
});
return unexpectedFailures;
};
+function TestHistoryWalker(builderName, testName)
+{
+ this._builderName = builderName;
+ this._testName = testName;
+ this._indexOfNextKeyToFetch = 0;
+ this._keyList = [];
+}
+
+TestHistoryWalker.prototype.init = function(callback)
+{
+ var self = this;
+
+ base.jsonp(directoryOfResultsSummaryURL(self._builderName, kResultsName), function(keyList) {
+ self._keyList = keyList.map(function (element) { return element.key; });
+ callback();
+ });
+};
+
+TestHistoryWalker.prototype._fetchNextResultNode = function(callback)
+{
+ var self = this;
+
+ if (self._indexOfNextKeyToFetch >= self._keyList) {
+ callback(0, null);
+ return;
+ }
+
+ var key = self._keyList[self._indexOfNextKeyToFetch];
+ ++self._indexOfNextKeyToFetch;
+ g_resultsCache.get(key, function(resultsTree) {
+ var resultNode = results.resultNodeForTest(resultsTree, self._testName);
+ callback(resultsTree['revision'], resultNode);
+ });
+};
+
+TestHistoryWalker.prototype.walkHistory = function(callback)
+{
+ var self = this;
+ self._fetchNextResultNode(function(revision, resultNode) {
+ var shouldContinue = callback(revision, resultNode);
+ if (!shouldContinue)
+ return;
+ self.walkHistory(callback);
+ });
+}
+
+results.regressionRangeForFailure = function(builderName, testName, callback)
+{
+ var oldestFailingRevision = 0;
+ var newestPassingRevision = 0;
+
+ var historyWalker = new TestHistoryWalker(builderName, testName);
+ historyWalker.init(function() {
+ historyWalker.walkHistory(function(revision, resultNode) {
+ if (!resultNode) {
+ newestPassingRevision = revision;
+ callback(oldestFailingRevision, newestPassingRevision);
+ return false;
+ }
+ if (isUnexpectedFailure(resultNode)) {
+ oldestFailingRevision = revision;
+ return true;
+ }
+ if (!oldestFailingRevision)
+ return true; // We need to keep looking for a failing revision.
+ newestPassingRevision = revision;
+ callback(oldestFailingRevision, newestPassingRevision);
+ return false;
+ });
+ });
+};
+
+results.resultNodeForTest = function(resultsTree, testName)
+{
+ var testNamePath = testName.split('/');
+ var currentNode = resultsTree['tests'];
+ $.each(testNamePath, function(index, segmentName) {
+ if (!currentNode)
+ return;
+ currentNode = (segmentName in currentNode) ? currentNode[segmentName] : null;
+ });
+ return currentNode;
+};
+
function resultsDirectoryForBuilder(builderName)
{
return builderName.replace(/[ .()]/g, '_');
@@ -189,23 +325,10 @@
});
};
-function resultsSummaryURL(builderName, name)
-{
- return kTestResultsServer + 'testfile' +
- '?builder=' + builderName +
- '&master=' + kMasterName +
- '&testtype=' + kTestType +
- '&name=' + name;
-}
-
results.fetchResultsForBuilder = function(builderName, onsuccess)
{
- $.ajax({
- url: resultsSummaryURL(builderName, kResultsName),
- dataType: 'jsonp',
- success: function(data) {
- onsuccess(new results.BuilderResults(data));
- }
+ base.jsonp(resultsSummaryURL(builderName, kResultsName), function(resultsTree) {
+ onsuccess(new results.BuilderResults(resultsTree));
});
};
@@ -214,8 +337,8 @@
var resultsByBuilder = {}
var requestsInFlight = builderNameList.length;
$.each(builderNameList, function(index, builderName) {
- results.fetchResultsForBuilder(builderName, function(builderResults) {
- resultsByBuilder[builderName] = builderResults;
+ results.fetchResultsForBuilder(builderName, function(resultsTree) {
+ resultsByBuilder[builderName] = resultsTree;
--requestsInFlight;
if (!requestsInFlight)
onsuccess(resultsByBuilder);
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js (90770 => 90771)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js 2011-07-11 18:51:13 UTC (rev 90770)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js 2011-07-11 18:54:13 UTC (rev 90771)
@@ -93,34 +93,156 @@
equals(results.resultType("http://example.com/foo.xyz"), "text");
});
-test("fetchResultsURLs", 3, function() {
+test("resultNodeForTest", 4, function() {
+ deepEqual(results.resultNodeForTest(kExampleResultsJSON, "userscripts/another-test.html"), {
+ "expected": "PASS",
+ "actual": "TEXT"
+ });
+ equals(results.resultNodeForTest(kExampleResultsJSON, "foo.html"), null);
+ equals(results.resultNodeForTest(kExampleResultsJSON, "userscripts/foo.html"), null);
+ equals(results.resultNodeForTest(kExampleResultsJSON, "userscripts/foo/bar.html"), null);
+});
+
+function NetworkSimulator()
+{
+ this._pendingCallbacks = [];
+};
+
+NetworkSimulator.prototype.scheduleCallback = function(callback)
+{
+ this._pendingCallbacks.push(callback);
+}
+
+NetworkSimulator.prototype.runTest = function(testCase)
+{
+ var self = this;
var realBase = window.base;
- var pendingCallbacks = {};
window.base = {};
- base.probe = function(url, options) {
- pendingCallbacks[url] = options;
- };
base.endsWith = realBase.endsWith;
base.trimExtension = realBase.trimExtension;
+ if (self.probeHook)
+ base.probe = self.probeHook;
+ if (self.jsonpHook)
+ base.jsonp = self.jsonpHook;
- results.fetchResultsURLs("Mock Builder", "userscripts/another-test.html", function(resultURLs) {
- deepEqual(resultURLs, [
- "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.txt",
- "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.txt",
- "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.txt",
- ]);
+ testCase();
+
+ while (this._pendingCallbacks.length) {
+ var callback = this._pendingCallbacks.shift();
+ callback();
+ }
+
+ window.base = realBase;
+ equal(window.base, realBase, "Failed to restore real base!");
+}
+
+test("regressionRangeForFailure", 3, function() {
+ simulator = new NetworkSimulator();
+
+ var keyMap = {
+ "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw": {
+ "tests": {
+ "userscripts": {
+ "another-test.html": {
+ "expected": "PASS",
+ "actual": "TEXT"
+ }
+ },
+ },
+ "revision": "90430"
+ },
+ "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw":{
+ "tests": {
+ "userscripts": {
+ "user-script-video-document.html": {
+ "expected": "FAIL",
+ "actual": "TEXT"
+ },
+ "another-test.html": {
+ "expected": "PASS",
+ "actual": "TEXT"
+ }
+ },
+ },
+ "revision": "90429"
+ },
+ "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw":{
+ "tests": {
+ "userscripts": {
+ "another-test.html": {
+ "expected": "PASS",
+ "actual": "TEXT"
+ }
+ },
+ },
+ "revision": "90426"
+ },
+ "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw":{
+ "tests": {
+ "userscripts": {
+ "user-script-video-document.html": {
+ "expected": "FAIL",
+ "actual": "TEXT"
+ },
+ },
+ },
+ "revision": "90424"
+ }
+ };
+
+ simulator.jsonpHook = function(url, callback) {
+ simulator.scheduleCallback(function() {
+ if (/dir=1/.test(url)) {
+ callback([
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGOj5UAw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGP-AUQw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGPL3UAw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNHJQAw" },
+ ]);
+ } else {
+ var key = url.match(/key=([^&]+)/)[1];
+ callback(keyMap[key]);
+ }
+ });
+ };
+ simulator.runTest(function() {
+ results.regressionRangeForFailure("Mock Builder", "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
+ equals(oldestFailingRevision, "90426");
+ equals(newestPassingRevision, "90424");
+ });
});
+});
+test("fetchResultsURLs", 3, function() {
+ var simulator = new NetworkSimulator();
+
var probedURLs = [];
- for (var url in pendingCallbacks) {
- probedURLs.push(url);
- if (realBase.endsWith(url, '.txt'))
- pendingCallbacks[url].success.call();
- else
- pendingCallbacks[url].error.call();
- }
+ simulator.probeHook = function(url, options)
+ {
+ simulator.scheduleCallback(function() {
+ probedURLs.push(url);
+ if (base.endsWith(url, '.txt'))
+ options.success.call();
+ else
+ options.error.call();
+ });
+ };
+ simulator.runTest(function() {
+ results.fetchResultsURLs("Mock Builder", "userscripts/another-test.html", function(resultURLs) {
+ deepEqual(resultURLs, [
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.txt",
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.txt",
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.txt",
+ ]);
+ });
+ });
+
deepEqual(probedURLs, [
"http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.png",
"http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.png",
@@ -129,7 +251,4 @@
"http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.txt",
"http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.txt",
]);
-
- window.base = realBase;
- equal(window.base, realBase, "Failed to restore real base!");
});
