jenkins-bot has submitted this change and it was merged.
Change subject: Use WebPageTest internal queue
......................................................................
Use WebPageTest internal queue
Always submit all batch jobs (within a file) to WebPageTest,
making use of the WebPageTest internal queue. Before we waited
on one test to finish before we submited the new one.
The new way makes the runs a little faster and opens up on using
scaling or having mulitiple instances (agents) running jobs making
the queue of jobs finish faster.
Also changed to promises instead of async, now we collect all runs
of a batch and return error if one or more fails. All finsihes first
though.
Bug: T120365
Change-Id: Ib605af9aba7ecfa1b1d0a346c7ba6a0c40ccebed
---
M bin/index.js
M lib/cli.js
M lib/index.js
M lib/util.js
M lib/wpt.js
M package.json
M scripts/batch/desktop-wpt-org.txt
M test/batchTest.js
8 files changed, 87 insertions(+), 93 deletions(-)
Approvals:
Krinkle: Looks good to me, approved
jenkins-bot: Verified
diff --git a/bin/index.js b/bin/index.js
index 2a23784..33e9840 100755
--- a/bin/index.js
+++ b/bin/index.js
@@ -11,6 +11,7 @@
var minimist = require('minimist');
var cli = require('../lib/cli');
var wpt = require('../lib/index');
+var Promise = require('bluebird');
var argv = cli.getMinimistArgv(process.argv.slice(2));
@@ -23,14 +24,28 @@
process.exit(1);
}
-var end = function(err) {
- if (err) {
- process.exit(1);
- }
-}
if (argv.batch) {
- wpt.runBatch(argv, end);
+ Promise.settle(wpt.runBatch(argv)).then(function(results) {
+ var ok = 0;
+ var failed = 0;
+ results.forEach(function(result) {
+ if (result.isFulfilled()) {
+ ok++;
+ } else {
+ console.error('Failing test:', result.reason());
+ failed++;
+ }
+ })
+ console.log('We got ' + ok + ' working tests and ' + failed + '
failing tests');
+ if (failed > 0) {
+ process.exit(1);
+ }
+ })
+
} else {
- wpt.runTest(argv, end);
+ wpt.runTest(argv).catch(function(err) {
+ console.error('Failing test:', err);
+ process.exit(1);
+ });
}
diff --git a/lib/cli.js b/lib/cli.js
index f8e6b99..63a78fc 100644
--- a/lib/cli.js
+++ b/lib/cli.js
@@ -44,7 +44,8 @@
console.log(' --runs Number of test runs [11]');
console.log(' --webPageTestHost The host of where to run your
test. ' +
'[' + DEFAULT_WEBPAGETEST_HOST + ']');
- console.log(' --timeout The timeout in seconds until we
end the test [1200]');
+ console.log(' --timeout The timeout in seconds until we
end the test. Zero ' +
+ 'is forever [0] ');
console.log(' --reporter Choose how you want to report the
metrics ' +
'[csv|json|graphite|statsv] ');
console.log(' --customMetrics A file with custom WPT metrics. '
+
diff --git a/lib/index.js b/lib/index.js
index f380658..8a108e6 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -11,16 +11,17 @@
var cli = require('./cli');
var eol = require('os').EOL;
var wpt = require('./wpt');
+var bluebird = require('bluebird');
var collectMetrics = require('./collectMetrics');
+
module.exports = {
/**
* Test multiple URLs using a batch file
* @param {array} argv The arguments for the run
*/
- runBatch: function(argv, cb) {
- var callback = cb || function() {};
+ runBatch: function(argv) {
var self = this;
- var series = [];
+ var promises = [];
var tests = util.readFile(argv.batch);
var lines = tests.split(eol);
lines.forEach(function(line) {
@@ -31,24 +32,12 @@
if (!cli.validateArgs(myArgs)) {
process.exit(1);
}
- series.push(function(callback) {
- self.runTest(myArgs, callback);
- });
+ promises.push(
+ self.runTest(myArgs)
+ );
}
});
- // lets run the tests one by one after each other. in that way
- // the wait time you configure (how long time you wait until a test is
finished)
- // is more logical, meaning the configured time is per test and not
for a
- // whole test suite.
- async.series(series, // jshint unused:false
- function(err, results) {
- if (err) {
- console.error('Couldn\'t execute all the runs.');
- } else {
- console.log('Succesfully run ' + series.length + '
tests.');
- }
- callback(err);
- });
+ return promises;
},
/**
@@ -56,8 +45,7 @@
* @param {array} argv The arguments for the run
* @param {function} cb The callback that will be called when the test
finished
*/
- runTest: function(argv, cb) {
- var callback = cb || function() {};
+ runTest: function(argv) {
var webPageTestHost = argv.webPageTestHost;
var arg = argv._[0];
@@ -72,41 +60,40 @@
wptOptions.custom = util.readFile(argv.customMetrics);
}
- console.log('Running WebPageTest [' + wptOptions.location + ' ' +
wptOptions.runs +
- ' time(s)] for ' + arg);
+ console.log('Add job to the WebPageTest server [' +
wptOptions.location + ' ' +
+ wptOptions.runs + ' time(s)] for ' + arg);
- wpt.run(webPageTestHost, argv.webPageTestKey, argv, input, wptOptions,
function(err, data) {
+ return wpt.run(webPageTestHost, argv.webPageTestKey, argv, input,
wptOptions)
+ .then(function(data) {
+ console.log('WebPageTest run: ' + data.data.summary);
- if (err) {
- // the errors from the WebPageTest API can be a little
confusing
- // but lets hope there is always an error when it doesn't work
- console.error('Couldn\'t fetch data from WebPageTest' + err);
- // lets report back using our internal reporting
- // so we don't break the next/coming runs
- return callback({});
- }
- var collectedMetrics = collectMetrics.collect(data, argv);
- // log browser and version if it's availible
- if (data.data.median && data.data.median.firstView) {
- var browserName = data.data.median.firstView.browser_name;
- var browserVersion =
data.data.median.firstView.browser_version;
- // for some browsers the name and version is not availible
- if (browserName && browserVersion) {
- console.log('Tested using ' + browserName + ' ' +
browserVersion);
+ if (argv.verbose) {
+ console.log(JSON.stringify(data, null, 1));
}
- }
- var reporter = require('./reporter').get(argv.reporter);
- reporter.report(collectedMetrics, argv);
- // did we get some error, then signal it!
- // in some cases WebPageTest doesn't report an error object but
the error code
- // shows an error, so lets check that
- // but WPT reports 304 as errors so make sure we don't catch them
- if (data.data.median && data.data.median.firstView &&
- data.data.median.firstView.result > 399) {
- return callback({});
- }
- return callback();
- });
+ var collectedMetrics = collectMetrics.collect(data, argv);
+ // log browser and version if it's availible
+ if (data.data.median && data.data.median.firstView) {
+ var browserName = data.data.median.firstView.browser_name;
+ var browserVersion =
data.data.median.firstView.browser_version;
+ // for some browsers the name and version is not availible
+ if (browserName && browserVersion) {
+ console.log('Tested using ' + browserName + ' ' +
browserVersion);
+ }
+ }
+ var reporter = require('./reporter').get(argv.reporter);
+ reporter.report(collectedMetrics, argv);
+
+ // did we get some error, then signal it!
+ // in some cases WebPageTest doesn't report an error object
but the error code
+ // shows an error, so lets check that
+ // but WPT reports 304 as errors so make sure we don't catch
them
+ if (data.data.median && data.data.median.firstView &&
+ data.data.median.firstView.result > 399) {
+ return bluebird.reject('We got error code ' +
+ data.data.median.firstView.result + ' for ' +
data.data.summary);
+ }
+ return;
+ });
}
};
diff --git a/lib/util.js b/lib/util.js
index db240bc..d4278eb 100644
--- a/lib/util.js
+++ b/lib/util.js
@@ -21,7 +21,7 @@
var wptOptions = {
pollResults: 10,
- timeout: 1200,
+ timeout: 0,
video: 'true',
label: argv.label ? this.getTodaysDate() + '-' + argv.label :
this.getTodaysDate(),
runs: 11
diff --git a/lib/wpt.js b/lib/wpt.js
index 4fdba0a..19fc9a4 100644
--- a/lib/wpt.js
+++ b/lib/wpt.js
@@ -6,29 +6,13 @@
*/
'use strict';
-
+var bluebird = require('bluebird');
var WebPageTest = require('webpagetest');
+bluebird.promisifyAll(WebPageTest.prototype);
module.exports = {
- run: function(host, key, argv, input, wptOptions, cb) {
+ run: function(host, key, argv, input, wptOptions) {
var wpt = new WebPageTest(host, key);
-
- wpt.runTest(input, wptOptions, function(err, data) {
-
- if (argv.verbose) {
- console.log(JSON.stringify(data, null, 1));
- }
-
- if (err) {
- console.error('Couldn\'t fetch data from WebPageTest:' +
JSON.stringify(err));
- console.error('Configuration:' + JSON.stringify(wptOptions,
null, 2));
- cb(err);
- return;
- }
-
- console.log('WebPageTest run: ' + data.data.summary);
- cb(null, data);
- });
-
+ return wpt.runTestAsync(input, wptOptions);
}
};
diff --git a/package.json b/package.json
index 883de07..57e2cce 100644
--- a/package.json
+++ b/package.json
@@ -38,9 +38,9 @@
},
"main": "./bin/index.js",
"dependencies": {
- "async": "1.4.2",
+ "bluebird": "3.0.6",
"minimist": "1.1.3",
"request": "2.61.0",
"webpagetest": "0.3.3"
}
-}
+}
\ No newline at end of file
diff --git a/scripts/batch/desktop-wpt-org.txt
b/scripts/batch/desktop-wpt-org.txt
index bbc1c3d..d3c4a89 100644
--- a/scripts/batch/desktop-wpt-org.txt
+++ b/scripts/batch/desktop-wpt-org.txt
@@ -11,4 +11,4 @@
# nodejs installed), just make sure to change the value of the WebPageTest key:
# $ WPT_ORG_WPT_KEY=SECRET_KEY STATSV_ENDPOINT=http://localhost WPT_ORG_RUNS=1
bin/index.js --batch ./scripts/batch/desktop-wpt-org.txt
---webPageTestKey <%WPT_ORG_WPT_KEY> --webPageTestHost www.webpagetest.org
--median SpeedIndex --location Dulles:Chrome --label chrome-dulles --runs
<%WPT_ORG_RUNS> --endpoint <%STATSV_ENDPOINT> --namespace
webpagetest.enwiki.anonymous.BlankPage --timeout 2400 --reporter statsv
https://en.wikipedia.org/wiki/Special:BlankPage
+--webPageTestKey <%WPT_ORG_WPT_KEY> --webPageTestHost www.webpagetest.org
--median SpeedIndex --location Dulles:Chrome --label chrome-dulles --runs
<%WPT_ORG_RUNS> --endpoint <%STATSV_ENDPOINT> --namespace
webpagetest.enwiki.anonymous.BlankPage --reporter statsv
https://en.wikipedia.org/wiki/Special:BlankPage
diff --git a/test/batchTest.js b/test/batchTest.js
index 579e63f..afefb67 100644
--- a/test/batchTest.js
+++ b/test/batchTest.js
@@ -2,18 +2,23 @@
var mockery = require('mockery');
var assert = require('assert');
var util = require('../lib/util');
+var Promise = require('bluebird');
var desktopJson = JSON.parse(util.readFile('test/files/desktop_result.json'));
var failingDesktopJson =
JSON.parse(util.readFile('test/files/desktop_result_failing.json'));
var wptMock = {
- run: function(host, key, argv, input, wptOptions, cb) {
- cb(null, desktopJson);
+ run: function(host, key, argv, input, wptOptions) {
+ return new Promise(function(resolve, reject) {
+ resolve(desktopJson);
+ });
}
};
var wptFailingMock = {
- run: function(host, key, argv, input, wptOptions, cb) {
- cb(null, failingDesktopJson);
+ run: function(host, key, argv, input, wptOptions) {
+ return new Promise(function(resolve, reject) {
+ reject();
+ });
}
};
@@ -35,12 +40,13 @@
batch: 'test/files/batch.txt'
};
var test = require('../lib/');
- test.runBatch(argv, function(err) {
+ Promise.all(test.runBatch(argv)).catch(function(err) {
assert.ifError(err);
+ }).finally(function() {
done();
});
-
});
+
after(function() {
mockery.disable();
@@ -61,8 +67,9 @@
batch: 'test/files/batch.txt'
};
var test = require('../lib/');
- test.runBatch(argv, function(err) {
- assert.ifError(!err)
+ Promise.all(test.runBatch(argv)).catch(function(err) {
+ assert.ifError(!err);
+ }).finally(function() {
done();
});
--
To view, visit https://gerrit.wikimedia.org/r/257569
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib605af9aba7ecfa1b1d0a346c7ba6a0c40ccebed
Gerrit-PatchSet: 6
Gerrit-Project: performance/WebPageTest
Gerrit-Branch: master
Gerrit-Owner: Phedenskog <[email protected]>
Gerrit-Reviewer: Krinkle <[email protected]>
Gerrit-Reviewer: Phedenskog <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
