GWicke has uploaded a new change for review.
https://gerrit.wikimedia.org/r/103486
Change subject: Improve HTTP error handling in rt test client
......................................................................
Improve HTTP error handling in rt test client
* Add a retryingHTTPRequest helper in Util using the node request module.
* Use it from both the rt test client and roundtrip-test.js, so that
idempotent requests are retried automatically with exponential back-off.
Change-Id: I83b5a266f9db4ce992aa06598ba97fd8f9597b3d
---
M lib/mediawiki.Util.js
M tests/apiServer.js
M tests/client/client.js
M tests/roundtrip-test.js
4 files changed, 56 insertions(+), 35 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/86/103486/1
diff --git a/lib/mediawiki.Util.js b/lib/mediawiki.Util.js
index 898a8bc..926aaff 100644
--- a/lib/mediawiki.Util.js
+++ b/lib/mediawiki.Util.js
@@ -7,6 +7,7 @@
var async = require('async'),
$ = require( './fakejquery' ),
+ request = require( 'request' ),
jsDiff = require( 'diff' ),
entities = require( 'entities' ),
TemplateRequest = require( './mediawiki.ApiRequest.js'
).TemplateRequest,
@@ -1273,6 +1274,30 @@
return s.replace(/[\^\\$*+?.()|{}\[\]\/]/g, '\\$&');
};
+/**
+ * Perform a HTTP request using the 'request' package, and retry on failures
+ *
+ * Only use on idempotent HTTP end points
+ * @param {number} retries -- the number of retries to attempt
+ * @param {object} paramOptions -- request options
+ * @param {function} cb -- request cb: function(error, response, body)
+ */
+Util.retryingHTTPRequest = function (retries, requestOptions, cb) {
+ var delay = 100; // start with 100ms
+ request(requestOptions, function(error, response, body) {
+ if (error) {
+ if (retries--) {
+ console.error('Title fetch failure, retrying in
30 seconds:', error);
+ setTimeout(function() { request(requestOptions,
cb); }, delay);
+ // exponential back-off
+ delay = delay * 2;
+ return;
+ }
+ }
+ cb(error, response, body);
+ });
+};
+
if (typeof module === "object") {
module.exports.Util = Util;
diff --git a/tests/apiServer.js b/tests/apiServer.js
index dc518e5..3c9b741 100644
--- a/tests/apiServer.js
+++ b/tests/apiServer.js
@@ -42,8 +42,8 @@
startParsoidServer();
} );
- // Wait 6 seconds to make sure it has had time to start
- setTimeout( cb, 6000, serverURL );
+ // Wait 1 second to make sure it has had time to start
+ setTimeout( cb, 1000, serverURL );
};
var stopParsoidServer = function () {
diff --git a/tests/client/client.js b/tests/client/client.js
index fae58ca..8a41300 100755
--- a/tests/client/client.js
+++ b/tests/client/client.js
@@ -5,9 +5,11 @@
*/
var http = require( 'http' ),
+ request = require('request'),
qs = require( 'querystring' ),
exec = require( 'child_process' ).exec,
apiServer = require( '../apiServer.js' ),
+ Util = require('../../lib/mediawiki.Util.js').Util,
commit, ctime,
lastCommit, lastCommitTime, lastCommitCheck,
@@ -17,43 +19,37 @@
parsoidURL = config.parsoidURL,
rtTest = require( '../roundtrip-test.js' );
+
var getTitle = function( cb ) {
var requestOptions = {
- host: config.server.host,
- port: config.server.port,
- path: '/title?commit=' + commit + '&ctime=' +
encodeURIComponent( ctime ),
+ uri: config.server.host + ':' +
+ config.server.port + '/title?commit=' + commit +
'&ctime=' + encodeURIComponent( ctime ),
method: 'GET'
+ },
+ retries = 10;
+
+ var callback = function ( error, response, body ) {
+ if (error) {
+ setTimeout( function () { cb( 'start' ); }, 15000 );
+ }
+
+ var resp;
+ switch ( response.statusCode ) {
+ case 200:
+ resp = JSON.parse( body );
+ cb( 'runTest', resp.prefix, resp.title );
+ break;
+ case 404:
+ console.log( 'The server doesn\'t have any work
for us right now, waiting half a minute....' );
+ setTimeout( function () { cb( 'start' ); },
30000 );
+ break;
+ default:
+ console.log( 'There was some error (' +
response.statusCode + '), but that is fine. Waiting 15 seconds to resume....' );
+ setTimeout( function () { cb( 'start' ); },
15000 );
+ }
};
- var callback = function ( res ) {
- var data = [];
-
- res.setEncoding( 'utf8' );
- res.on( 'data', function ( chunk ) {
- if(res.statusCode === 200) {
- data.push( chunk );
- }
- } );
-
- res.on( 'end', function () {
- var resp;
- switch ( res.statusCode ) {
- case 200:
- resp = JSON.parse( data.join( '' ) );
- cb( 'runTest', resp.prefix, resp.title
);
- break;
- case 404:
- console.log( 'The server doesn\'t have
any work for us right now, waiting half a minute....' );
- setTimeout( function () { cb( 'start'
); }, 30000 );
- break;
- default:
- console.log( 'There was some error (' +
res.statusCode + '), but that is fine. Waiting 15 seconds to resume....' );
- setTimeout( function () { cb( 'start'
); }, 15000 );
- }
- } );
- };
-
- http.get( requestOptions, callback );
+ Util.retryingHTTPRequest(10, requestOptions, callback );
};
var runTest = function( cb, prefix, title ) {
diff --git a/tests/roundtrip-test.js b/tests/roundtrip-test.js
index 48ce192..3e35966 100755
--- a/tests/roundtrip-test.js
+++ b/tests/roundtrip-test.js
@@ -418,7 +418,7 @@
form: data
};
- var req = request( options, function( err, res, body ) {
+ Util.retryingHTTPRequest( 10, options, function( err, res, body ) {
if ( err || res.statusCode !== 200 ) {
cb( err, null );
} else {
--
To view, visit https://gerrit.wikimedia.org/r/103486
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I83b5a266f9db4ce992aa06598ba97fd8f9597b3d
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: GWicke <gwi...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
