Subramanya Sastry has uploaded a new change for review.
https://gerrit.wikimedia.org/r/169226
Change subject: Revert "Revert "Add a 5 minute timeout for cpu hogs""
......................................................................
Revert "Revert "Add a 5 minute timeout for cpu hogs""
This reverts commit eb953d0dbd15bb7550688fad720da20214c59d63 and re-enables
84ad6b2e on master now that we have deployed.
Change-Id: I7ba672eb7a1a7aae6ac92b44fdab837a354064c5
---
M api/routes.js
M api/server.js
2 files changed, 68 insertions(+), 22 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/26/169226/1
diff --git a/api/routes.js b/api/routes.js
index eb6e873..b6360c4 100644
--- a/api/routes.js
+++ b/api/routes.js
@@ -8,6 +8,7 @@
url = require('url'),
util = require('util'),
childProc = require('child_process'),
+ cluster = require('cluster'),
domino = require('domino'),
pkg = require('../package.json'),
apiUtils = require('./utils');
@@ -33,6 +34,26 @@
// Helpers
+
+var CPU_TIMEOUT = 5 * 60 * 1000; // 5 minutes
+var cpuTimeout = function( p ) {
+ return new Promise(function( resolve, reject ) {
+ if ( cluster.isMaster ) {
+ return p.then( resolve, reject );
+ }
+ process.send({
+ type: "timeout",
+ timeout: CPU_TIMEOUT
+ });
+ p.then( function() {
+ process.send({
+ type: "timeout",
+ done: true
+ });
+ resolve.apply(this, arguments);
+ }, reject );
+ });
+};
var promiseTemplateReq = function( env, target, oldid ) {
return new Promise(function( resolve, reject ) {
@@ -169,7 +190,7 @@
}
var out = [];
- return new Promise(function( resolve, reject ) {
+ var p = new Promise(function( resolve, reject ) {
if ( !env.conf.parsoid.fetchWT ) {
return resolve();
}
@@ -195,7 +216,8 @@
apiUtils.setHeader(res, env, 'Content-Type', 'text/x-mediawiki;
charset=UTF-8');
apiUtils.setHeader(res, env, 'X-Parsoid-Performance',
env.getPerformanceHeader());
apiUtils.endResponse(res, env, out.join(''));
- }).catch(function( err ) {
+ });
+ return cpuTimeout(p).catch(function( err ) {
env.log("fatal/request", err);
});
};
@@ -209,9 +231,6 @@
if ( wt ) {
wt = wt.replace(/\r/g, '');
}
-
- // Set the timeout to 600 seconds..
- req.connection.setTimeout( 600 * 1000 );
if ( env.conf.parsoid.allowCORS ) {
// allow cross-domain requests (CORS) so that parsoid service
@@ -311,7 +330,7 @@
p = p.then( redirectToOldid );
}
- return p.catch(function( err ) {
+ return cpuTimeout(p).catch(function( err ) {
env.log("fatal/request", err);
});
};
@@ -455,18 +474,17 @@
var env = res.local('env');
var target = env.resolveTitle( env.normalizeTitle( env.page.name ), ''
);
- req.connection.setTimeout(300 * 1000);
-
var oldid = null;
if ( req.query.oldid ) {
oldid = req.query.oldid;
}
- promiseTemplateReq( env, target, oldid ).then(
+ var p = promiseTemplateReq( env, target, oldid ).then(
parse.bind( null, env, req, res )
).then(
roundTripDiff.bind( null, env, req, res, false )
- ).catch(function(err) {
+ );
+ cpuTimeout(p).catch(function(err) {
env.log("fatal/request", err);
});
};
@@ -482,13 +500,14 @@
oldid = req.query.oldid;
}
- promiseTemplateReq( env, target, oldid ).then(
+ var p = promiseTemplateReq( env, target, oldid ).then(
parse.bind( null, env, req, res )
).then(function( doc ) {
// strip newlines from the html
var html = doc.innerHTML.replace(/[\r\n]/g, '');
return roundTripDiff( env, req, res, false, DU.parseHTML(html)
);
- }).catch(function(err) {
+ });
+ cpuTimeout(p).catch(function(err) {
env.log("fatal/request", err);
});
};
@@ -503,14 +522,15 @@
oldid = req.query.oldid;
}
- promiseTemplateReq( env, target, oldid ).then(
+ var p = promiseTemplateReq( env, target, oldid ).then(
parse.bind( null, env, req, res )
).then(function( doc ) {
doc = DU.parseHTML( DU.serializeNode(doc) );
var comment = doc.createComment('rtSelserEditTestComment');
doc.body.appendChild(comment);
return roundTripDiff( env, req, res, true, doc );
- }).catch(function(err) {
+ });
+ cpuTimeout(p).catch(function(err) {
env.log("fatal/request", err);
});
};
diff --git a/api/server.js b/api/server.js
index ed7215d..66b9b99 100755
--- a/api/server.js
+++ b/api/server.js
@@ -19,6 +19,8 @@
*/
"use strict";
+require('es6-shim');
+
var cluster = require('cluster'),
path = require('path'),
// process arguments
@@ -61,19 +63,43 @@
process.exit( 0 );
}
+ var timeoutHandler, timeouts = new Map();
+ var spawn = function( pid ) {
+ if ( pid ) {
+ timeouts.delete( pid );
+ }
+ var worker = cluster.fork();
+ worker.on('message', timeoutHandler.bind(null, worker));
+ };
+
+ // Kill cpu hogs
+ timeoutHandler = function( worker, msg ) {
+ if ( msg.type !== "timeout" ) { return; }
+ if ( msg.done ) {
+ clearTimeout( timeouts.get( worker.process.pid ) );
+ timeouts.delete( worker.process.pid );
+ } else if ( msg.timeout ) {
+ var pid = worker.process.pid;
+ timeouts.set(pid, setTimeout(function() {
+ console.log("Cpu timeout; killing worker %s.",
pid);
+ worker.kill();
+ spawn( pid );
+ }, msg.timeout));
+ }
+ };
+
// Fork workers.
- console.log('master(' + process.pid + ') initializing ' +
- argv.n + ' workers');
+ var worker;
+ console.log('master(%s) initializing %s workers', process.pid, argv.n);
for (var i = 0; i < argv.n; i++) {
- cluster.fork();
+ spawn();
}
cluster.on('exit', function(worker, code, signal) {
- if (!worker.suicide) {
- var exitCode = worker.process.exitCode;
- console.log('worker', worker.process.pid,
- 'died ('+exitCode+'), restarting.');
- cluster.fork();
+ if ( !worker.suicide ) {
+ var pid = worker.process.pid;
+ console.log('worker %s died (%s), restarting.', pid,
code);
+ spawn( pid );
}
});
--
To view, visit https://gerrit.wikimedia.org/r/169226
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7ba672eb7a1a7aae6ac92b44fdab837a354064c5
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Subramanya Sastry <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
