Memeht has uploaded a new change for review.
https://gerrit.wikimedia.org/r/186219
Change subject: Added performance instrumentation to capture counts of and
timings for html2wt/wt2html.
......................................................................
Added performance instrumentation to capture counts of and timings for
html2wt/wt2html.
Change-Id: Ifbe1b7191fd17830074672217a33f777131f8c37
---
M api/routes.js
M lib/mediawiki.SelectiveSerializer.js
M lib/mediawiki.Util.js
M package.json
4 files changed, 79 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/19/186219/1
diff --git a/api/routes.js b/api/routes.js
index cd6d61f..646532b 100644
--- a/api/routes.js
+++ b/api/routes.js
@@ -21,10 +21,13 @@
LogData = require( mp + 'LogData.js' ).LogData,
DU = require( mp + 'mediawiki.DOMUtils.js' ).DOMUtils,
ApiRequest = require( mp + 'mediawiki.ApiRequest.js' ),
- Diff = require( mp + 'mediawiki.Diff.js' ).Diff;
+ Diff = require( mp + 'mediawiki.Diff.js' ).Diff,
+ rbUtil = require( mp + './mediawiki.Util.js' ).rbUtil;
var ParsoidCacheRequest = ApiRequest.ParsoidCacheRequest,
TemplateRequest = ApiRequest.TemplateRequest;
+
+var GetTiming = new rbUtil.StatsD( 'statsd.eqiad.wmnet', 8125 );
module.exports = function( parsoidConfig ) {
@@ -663,7 +666,9 @@
routes.get_article = function( req, res ) {
// Regular article parsing
+ GetTiming.startTimer( 'timewt2htmlparsing' );
wt2html( req, res );
+ GetTiming.stopTimer( 'timewt2htmlparsing' );
};
routes.post_article = function( req, res ) {
@@ -673,7 +678,9 @@
wt2html( req, res, body.wt );
} else {
// Regular and form-based article serialization
+ GetTiming.startTimer( 'time2html2wtserialization' );
html2wt( req, res, body.html || body.content || '' );
+ GetTiming.stopTimer( 'time2html2wtserialization' );
}
};
diff --git a/lib/mediawiki.SelectiveSerializer.js
b/lib/mediawiki.SelectiveSerializer.js
index 7128d63..9c23de7 100644
--- a/lib/mediawiki.SelectiveSerializer.js
+++ b/lib/mediawiki.SelectiveSerializer.js
@@ -13,8 +13,10 @@
ParserPipelineFactory =
require('./mediawiki.parser.js').ParserPipelineFactory,
DOMDiff = require('./mediawiki.DOMDiff.js').DOMDiff,
ParsoidCacheRequest =
require('./mediawiki.ApiRequest.js').ParsoidCacheRequest,
- async = require('async');
+ async = require('async'),
+ rbUtil = require( './mediawiki.Util.js' ).rbUtil;
+var GetTiming = new rbUtil.StatsD( 'statsd.eqiad.wmnet', 8125 );
/**
* @class
* @constructor
@@ -57,9 +59,15 @@
SSP.doSerializeDOM = function( err, doc, cb, finalcb ) {
var self = this;
+ GetTiming.startTimer( 'time2FullSerialization' );
+ GetTiming.startTimer( 'time2PartialSerialization' );
+
if ( err || (!this.env.page.dom && !this.env.page.domdiff) ||
!this.env.page.src) {
// If there's no old source, fall back to non-selective
serialization.
this.wts.serializeDOM( doc, cb, false, finalcb );
+
+ GetTiming.stopTimer( 'time2FullSerialization' );
+ GetTiming.count( 'numFullSerializations', '' );
} else {
// Use provided diff-marked DOM (used during testing)
// or generate one (used in production)
@@ -81,10 +89,15 @@
// Call the WikitextSerializer to do our bidding
this.wts.serializeDOM( doc, cb, true, finalcb );
+
+ GetTiming.stopTimer( 'time2PartialSerialization' );
+ GetTiming.count( 'countPartialSerializations', '' );
} else {
// Nothing was modified, just re-use the original source
cb( this.env.page.src );
finalcb();
+
+ GetTiming.count( 'countNoSerializations', '' );
}
}
};
diff --git a/lib/mediawiki.Util.js b/lib/mediawiki.Util.js
index 3993e77..3b82e5a 100644
--- a/lib/mediawiki.Util.js
+++ b/lib/mediawiki.Util.js
@@ -9,6 +9,7 @@
var async = require('async'),
request = require( 'request' ),
entities = require( 'entities' ),
+ StatsD = require('node-txstatsd'),
TemplateRequest = require( './mediawiki.ApiRequest.js'
).TemplateRequest,
Consts = require('./mediawiki.wikitext.constants.js').WikitextConstants;
@@ -1472,6 +1473,60 @@
/* Magic words masquerading as templates. */
Util.magicMasqs = new Set(["defaultsort", "displaytitle"]);
+var rbUtil = {};
+
+// Timer that can report to StatsD
+rbUtil.StatsD = function ( statsdHost, statsdPort ) {
+ var timers = {};
+ var statsd = new StatsD(
+ statsdHost,
+ statsdPort,
+ 'restbase.routes.',
+ '',
+ true, // is txstatsd
+ false, // Don't globalize, we're doing that here
+ true // Do cache DNS queries
+ );
+
+ function makeName(name) {
+ // See https://github.com/etsy/statsd/issues/110
+ // Only [\w_.-] allowed, with '.' being the hierarchy separator.
+ return name.replace( /[^\/a-zA-Z0-9\.\-]/g, '-' )
+ .replace(/\//g, '_');
+ }
+
+ this.startTimer = function (name) {
+ timers[name] = new Date();
+ };
+
+ this.stopTimer = function (name, suffix) {
+ var startTime = timers[name];
+ if (!startTime) {
+ throw new Error('Tried to stop a timer that does not exist: ' +
name);
+ }
+ var delta = new Date() - startTime;
+
+ name = makeName(name);
+ if (Array.isArray(suffix)) {
+ // Send several timings at once
+ var stats = suffix.map(function(s) {
+ return name + (s ? '.' + s : '');
+ });
+ statsd.sendAll(stats, delta, 'ms');
+ } else {
+ suffix = suffix ? '.' + suffix : '';
+ statsd.timing(makeName(name) + suffix, delta);
+ }
+ return delta;
+ };
+
+ this.count = function (name, suffix) {
+ suffix = suffix ? '.' + suffix : '';
+ statsd.increment(makeName(name) + suffix);
+ };
+};
+
if (typeof module === "object") {
module.exports.Util = Util;
+ module.exports.rbUtil = rbUtil;
}
diff --git a/package.json b/package.json
index b21cf63..fcacf40 100644
--- a/package.json
+++ b/package.json
@@ -20,7 +20,8 @@
"prfun": "~1.0.2",
"request": "~2.40.0",
"simplediff": "~0.1.1",
- "yargs": "~1.3.1"
+ "yargs": "~1.3.1",
+ "node-txstatsd": "~0.1.5"
},
"devDependencies": {
"chai": "~1.9.1",
--
To view, visit https://gerrit.wikimedia.org/r/186219
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifbe1b7191fd17830074672217a33f777131f8c37
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Memeht <eco...@gmail.com>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
